# /// script # requires-python = ">=3.10,<3.14" # dependencies = [ # "accelerate", # "bitsandbytes>=0.43.0", # "datasets==4.3.0", # "hf_transfer", # "huggingface_hub>=0.34.0", # "marimo", # "peft", # "protobuf", # "sentencepiece", # "torchao>=0.16.0", # "transformers==4.56.2", # "triton>=3.2.0", # "trl==0.22.2", # "unsloth @ git+https://github.com/unslothai/unsloth", # "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo", # ] # # [tool.uv] # no-build-package = [ # "bitsandbytes", # "triton", # "vllm", # "xformers", # ] # /// import marimo __generated_with = "0.23.8" app = marimo.App() @app.cell def _(): import marimo as mo return (mo,) @app.cell(hide_code=True) def _(mo): mo.md(r""" To run this notebook, hit the **▶ Run all** button in the bottom-right corner - or use `Ctrl/Cmd + Shift + R`.

Join Discord if you need help + ⭐ Star us on Github ⭐

To install Unsloth on your local device, follow [our guide](https://unsloth.ai/docs/get-started/install). This notebook is licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme). You will learn how to do [data prep](#Data), how to [train](#Train), how to [run the model](#Inference), & how to save it """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### News """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Introducing **Unsloth Studio** - a new open source, no-code web UI to train and run LLMs. [Blog](https://unsloth.ai/docs/new/studio) • [Notebook](https://github.com/unslothai/unsloth/blob/main/studio/Unsloth_Studio_Colab.ipynb)

_{Train models — no code needed}

_{Run GGUF models on Mac, Windows & Linux}

Train MoEs - DeepSeek, GLM, Qwen and gpt-oss 12x faster with 35% less VRAM. [Blog](https://unsloth.ai/docs/new/faster-moe) Ultra Long-Context Reinforcement Learning is here with 7x more context windows! [Blog](https://unsloth.ai/docs/new/grpo-long-context) New in Reinforcement Learning: [FP8 RL](https://unsloth.ai/docs/new/fp8-reinforcement-learning) • [Vision RL](https://unsloth.ai/docs/new/vision-reinforcement-learning-vlm-rl) • [Standby](https://unsloth.ai/docs/basics/memory-efficient-rl) • [gpt-oss RL](https://unsloth.ai/docs/new/gpt-oss-reinforcement-learning) Visit our docs for all our [model uploads](https://unsloth.ai/docs/get-started/unsloth-model-catalog) and [notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks). """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Unsloth """) return @app.cell def _(): from unsloth import FastSentenceTransformer fourbit_models = [ "unsloth/all-MiniLM-L6-v2", "unsloth/embeddinggemma-300m", "unsloth/Qwen3-Embedding-4B", "unsloth/Qwen3-Embedding-0.6B", "unsloth/all-mpnet-base-v2", "unsloth/gte-modernbert-base", "unsloth/bge-m3", ] # More models at https://huggingface.co/unsloth model = FastSentenceTransformer.from_pretrained( model_name="unsloth/embeddinggemma-300m", max_seq_length=1024, # Choose any for long context! full_finetuning=False, # [NEW!] We have full finetuning now! ) return FastSentenceTransformer, model @app.cell(hide_code=True) def _(mo): mo.md(r""" We now add LoRA adapters so we only need to update a small amount of parameters! """) return @app.cell def _(FastSentenceTransformer, model): model_1 = FastSentenceTransformer.get_peft_model( model, r=32, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128 target_modules=[ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", ], lora_alpha=64, lora_dropout=0, # Supports any, but = 0 is optimized bias="none", # Supports any, but = "none" is optimized use_gradient_checkpointing="unsloth", # True or "unsloth" for very long context random_state=3407, use_rslora=False, # We support rank stabilized LoRA loftq_config=None, # And LoftQ task_type="FEATURE_EXTRACTION", ) return (model_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Data Prep We now use the ``tomaarsen/miriad-4.4M-split`` dataset, a large-scale collection of 4.4 million medical question-answer pairs distilled from peer-reviewed biomedical literature. To maintain efficiency, we use data streaming to ingest a subset of 10,000 training samples and 2,000 evaluation samples. """) return @app.cell def _(): from datasets import load_dataset, Dataset stream_train = list( load_dataset("tomaarsen/miriad-4.4M-split", split="train", streaming=True).take( 10000 ) ) stream_eval = list( load_dataset("tomaarsen/miriad-4.4M-split", split="eval", streaming=True).take( 2000 ) ) train_dataset = Dataset.from_generator(lambda: (yield from stream_train)) eval_dataset = Dataset.from_generator(lambda: (yield from stream_eval)) return eval_dataset, train_dataset @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's take a look at the dataset structure: """) return @app.cell def _(train_dataset): train_dataset[0] return @app.cell(hide_code=True) def _(mo): mo.md(r""" ## Baseline Performance ### Now after fine-tuning lets evaluate the model! """) return @app.cell def _(eval_dataset, model_1, train_dataset): import torch from sentence_transformers.evaluation import InformationRetrievalEvaluator queries = dict(enumerate(eval_dataset["question"])) corpus = dict( enumerate( list(eval_dataset["passage_text"]) + train_dataset["passage_text"][:2000] ) ) relevant_docs = {idx: [idx] for idx in queries} evaluator = InformationRetrievalEvaluator( # Optional, will make training slower queries=queries, corpus=corpus, relevant_docs=relevant_docs, show_progress_bar=False, batch_size=64, ) with torch.autocast( device_type="cuda", dtype=model_1.dtype, enabled=model_1.dtype != torch.float16 ): print(evaluator(model_1)) return evaluator, torch @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Train the model Now let's train our model. We use `MultipleNegativesRankingLoss` This loss function uses other positives in the same batch as negative examples, which is efficient for contrastive learning. We do 30 steps to speed things up, but you can set `num_train_epochs=1` for a full run, and turn off `max_steps=None`. """) return @app.cell def _(eval_dataset, evaluator, model_1, train_dataset): from sentence_transformers import ( SentenceTransformerTrainer, SentenceTransformerTrainingArguments, losses, ) from sentence_transformers.training_args import BatchSamplers from unsloth import is_bf16_supported loss = losses.MultipleNegativesRankingLoss(model_1) # This will use other positives in the same batch as negative examples trainer = SentenceTransformerTrainer( model=model_1, train_dataset=train_dataset, eval_dataset=eval_dataset, loss=loss, args=SentenceTransformerTrainingArguments( max_steps=30, per_device_train_batch_size=64, per_device_eval_batch_size=64, gradient_accumulation_steps=2, # Use GA to mimic batch size! learning_rate=2e-05, logging_steps=5, warmup_ratio=0.03, prompts={ # Map training column names to model prompts "question": model_1.prompts["query"], "passage_text": model_1.prompts["document"], }, report_to="none", # Use TrackIO/WandB etc eval_strategy="steps", eval_steps=5, bf16=is_bf16_supported(), output_dir="output", lr_scheduler_type="linear", batch_sampler=BatchSamplers.NO_DUPLICATES, ), evaluator=evaluator, # Optional, will make training slower ) return (trainer,) @app.cell def _(torch): # @title Show current memory stats gpu_stats = torch.cuda.get_device_properties(0) start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) print(f"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") print(f"{start_gpu_memory} GB of memory reserved.") return max_memory, start_gpu_memory @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's train the model! To resume a training run, set `trainer.train(resume_from_checkpoint = True)` """) return @app.cell def _(trainer): trainer_stats = trainer.train() return (trainer_stats,) @app.cell def _(max_memory, start_gpu_memory, torch, trainer_stats): # @title Show final memory and time stats used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) used_memory_for_lora = round(used_memory - start_gpu_memory, 3) used_percentage = round(used_memory / max_memory * 100, 3) lora_percentage = round(used_memory_for_lora / max_memory * 100, 3) print(f"{trainer_stats.metrics['train_runtime']} seconds used for training.") print( f"{round(trainer_stats.metrics['train_runtime'] / 60, 2)} minutes used for training." ) print(f"Peak reserved memory = {used_memory} GB.") print(f"Peak reserved memory for training = {used_memory_for_lora} GB.") print(f"Peak reserved memory % of max memory = {used_percentage} %.") print(f"Peak reserved memory for training % of max memory = {lora_percentage} %.") return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Now after fine-tuning lets evaluate the model! """) return @app.cell def _(evaluator, model_1, torch): with torch.autocast( device_type="cuda", dtype=model_1.dtype, enabled=model_1.dtype != torch.float16 ): print(evaluator(model_1)) return @app.cell(hide_code=True) def _(mo): mo.md(r""" In just 30 steps, the model's Accuracy@1 increased from 0.871 to 0.883, demonstrating how quickly the model adapts to the specific dataset. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Inference Let's run the model after training to see the improvements! """) return @app.cell def _(model_1): query = "Patient presents with sharp chest pain that improves when leaning forward." candidates = [ "Acute Pericarditis often involves pleuritic chest pain relieved by sitting up and leaning forward.", "Myocardial Infarction typically presents with crushing substernal pressure and radiation to the left arm.", "Pneumothorax is characterized by sudden onset shortness of breath and unilateral chest pain.", "Gastroesophageal Reflux Disease (GERD) causes burning retrosternal pain usually after meals.", ] query_emb = model_1.encode(query, convert_to_tensor=True) candidate_embs = model_1.encode(candidates, convert_to_tensor=True) similarities = model_1.similarity(query_emb, candidate_embs) ranking = similarities.argsort(descending=True)[0] for idx in ranking.tolist(): score = similarities[0][idx].item() text = candidates[idx] print(f"{score:.4f} | {text}") return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Saving, loading finetuned models To save the final model as LoRA adapters, either use Hugging Face's `push_to_hub` for an online save or `save_pretrained` for a local save. **[NOTE]** This ONLY saves the LoRA adapters, and not the full model. To save to 16bit, scroll down! """) return @app.cell def _(model_1): model_1.save_pretrained("embeddinggemma_lora") model_1.tokenizer.save_pretrained("embeddinggemma_lora") return @app.cell(hide_code=True) def _(mo): mo.md(r""" Now if you want to load the LoRA adapters we just saved for inference, set `False` to `True`: """) return @app.cell def _(): if False: from unsloth import FastSentenceTransformer as _FastSentenceTransformer _model = _FastSentenceTransformer.from_pretrained("model") return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Saving to float16 for VLLM We also support saving to `float16` directly. Select `merged_16bit` for float16 or `merged_4bit` for int4. We also allow `lora` adapters as a fallback. Use `push_to_hub_merged` to upload to your Hugging Face account! You can go to https://huggingface.co/settings/tokens for your personal tokens. See [our docs](https://unsloth.ai/docs/basics/inference-and-deployment) for more deployment options. """) return @app.cell def _(model_1): # Merge to 16bit if False: model_1.save_pretrained_merged( "embeddinggemma_finetune_16bit", tokenizer=model_1.tokenizer, save_method="merged_16bit", ) if False: # Pushing to HF Hub model_1.push_to_hub_merged( "HF_USERNAME/embeddinggemma_finetune_16bit", tokenizer=model_1.tokenizer, save_method="merged_16bit", token="YOUR_HF_TOKEN", # Get a token at https://huggingface.co/settings/tokens ) if False: # Just LoRA adapters model_1.save_pretrained("embeddinggemma_lora") if False: model_1.push_to_hub( "HF_USERNAME/embeddinggemma_lora", token="YOUR_HF_TOKEN" ) # Pushing to HF Hub return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### GGUF / llama.cpp Conversion To save to `GGUF` / `llama.cpp`, we support it natively now! We clone `llama.cpp` and we default save it to `q8_0`. We allow all methods like `q4_k_m`. Use `save_pretrained_gguf` for local saving and `push_to_hub_gguf` for uploading to HF. Some supported quant methods (full list on our [docs page](https://unsloth.ai/docs/basics/inference-and-deployment/saving-to-gguf)): * `q8_0` - Fast conversion. High resource use, but generally acceptable. * `q4_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_K. * `q5_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_K. """) return @app.cell def _(model_1): # Save to 8bit Q8_0 if False: model_1.save_pretrained_gguf("embeddinggemma_finetune") if False: model_1.push_to_hub_gguf( "HF_USERNAME/embeddinggemma_finetune", token="YOUR_HF_TOKEN" ) if False: # Save to 16bit GGUF model_1.save_pretrained_gguf( "embeddinggemma_finetune", quantization_method="f16" ) if False: model_1.push_to_hub_gguf( "HF_USERNAME/embeddinggemma_finetune", quantization_method="f16", token="YOUR_HF_TOKEN", # Get a token at https://huggingface.co/settings/tokens ) # Pushing to HF Hub if False: model_1.save_pretrained_gguf( "embeddinggemma_finetune", quantization_method="q4_k_m" ) # Save to q4_k_m GGUF if False: model_1.push_to_hub_gguf( "HF_USERNAME/embeddinggemma_finetune", quantization_method="q4_k_m", token="YOUR_HF_TOKEN", # Get a token at https://huggingface.co/settings/tokens ) if False: # Pushing to HF Hub # Save to multiple GGUF options - much faster if you want multiple! model_1.push_to_hub_gguf( "HF_USERNAME/embeddinggemma_finetune", quantization_method=["q4_k_m", "q8_0", "q5_k_m"], token="YOUR_HF_TOKEN", # Get a token at https://huggingface.co/settings/tokens ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord! Some other resources: 1. Looking to use Unsloth locally? Read our [Installation Guide](https://unsloth.ai/docs/get-started/install) for details on installing Unsloth on Windows, Docker, AMD, Intel GPUs. 2. Learn how to do Reinforcement Learning with our [RL Guide and notebooks](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide). 3. Read our guides and notebooks for [Text-to-speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning) and [vision](https://unsloth.ai/docs/basics/vision-fine-tuning) model support. 4. Explore our [LLM Tutorials Directory](https://unsloth.ai/docs/models/tutorials-how-to-fine-tune-and-run-llms) to find dedicated guides for each model. 5. Need help with Inference? Read our [Inference & Deployment page](https://unsloth.ai/docs/basics/inference-and-deployment) for details on using vLLM, llama.cpp, Ollama etc.

Join Discord if you need help + ⭐️ Star us on Github ⭐️ This notebook and all Unsloth notebooks are licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme)

""") return if __name__ == "__main__": app.run()