# /// script # requires-python = ">=3.10,<3.14" # dependencies = [ # "accelerate", # "bitsandbytes>=0.43.0", # "datasets==4.3.0", # "hf_transfer", # "huggingface_hub>=0.34.0", # "marimo", # "peft", # "protobuf", # "sentencepiece", # "torchao>=0.16.0", # "transformers==4.56.2", # "triton>=3.2.0", # "trl==0.22.2", # "unsloth @ git+https://github.com/unslothai/unsloth", # "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo", # ] # # [tool.uv] # no-build-package = [ # "bitsandbytes", # "triton", # "vllm", # "xformers", # ] # /// import marimo __generated_with = "0.23.8" app = marimo.App() @app.cell def _(): import marimo as mo return (mo,) @app.cell(hide_code=True) def _(mo): mo.md(r"""

""") return @app.cell(hide_code=True) def _(mo): mo.md(r""" To run this notebook, hit the **▶ Run all** button in the bottom-right corner - or use `Ctrl/Cmd + Shift + R`.

Join Discord if you need help + ⭐ Star us on Github ⭐

To install Unsloth on your local device, follow [our guide](https://unsloth.ai/docs/get-started/install). This notebook is licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme). You will learn how to do [data prep](#Data), how to [train](#Train), how to [run the model](#Inference), & how to save it """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### News """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Introducing **Unsloth Studio** - a new open source, no-code web UI to train and run LLMs. [Blog](https://unsloth.ai/docs/new/studio) • [Notebook](https://github.com/unslothai/unsloth/blob/main/studio/Unsloth_Studio_Colab.ipynb)

_{Train models — no code needed}

_{Run GGUF models on Mac, Windows & Linux}

Train MoEs - DeepSeek, GLM, Qwen and gpt-oss 12x faster with 35% less VRAM. [Blog](https://unsloth.ai/docs/new/faster-moe) Ultra Long-Context Reinforcement Learning is here with 7x more context windows! [Blog](https://unsloth.ai/docs/new/grpo-long-context) New in Reinforcement Learning: [FP8 RL](https://unsloth.ai/docs/new/fp8-reinforcement-learning) • [Vision RL](https://unsloth.ai/docs/new/vision-reinforcement-learning-vlm-rl) • [Standby](https://unsloth.ai/docs/basics/memory-efficient-rl) • [gpt-oss RL](https://unsloth.ai/docs/new/gpt-oss-reinforcement-learning) Visit our docs for all our [model uploads](https://unsloth.ai/docs/get-started/unsloth-model-catalog) and [notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks). """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Unsloth """) return @app.cell def _(): from unsloth import FastLanguageModel import torch max_seq_length = 4096 # Can choose any sequence length! fourbit_models = [ # 4bit Gemma 3 dynamic quants for superior accuracy and low memory use "unsloth/gemma-3-270m-it-unsloth-bnb-4bit", "unsloth/gemma-3-1b-it-unsloth-bnb-4bit", "unsloth/gemma-3-4b-it-unsloth-bnb-4bit", "unsloth/gemma-3-12b-it-unsloth-bnb-4bit", "unsloth/gemma-3-27b-it-unsloth-bnb-4bit", # Function Gemma models "unsloth/functiongemma-270m-it", "unsloth/functiongemma-270m-it-unsloth-bnb-4bit", "unsloth/functiongemma-270m-it-bnb-4bit", ] # More models at https://huggingface.co/unsloth model, tokenizer = FastLanguageModel.from_pretrained( model_name="unsloth/functiongemma-270m-it", # YOUR MODEL YOU USED FOR TRAINING max_seq_length=max_seq_length, # Choose any for long context! load_in_4bit=False, # 4 bit quantization to reduce memory load_in_8bit=False, # [NEW!] A bit more accurate, uses 2x memory load_in_16bit=True, # [NEW!] Enables 16bit LoRA full_finetuning=False, # [NEW!] We have full finetuning now! # token = "YOUR_HF_TOKEN", # HF Token for gated models ) return FastLanguageModel, model, tokenizer, torch @app.cell(hide_code=True) def _(mo): mo.md(r""" We now add LoRA adapters so we only need to update 1 to 10% of all parameters! """) return @app.cell def _(FastLanguageModel, model): model_1 = FastLanguageModel.get_peft_model( model, r=16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128 target_modules=[ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", ], lora_alpha=16, lora_dropout=0, # Supports any, but = 0 is optimized bias="none", # Supports any, but = "none" is optimized use_gradient_checkpointing="unsloth", # True or "unsloth" for very long context random_state=3407, use_rslora=False, # We support rank stabilized LoRA loftq_config=None, # And LoftQ ) return (model_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Data Prep We now use Google's Mobile Actions dataset, which contains a list of tools a phone could call, like making a calendar invite, setting an alarm etc. """) return @app.cell def _(): from datasets import load_dataset dataset = load_dataset("google/mobile-actions", split="train") return (dataset,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's take a look at the first row for messages and the tools present. We see the person is asking `Please set a reminder for a "Team Sync Meeting" this Friday, June 6th, 2025, at 2 PM.` so we expect the calendar or reminder tool to be used """) return @app.cell def _(dataset): dataset[0]["messages"] return @app.cell def _(dataset): dataset[0]["tools"] return @app.cell(hide_code=True) def _(mo): mo.md(r""" We can then apply FunctionGemma's chat template: """) return @app.cell def _(dataset, tokenizer): tokenizer.apply_chat_template( dataset[0]["messages"], tools=dataset[0]["tools"], tokenize=False, add_generation_prompt=False, # Must add for generation ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" We then create a dataset of just these mapped sequences. """) return @app.cell def _(dataset, tokenizer): def process_dataset(row, tokenizer): text = tokenizer.apply_chat_template( row["messages"], tools=row["tools"], tokenize=False, add_generation_prompt=False, # Must add for generation ) return {"text": text} dataset_1 = dataset.map(process_dataset, fn_kwargs={"tokenizer": tokenizer}) return (dataset_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Look at the first row """) return @app.cell def _(dataset_1): dataset_1[0]["text"] return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Train the model Now let's train our model. We do 100 steps to speed things up, but you can set `num_train_epochs=1` for a full run, and turn off `max_steps=None`. """) return @app.cell def _(dataset_1, model_1, tokenizer): # We split the dataset into some training and testing split_dataset = dataset_1.train_test_split(test_size=50, shuffle=True, seed=3407) from trl import SFTTrainer, SFTConfig trainer = SFTTrainer( model=model_1, tokenizer=tokenizer, train_dataset=split_dataset["train"], eval_dataset=split_dataset["test"], # For evaluation! args=SFTConfig( dataset_text_field="text", per_device_train_batch_size=4, gradient_accumulation_steps=2, # Use GA to mimic batch size! warmup_steps=5, max_steps=100, eval_steps=2, eval_strategy="steps", learning_rate=0.0002, # Reduce to 2e-5 for long training runs logging_steps=1, optim="adamw_8bit", weight_decay=0.001, lr_scheduler_type="linear", seed=3407, report_to="none", # Use TrackIO/WandB etc ), ) return (trainer,) @app.cell(hide_code=True) def _(mo): mo.md(r""" We also mask all system and user instructions so our goal is to force the finetune to actually respond with the correct function calls. If not, the finetune might just learn the format, and not learn anything else. """) return @app.cell def _(trainer): from unsloth.chat_templates import train_on_responses_only trainer_1 = train_on_responses_only( trainer, instruction_part="user\n", response_part="model\n", ) return (trainer_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" We print the full un-masked string: """) return @app.cell def _(tokenizer, trainer_1): tokenizer.decode(trainer_1.train_dataset[100]["input_ids"]) return @app.cell(hide_code=True) def _(mo): mo.md(r""" See if we did the masking correctly - "-" is the mask """) return @app.cell def _(tokenizer, trainer_1): [ tokenizer.decode( [ tokenizer.pad_token_id if x == -100 else x for x in trainer_1.train_dataset[100]["labels"] ] ).replace(tokenizer.pad_token, "-") ] return @app.cell def _(torch): # @title Show current memory stats gpu_stats = torch.cuda.get_device_properties(0) start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) print(f"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") print(f"{start_gpu_memory} GB of memory reserved.") return max_memory, start_gpu_memory @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's train the model! To resume a training run, set `trainer.train(resume_from_checkpoint = True)` """) return @app.cell def _(trainer_1): trainer_stats = trainer_1.train() return (trainer_stats,) @app.cell def _(max_memory, start_gpu_memory, torch, trainer_stats): # @title Show final memory and time stats used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) used_memory_for_lora = round(used_memory - start_gpu_memory, 3) used_percentage = round(used_memory / max_memory * 100, 3) lora_percentage = round(used_memory_for_lora / max_memory * 100, 3) print(f"{trainer_stats.metrics['train_runtime']} seconds used for training.") print( f"{round(trainer_stats.metrics['train_runtime'] / 60, 2)} minutes used for training." ) print(f"Peak reserved memory = {used_memory} GB.") print(f"Peak reserved memory for training = {used_memory_for_lora} GB.") print(f"Peak reserved memory % of max memory = {used_percentage} %.") print(f"Peak reserved memory for training % of max memory = {lora_percentage} %.") return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Inference Let's run the model via Unsloth native inference for the first message which should create a calendar reminder """) return @app.cell def _(dataset_1): dataset_1[0]["messages"][:2] return @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's try inference: """) return @app.cell def _(dataset_1, model_1, tokenizer): text = tokenizer.apply_chat_template( dataset_1[0]["messages"][:2], tools=dataset_1[0]["tools"], tokenize=False, add_generation_prompt=True, # Must add for generation ).removeprefix("") from transformers import TextStreamer _ = model_1.generate( **tokenizer(text, return_tensors="pt").to("cuda"), max_new_tokens=1024, streamer=TextStreamer(tokenizer, skip_prompt=False), top_p=0.95, top_k=64, temperature=1.0, ) # Must add for generation return @app.cell(hide_code=True) def _(mo): mo.md(r""" It looks correct! """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Saving, loading finetuned models To save the final model as LoRA adapters, either use Hugging Face's `push_to_hub` for an online save or `save_pretrained` for a local save. **[NOTE]** This ONLY saves the LoRA adapters, and not the full model. To save to 16bit or GGUF, scroll down! """) return @app.cell def _(model_1, tokenizer): model_1.save_pretrained("functiongemma_lora") tokenizer.save_pretrained("functiongemma_lora") return @app.cell(hide_code=True) def _(mo): mo.md(r""" Now if you want to load the LoRA adapters we just saved for inference, set `False` to `True`: """) return @app.cell def _(): if False: from unsloth import FastLanguageModel as _FastLanguageModel _model, _tokenizer = _FastLanguageModel.from_pretrained( model_name="functiongemma_lora", max_seq_length=2048, load_in_4bit=False # YOUR MODEL YOU USED FOR TRAINING ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Saving to float16 for VLLM We also support saving to `float16` directly. Select `merged_16bit` for float16 or `merged_4bit` for int4. We also allow `lora` adapters as a fallback. Use `push_to_hub_merged` to upload to your Hugging Face account! You can go to https://huggingface.co/settings/tokens for your personal tokens. See [our docs](https://unsloth.ai/docs/basics/inference-and-deployment) for more deployment options. """) return @app.cell def _(model_1, tokenizer): # Merge to 16bit if False: model_1.save_pretrained_merged( "functiongemma_finetune_16bit", tokenizer, save_method="merged_16bit" ) if False: # Pushing to HF Hub model_1.push_to_hub_merged( "HF_USERNAME/functiongemma_finetune_16bit", tokenizer, save_method="merged_16bit", token="YOUR_HF_TOKEN", ) if False: # Merge to 4bit model_1.save_pretrained_merged( "functiongemma_finetune_4bit", tokenizer, save_method="merged_4bit" ) if False: model_1.push_to_hub_merged( "HF_USERNAME/functiongemma_finetune_4bit", tokenizer, save_method="merged_4bit", token="YOUR_HF_TOKEN", ) # Pushing to HF Hub if False: model_1.save_pretrained("functiongemma_lora") # Just LoRA adapters tokenizer.save_pretrained("functiongemma_lora") if False: model_1.push_to_hub("HF_USERNAME/functiongemma_lora", token="YOUR_HF_TOKEN") tokenizer.push_to_hub( "HF_USERNAME/functiongemma_lora", token="YOUR_HF_TOKEN" ) # Pushing to HF Hub return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### GGUF / llama.cpp Conversion To save to `GGUF` / `llama.cpp`, we support it natively now for all models! For now, you can convert easily to `Q8_0, F16 or BF16` precision. `Q4_K_M` for 4bit will come later! """) return @app.cell def _(model_1, tokenizer): if False: # Change to True to save to GGUF model_1.save_pretrained_gguf( "functiongemma_finetune", tokenizer, quantization_method="Q8_0" ) # For now only Q8_0, BF16, F16 supported return @app.cell(hide_code=True) def _(mo): mo.md(r""" Likewise, if you want to instead push to GGUF to your Hugging Face account, set `if False` to `if True` and add your Hugging Face token and upload location! """) return @app.cell def _(model_1, tokenizer): if False: # Change to True to upload GGUF model_1.push_to_hub_gguf( "HF_ACCOUNT/functiongemma_finetune", tokenizer, quantization_method="Q8_0", # For now only Q8_0, BF16, F16 supported token="YOUR_HF_TOKEN", ) # Only Q8_0, BF16, F16 supported return @app.cell(hide_code=True) def _(mo): mo.md(r""" Now, use the `functiongemma-finetune.gguf` file or `functiongemma-finetune-Q4_K_M.gguf` file in llama.cpp. And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord! Some other resources: 1. Train your own reasoning model - Llama GRPO notebook [Open in molab](https://github.com/unslothai/notebooks/blob/main/nb/Llama3.1_(8B)-GRPO.ipynb) 2. Saving finetunes to Ollama. [Free notebook](https://github.com/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Ollama.ipynb) 3. Llama 3.2 Vision finetuning - Radiography use case. [Open in molab](https://github.com/unslothai/notebooks/blob/main/nb/Llama3.2_(11B)-Vision.ipynb) 4. See notebooks for DPO, ORPO, Continued pretraining, conversational finetuning and more on our [documentation](https://unsloth.ai/docs/get-started/unsloth-notebooks)!

Join Discord if you need help + ⭐️ Star us on Github ⭐️

This notebook and all Unsloth notebooks are licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme). """) return if __name__ == "__main__": app.run()