# /// script # requires-python = ">=3.10,<3.14" # dependencies = [ # "accelerate", # "bitsandbytes>=0.43.0", # "datasets==4.3.0", # "hf_transfer", # "huggingface_hub>=0.34.0", # "marimo", # "peft", # "protobuf", # "sentencepiece", # "torchao>=0.16.0", # "transformers==4.56.2", # "triton>=3.2.0", # "trl==0.22.2", # "unsloth @ git+https://github.com/unslothai/unsloth", # "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo", # ] # # [tool.uv] # no-build-package = [ # "bitsandbytes", # "triton", # "vllm", # "xformers", # ] # /// import marimo __generated_with = "0.23.8" app = marimo.App() @app.cell def _(): import marimo as mo return (mo,) @app.cell(hide_code=True) def _(mo): mo.md(r""" To run this notebook, hit the **▶ Run all** button in the bottom-right corner - or use `Ctrl/Cmd + Shift + R`.

Join Discord if you need help + ⭐ Star us on Github ⭐

To install Unsloth on your local device, follow [our guide](https://unsloth.ai/docs/get-started/install). This notebook is licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme). You will learn how to do [data prep](#Data), how to [train](#Train), how to [run the model](#Inference), & how to save it """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### News """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Introducing **Unsloth Studio** - a new open source, no-code web UI to train and run LLMs. [Blog](https://unsloth.ai/docs/new/studio) • [Notebook](https://github.com/unslothai/unsloth/blob/main/studio/Unsloth_Studio_Colab.ipynb)

_{Train models — no code needed}

_{Run GGUF models on Mac, Windows & Linux}

Train MoEs - DeepSeek, GLM, Qwen and gpt-oss 12x faster with 35% less VRAM. [Blog](https://unsloth.ai/docs/new/faster-moe) Ultra Long-Context Reinforcement Learning is here with 7x more context windows! [Blog](https://unsloth.ai/docs/new/grpo-long-context) New in Reinforcement Learning: [FP8 RL](https://unsloth.ai/docs/new/fp8-reinforcement-learning) • [Vision RL](https://unsloth.ai/docs/new/vision-reinforcement-learning-vlm-rl) • [Standby](https://unsloth.ai/docs/basics/memory-efficient-rl) • [gpt-oss RL](https://unsloth.ai/docs/new/gpt-oss-reinforcement-learning) Visit our docs for all our [model uploads](https://unsloth.ai/docs/get-started/unsloth-model-catalog) and [notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks). """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Unsloth """) return @app.cell def _(): from unsloth import FastLanguageModel import torch max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally! dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False. # 4bit pre quantized models we support for 4x faster downloading + no OOMs. fourbit_models = [ "unsloth/mistral-7b-v0.3-bnb-4bit", # New Mistral v3 2x faster! "unsloth/mistral-7b-instruct-v0.3-bnb-4bit", "unsloth/llama-3-8b-bnb-4bit", # Llama-3 15 trillion tokens model 2x faster! "unsloth/llama-3-8b-Instruct-bnb-4bit", "unsloth/llama-3-70b-bnb-4bit", "unsloth/Phi-3-mini-4k-instruct", # Phi-3 2x faster! "unsloth/Phi-3-medium-4k-instruct", "unsloth/mistral-7b-bnb-4bit", "unsloth/gemma-7b-bnb-4bit", # Gemma 2.2x faster! ] # More models at https://huggingface.co/unsloth model, tokenizer = FastLanguageModel.from_pretrained( model_name="unsloth/llama-3-8b-bnb-4bit", # YOUR MODEL YOU USED FOR TRAINING max_seq_length=max_seq_length, # Choose any! We auto support RoPE Scaling internally! dtype=dtype, # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ load_in_4bit=load_in_4bit, # Use 4bit quantization to reduce memory usage. Can be False. # token = "YOUR_HF_TOKEN", # HF Token for gated models ) return ( FastLanguageModel, dtype, load_in_4bit, max_seq_length, model, tokenizer, torch, ) @app.cell(hide_code=True) def _(mo): mo.md(r""" We now add LoRA adapters so we only need to update 1 to 10% of all parameters! """) return @app.cell def _(FastLanguageModel, model): model_1 = FastLanguageModel.get_peft_model( model, r=16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128 target_modules=[ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", ], lora_alpha=16, lora_dropout=0, # Supports any, but = 0 is optimized bias="none", # Supports any, but = "none" is optimized use_gradient_checkpointing="unsloth", # True or "unsloth" for very long context random_state=3407, use_rslora=False, # We support rank stabilized LoRA loftq_config=None, # And LoftQ ) return (model_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Data Prep We now use the Alpaca dataset from [vicgalle](https://huggingface.co/datasets/vicgalle/alpaca-gpt4), which is a version of 52K of the original [Alpaca dataset](https://crfm.stanford.edu/2023/03/13/alpaca.html) generated from GPT4. You can replace this code section with your own data prep. """) return @app.cell def _(): from datasets import load_dataset dataset = load_dataset("vicgalle/alpaca-gpt4", split="train") print(dataset.column_names) return (dataset,) @app.cell(hide_code=True) def _(mo): mo.md(r""" One issue is this dataset has multiple columns. For `Ollama` and `llama.cpp` to function like a custom `ChatGPT` Chatbot, we must only have 2 columns - an `instruction` and an `output` column. """) return @app.cell def _(dataset): print(dataset.column_names) return @app.cell(hide_code=True) def _(mo): mo.md(r""" To solve this, we shall do the following: * Merge all columns into 1 instruction prompt. * Remember LLMs are text predictors, so we can customize the instruction to anything we like! * Use the `to_sharegpt` function to do this column merging process! For example below in our [Titanic CSV finetuning notebook](https://github.com/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Ollama.ipynb), we merged multiple columns in 1 prompt:

""") return @app.cell(hide_code=True) def _(mo): mo.md(r""" To merge multiple columns into 1, use `merged_prompt`. * Enclose all columns in curly braces `{}`. * Optional text must be enclosed in `[[]]`. For example if the column "Pclass" is empty, the merging function will not show the text and skp this. This is useful for datasets with missing values. * You can select every column, or a few! * Select the output or target / prediction column in `output_column_name`. For the Alpaca dataset, this will be `output`. To make the finetune handle multiple turns (like in ChatGPT), we have to create a "fake" dataset with multiple turns - we use `conversation_extension` to randomly select some conversations from the dataset, and pack them together into 1 conversation. """) return @app.cell def _(dataset): from unsloth import to_sharegpt dataset_1 = to_sharegpt( dataset, merged_prompt="{instruction}[[\nYour input is:\n{input}]]", output_column_name="output", conversation_extension=3, # Select more to handle longer conversations ) # Select more to handle longer conversations return (dataset_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Finally use `standardize_sharegpt` to fix up the dataset! """) return @app.cell def _(dataset_1): from unsloth import standardize_sharegpt dataset_2 = standardize_sharegpt(dataset_1) return (dataset_2,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Customizable Chat Templates You also need to specify a chat template. Previously, you could use the Alpaca format as shown below. """) return @app.cell def _(): alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. ### Instruction: {} ### Input: {} ### Response: {}""" return @app.cell(hide_code=True) def _(mo): mo.md(r""" Now, you have to use `{INPUT}` for the instruction and `{OUTPUT}` for the response. We also allow you to use an optional `{SYSTEM}` field. This is useful for Ollama when you want to use a custom system prompt (also like in ChatGPT). You can also not put a `{SYSTEM}` field, and just put plain text. ```python chat_template = \"\"\"{SYSTEM} USER: {INPUT} ASSISTANT: {OUTPUT}\"\"\" ``` Use below if you want to use the Llama-3 prompt format. You must use the `instruct` and not the `base` model if you use this! ```python chat_template = \"\"\"<|begin_of_text|><|start_header_id|>system<|end_header_id|> {SYSTEM}<|eot_id|><|start_header_id|>user<|end_header_id|> {INPUT}<|eot_id|><|start_header_id|>assistant<|end_header_id|> {OUTPUT}<|eot_id|>\"\"\" ``` For the ChatML format: ```python chat_template = \"\"\"<|im_start|>system {SYSTEM}<|im_end|> <|im_start|>user {INPUT}<|im_end|> <|im_start|>assistant {OUTPUT}<|im_end|>\"\"\" ``` """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" The issue is the Alpaca format has 3 fields, whilst OpenAI style chatbots must only use 2 fields (instruction and response). That's why we used the `to_sharegpt` function to merge these columns into 1. """) return @app.cell def _(dataset_2, tokenizer): chat_template = "Below are some instructions that describe some tasks. Write responses that appropriately complete each request.\n\n### Instruction:\n{INPUT}\n\n### Response:\n{OUTPUT}" from unsloth import apply_chat_template dataset_3 = apply_chat_template( dataset_2, tokenizer=tokenizer, chat_template=chat_template ) # default_system_message = "You are a helpful assistant", << [OPTIONAL] return (dataset_3,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Train the model Now let's train our model. We do 60 steps to speed things up, but you can set `num_train_epochs=1` for a full run, and turn off `max_steps=None`. We also support `DPOTrainer` and `GRPOTrainer` for reinforcement learning!! """) return @app.cell def _(dataset_3, max_seq_length, model_1, tokenizer): from trl import SFTConfig, SFTTrainer trainer = SFTTrainer( model=model_1, tokenizer=tokenizer, train_dataset=dataset_3, dataset_text_field="text", max_seq_length=max_seq_length, # Choose any! We auto support RoPE Scaling internally! packing=False, # Can make training 5x faster for short sequences. args=SFTConfig( per_device_train_batch_size=2, gradient_accumulation_steps=4, warmup_steps=5, max_steps=60, learning_rate=0.0002, logging_steps=1, optim="adamw_8bit", weight_decay=0.001, lr_scheduler_type="linear", seed=3407, output_dir="outputs", report_to="none", # Use TrackIO/WandB etc ), ) return (trainer,) @app.cell def _(torch): # @title Show current memory stats gpu_stats = torch.cuda.get_device_properties(0) start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) print(f"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") print(f"{start_gpu_memory} GB of memory reserved.") return max_memory, start_gpu_memory @app.cell def _(trainer): trainer_stats = trainer.train() return (trainer_stats,) @app.cell def _(max_memory, start_gpu_memory, torch, trainer_stats): # @title Show final memory and time stats used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) used_memory_for_lora = round(used_memory - start_gpu_memory, 3) used_percentage = round(used_memory / max_memory * 100, 3) lora_percentage = round(used_memory_for_lora / max_memory * 100, 3) print(f"{trainer_stats.metrics['train_runtime']} seconds used for training.") print( f"{round(trainer_stats.metrics['train_runtime'] / 60, 2)} minutes used for training." ) print(f"Peak reserved memory = {used_memory} GB.") print(f"Peak reserved memory for training = {used_memory_for_lora} GB.") print(f"Peak reserved memory % of max memory = {used_percentage} %.") print(f"Peak reserved memory for training % of max memory = {lora_percentage} %.") return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Inference Let's run the model! Unsloth makes inference natively 2x faster as well! You should use prompts which are similar to the ones you had finetuned on, otherwise you might get bad results! """) return @app.cell def _(FastLanguageModel, model_1, tokenizer): FastLanguageModel.for_inference(model_1) # Enable native 2x faster inference messages = [ # Change below! { "role": "user", "content": "Continue the fibonacci sequence! Your input is 1, 1, 2, 3, 5, 8,", } ] # Change below! input_ids = tokenizer.apply_chat_template( messages, add_generation_prompt=True, return_tensors="pt" ).to("cuda") from transformers import TextStreamer text_streamer = TextStreamer(tokenizer, skip_prompt=True) _ = model_1.generate( input_ids, streamer=text_streamer, max_new_tokens=128, pad_token_id=tokenizer.eos_token_id, ) return (TextStreamer,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Since we created an actual chatbot, you can also do longer conversations by manually adding alternating conversations between the user and assistant! """) return @app.cell def _(FastLanguageModel, TextStreamer, model_1, tokenizer): FastLanguageModel.for_inference(model_1) # Enable native 2x faster inference messages_1 = [ { "role": "user", "content": "Continue the fibonacci sequence! Your input is 1, 1, 2, 3, 5, 8", }, { "role": "assistant", "content": "The fibonacci sequence continues as 13, 21, 34, 55 and 89.", }, {"role": "user", "content": "What is France's tallest tower called?"}, ] # Change below! input_ids_1 = tokenizer.apply_chat_template( messages_1, add_generation_prompt=True, return_tensors="pt" ).to("cuda") text_streamer_1 = TextStreamer(tokenizer, skip_prompt=True) _ = model_1.generate( input_ids_1, streamer=text_streamer_1, max_new_tokens=128, pad_token_id=tokenizer.eos_token_id, ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Saving, loading finetuned models To save the final model as LoRA adapters, either use Hugging Face's `push_to_hub` for an online save or `save_pretrained` for a local save. **[NOTE]** This ONLY saves the LoRA adapters, and not the full model. To save to 16bit or GGUF, scroll down! """) return @app.cell def _(model_1, tokenizer): model_1.save_pretrained("llama_lora") tokenizer.save_pretrained("llama_lora") return @app.cell(hide_code=True) def _(mo): mo.md(r""" Now if you want to load the LoRA adapters we just saved for inference, set `False` to `True`: """) return @app.cell def _(TextStreamer, dtype, load_in_4bit, max_seq_length, model_1, tokenizer): if False: from unsloth import FastLanguageModel as _FastLanguageModel _model, _tokenizer = _FastLanguageModel.from_pretrained( model_name="llama_lora", # YOUR MODEL YOU USED FOR TRAINING max_seq_length=max_seq_length, # Choose any! We auto support RoPE Scaling internally! dtype=dtype, # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ load_in_4bit=load_in_4bit, # Use 4bit quantization to reduce memory usage. Can be False. ) _FastLanguageModel.for_inference(_model) messages_2 = [ { "role": "user", "content": "Describe anything special about a sequence. Your input is 1, 1, 2, 3, 5, 8,", } ] input_ids_2 = tokenizer.apply_chat_template( messages_2, add_generation_prompt=True, return_tensors="pt" ).to("cuda") # Change below! text_streamer_2 = TextStreamer(tokenizer, skip_prompt=True) _ = model_1.generate( input_ids_2, streamer=text_streamer_2, max_new_tokens=128, pad_token_id=tokenizer.eos_token_id, ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" You can also use Hugging Face's `AutoPeftModelForCausalLM`. Only use this if you do not have `unsloth` installed. It can be hopelessly slow, since `4bit` model downloading is not supported, and Unsloth's **inference is 2x faster**. """) return @app.cell def _(load_in_4bit): if False: from peft import AutoPeftModelForCausalLM as _AutoPeftModelForCausalLM from transformers import AutoTokenizer as _AutoTokenizer _model = _AutoPeftModelForCausalLM.from_pretrained( "llama_lora", load_in_4bit=load_in_4bit ) _tokenizer = _AutoTokenizer.from_pretrained("llama_lora") return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Ollama Support [Unsloth](https://github.com/unslothai/unsloth) now allows you to automatically finetune and create a [Modelfile](https://github.com/ollama/ollama/blob/main/docs/modelfile.md), and export to [Ollama](https://ollama.com/)! This makes finetuning much easier and provides a seamless workflow from `Unsloth` to `Ollama`! Let's first install `Ollama`! """) return @app.cell def _(): import subprocess #! curl -fsSL https://ollama.com/install.sh | sh subprocess.call("curl -fsSL https://ollama.com/install.sh | sh", shell=True) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Next, we shall save the model to GGUF / llama.cpp We clone `llama.cpp` and we default save it to `q8_0`. We allow all methods like `q4_k_m`. Use `save_pretrained_gguf` for local saving and `push_to_hub_gguf` for uploading to HF. Some supported quant methods (full list on our [docs page](https://unsloth.ai/docs/basics/inference-and-deployment/saving-to-gguf)): * `q8_0` - Fast conversion. High resource use, but generally acceptable. * `q4_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_K. * `q5_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_K. We also support saving to multiple GGUF options in a list fashion! This can speed things up by 10 minutes or more if you want multiple export formats! """) return @app.cell def _(model_1, tokenizer): # Save to 8bit Q8_0 if True: # Remember to go to https://huggingface.co/settings/tokens for a token! # And change hf to your username! model_1.save_pretrained_gguf("llama_finetune", tokenizer) if False: # Save to 16bit GGUF model_1.push_to_hub_gguf( "HF_USERNAME/llama_finetune", tokenizer, token="YOUR_HF_TOKEN" ) if False: model_1.save_pretrained_gguf( "llama_finetune", tokenizer, quantization_method="f16" ) # Save to q4_k_m GGUF if False: model_1.push_to_hub_gguf( "HF_USERNAME/llama_finetune", tokenizer, quantization_method="f16", token="YOUR_HF_TOKEN", # Get a token at https://huggingface.co/settings/tokens ) if False: # Save to multiple GGUF options - much faster if you want multiple! model_1.save_pretrained_gguf( "llama_finetune", tokenizer, quantization_method="q4_k_m" ) if False: model_1.push_to_hub_gguf( "HF_USERNAME/llama_finetune", tokenizer, quantization_method="q4_k_m", token="YOUR_HF_TOKEN", # Get a token at https://huggingface.co/settings/tokens ) # Change hf to your username! if False: model_1.push_to_hub_gguf( "HF_USERNAME/llama_finetune", tokenizer, quantization_method=["q4_k_m", "q8_0", "q5_k_m"], token="YOUR_HF_TOKEN", # Get a token at https://huggingface.co/settings/tokens ) # Get a token at https://huggingface.co/settings/tokens return @app.cell(hide_code=True) def _(mo): mo.md(r""" We use `subprocess` to start `Ollama` up in a non blocking fashion! In your own desktop, you can simply open up a new `terminal` and type `ollama serve`, but in molab, we have to use this hack! """) return @app.cell def _(): import subprocess as _molab_subprocess _molab_subprocess.Popen(["ollama", "serve"]) import time time.sleep(3) return @app.cell(hide_code=True) def _(mo): mo.md(r""" `Ollama` needs a `Modelfile`, which specifies the model's prompt format. Let's print Unsloth's auto generated one: """) return @app.cell def _(tokenizer): print(tokenizer._ollama_modelfile) return @app.cell(hide_code=True) def _(mo): mo.md(r""" We now will create an `Ollama` model called `unsloth_model` using the `Modelfile` which we auto generated! """) return @app.cell def _(): import subprocess as _molab_subprocess_2 _molab_subprocess_2.call( ["ollama", "create", "unsloth_model", "-f", "./model/Modelfile"] ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" And now we can do inference on it via `Ollama`! You can also upload to `Ollama` and try the `Ollama` Desktop app by heading to https://www.ollama.com/ """) return @app.cell def _(): # Note: Command contains invalid template expression # !curl http://localhost:11434/api/chat -d '{ "model": "unsloth_model", "messages": [ { "role": "user", "content": "Continue the Fibonacci sequence: 1, 1, 2, 3, 5, 8," } ] }' return @app.cell(hide_code=True) def _(mo): mo.md(r""" # ChatGPT interactive mode ### ⭐ To run the finetuned model like in a ChatGPT style interface, first click the **| >_ |** button. ![](https://raw.githubusercontent.com/unslothai/unsloth/nightly/images/Where_Terminal.png) --- --- --- ### ⭐ Then, type `ollama run unsloth_model` ![](https://raw.githubusercontent.com/unslothai/unsloth/nightly/images/Terminal_Type.png) --- --- --- ### ⭐ And you have a ChatGPT style assistant! ### Type any question you like and press `ENTER`. If you want to exit, hit `CTRL + D` ![](https://raw.githubusercontent.com/unslothai/unsloth/nightly/images/Assistant.png) And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord! Some other resources: 1. Looking to use Unsloth locally? Read our [Installation Guide](https://unsloth.ai/docs/get-started/install) for details on installing Unsloth on Windows, Docker, AMD, Intel GPUs. 2. Learn how to do Reinforcement Learning with our [RL Guide and notebooks](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide). 3. Read our guides and notebooks for [Text-to-speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning) and [vision](https://unsloth.ai/docs/basics/vision-fine-tuning) model support. 4. Explore our [LLM Tutorials Directory](https://unsloth.ai/docs/models/tutorials-how-to-fine-tune-and-run-llms) to find dedicated guides for each model. 5. Need help with Inference? Read our [Inference & Deployment page](https://unsloth.ai/docs/basics/inference-and-deployment) for details on using vLLM, llama.cpp, Ollama etc.

Join Discord if you need help + ⭐️ Star us on Github ⭐️ This notebook and all Unsloth notebooks are licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme)

""") return if __name__ == "__main__": app.run()