# /// script # requires-python = ">=3.10,<3.14" # dependencies = [ # "accelerate", # "bitsandbytes>=0.43.0", # "datasets==4.3.0", # "hf_transfer", # "huggingface_hub>=0.34.0", # "marimo", # "peft", # "protobuf", # "sentencepiece", # "torchao>=0.16.0", # "transformers==4.56.2", # "triton>=3.2.0", # "trl==0.22.2", # "unsloth @ git+https://github.com/unslothai/unsloth", # "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo", # ] # # [tool.uv] # no-build-package = [ # "bitsandbytes", # "triton", # "vllm", # "xformers", # ] # /// import marimo __generated_with = "0.23.8" app = marimo.App() @app.cell def _(): import marimo as mo return (mo,) @app.cell(hide_code=True) def _(mo): mo.md(r""" To run this notebook, hit the **▶ Run all** button in the bottom-right corner - or use `Ctrl/Cmd + Shift + R`.

Join Discord if you need help + ⭐ Star us on Github ⭐

To install Unsloth on your local device, follow [our guide](https://unsloth.ai/docs/get-started/install). This notebook is licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme). You will learn how to do [data prep](#Data), how to [train](#Train), how to [run the model](#Inference), & how to save it """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### News """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Introducing **Unsloth Studio** - a new open source, no-code web UI to train and run LLMs. [Blog](https://unsloth.ai/docs/new/studio) • [Notebook](https://github.com/unslothai/unsloth/blob/main/studio/Unsloth_Studio_Colab.ipynb)

_{Train models — no code needed}

_{Run GGUF models on Mac, Windows & Linux}

Train MoEs - DeepSeek, GLM, Qwen and gpt-oss 12x faster with 35% less VRAM. [Blog](https://unsloth.ai/docs/new/faster-moe) Ultra Long-Context Reinforcement Learning is here with 7x more context windows! [Blog](https://unsloth.ai/docs/new/grpo-long-context) New in Reinforcement Learning: [FP8 RL](https://unsloth.ai/docs/new/fp8-reinforcement-learning) • [Vision RL](https://unsloth.ai/docs/new/vision-reinforcement-learning-vlm-rl) • [Standby](https://unsloth.ai/docs/basics/memory-efficient-rl) • [gpt-oss RL](https://unsloth.ai/docs/new/gpt-oss-reinforcement-learning) Visit our docs for all our [model uploads](https://unsloth.ai/docs/get-started/unsloth-model-catalog) and [notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks). """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Unsloth """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" If you want to finetune Llama-3 2x faster and use 70% less VRAM, go to our [finetuning notebook](https://github.com/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Alpaca.ipynb)! """) return @app.cell def _(): from unsloth import FastLanguageModel # 4bit pre quantized models we support for 4x faster downloading + no OOMs. fourbit_models = [ "unsloth/mistral-7b-instruct-v0.2-bnb-4bit", "unsloth/gemma-7b-it-bnb-4bit", ] # More models at https://huggingface.co/unsloth model, tokenizer = FastLanguageModel.from_pretrained( model_name="unsloth/Llama-3.1-8B-Instruct", max_seq_length=8192, load_in_4bit=True, # token = "YOUR_HF_TOKEN", # HF Token for gated models ) return FastLanguageModel, model, tokenizer @app.cell def _(FastLanguageModel, model, tokenizer): from transformers import TextStreamer from unsloth.chat_templates import get_chat_template tokenizer_1 = get_chat_template( tokenizer, chat_template="llama-3.1", mapping={ # ShareGPT style "role": "from", "content": "value", "user": "human", "assistant": "gpt", }, ) FastLanguageModel.for_inference( model ) return TextStreamer, tokenizer_1 @app.cell(hide_code=True) def _(mo): mo.md(r""" Change the "value" part to call the model! Unsloth makes inference natively 2x faster! No need to change or do anything! """) return @app.cell def _(TextStreamer, model, tokenizer_1): messages = [ {"from": "human", "value": "Continue the fibonacci sequence: 1, 1, 2, 3, 5, 8,"} ] inputs = tokenizer_1.apply_chat_template( messages, tokenize=True, add_generation_prompt=True, return_tensors="pt" ).to("cuda") # EDIT HERE! text_streamer = TextStreamer(tokenizer_1) _ = model.generate( input_ids=inputs, streamer=text_streamer, max_new_tokens=1024, use_cache=True ) return @app.cell def _(TextStreamer, model, tokenizer_1): messages_1 = [ {"from": "human", "value": "Describe the tallest tower in the world."} ] inputs_1 = tokenizer_1.apply_chat_template( messages_1, tokenize=True, add_generation_prompt=True, return_tensors="pt" ).to("cuda") text_streamer_1 = TextStreamer(tokenizer_1) _ = model.generate( input_ids=inputs_1, streamer=text_streamer_1, max_new_tokens=1024, use_cache=True, ) return @app.cell def _(TextStreamer, model, tokenizer_1): messages_2 = [{"from": "human", "value": "What is Unsloth?"}] inputs_2 = tokenizer_1.apply_chat_template( messages_2, tokenize=True, add_generation_prompt=True, return_tensors="pt" ).to("cuda") text_streamer_2 = TextStreamer(tokenizer_1) _ = model.generate( input_ids=inputs_2, streamer=text_streamer_2, max_new_tokens=1024, use_cache=True, ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord! Some other resources: 1. Looking to use Unsloth locally? Read our [Installation Guide](https://unsloth.ai/docs/get-started/install) for details on installing Unsloth on Windows, Docker, AMD, Intel GPUs. 2. Learn how to do Reinforcement Learning with our [RL Guide and notebooks](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide). 3. Read our guides and notebooks for [Text-to-speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning) and [vision](https://unsloth.ai/docs/basics/vision-fine-tuning) model support. 4. Explore our [LLM Tutorials Directory](https://unsloth.ai/docs/models/tutorials-how-to-fine-tune-and-run-llms) to find dedicated guides for each model. 5. Need help with Inference? Read our [Inference & Deployment page](https://unsloth.ai/docs/basics/inference-and-deployment) for details on using vLLM, llama.cpp, Ollama etc.

Join Discord if you need help + ⭐️ Star us on Github ⭐️ This notebook and all Unsloth notebooks are licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme)

""") return if __name__ == "__main__": app.run()