# /// script # requires-python = ">=3.10,<3.14" # dependencies = [ # "bitsandbytes>=0.43.0", # "marimo", # "timm", # "tokenizers>=0.22.0,<=0.23.0", # "torch>=2.8.0", # "torchvision", # "transformers>=5.5.0", # "triton>=3.2.0", # "triton_kernels @ git+https://github.com/triton-lang/triton.git@0add68262ab0a2e33b84524346cb27cbb2787356#subdirectory=python/triton_kernels", # "trl>=0.28.0", # "unsloth @ git+https://github.com/unslothai/unsloth", # "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo", # "uv", # ] # # [tool.uv] # no-build-package = [ # "bitsandbytes", # "triton", # "vllm", # "xformers", # ] # /// import marimo __generated_with = "0.23.8" app = marimo.App() @app.cell def _(): import marimo as mo return (mo,) @app.cell(hide_code=True) def _(mo): mo.md(r""" To run this notebook, hit the **▶ Run all** button in the bottom-right corner - or use `Ctrl/Cmd + Shift + R`.

Join Discord if you need help + ⭐ Star us on Github ⭐

To install Unsloth on your local device, follow [our guide](https://unsloth.ai/docs/get-started/install). This notebook is licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme). You will learn how to do [data prep](#Data), how to [train](#Train), how to [run the model](#Inference), & how to save it """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" # Goal: Make Gemma 4 solve Sudoku puzzles with Reinforcement Learning Our goal is to make Gemma 4 learn to solve Sudoku puzzles using reinforcement learning (GRPO). The model will devise a strategy to fill in empty cells, and we'll reward it for correct placements and completing valid puzzles.

""") return @app.cell(hide_code=True) def _(mo): mo.md(r""" # Installation We'll be using [Unsloth](https://github.com/unslothai/unsloth) to do RL on Gemma 4. Unsloth saves 70% VRAM usage and makes reinforcement learning 2 to 6x faster. """) return @app.cell def _(): # For Gemma 4 vision/audio return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Unsloth """) return @app.cell def _(): from unsloth import FastVisionModel import torch max_seq_length = 4096 # Can increase for longer reasoning traces lora_rank = 32 # Larger rank = smarter, but slower gemma4_models = [ # Gemma-4 instruct models: "unsloth/gemma-4-E2B-it", "unsloth/gemma-4-E4B-it", "unsloth/gemma-4-31B-it", "unsloth/gemma-4-26B-A4B-it", # Gemma-4 base models: "unsloth/gemma-4-E2B", "unsloth/gemma-4-E4B", "unsloth/gemma-4-31B", "unsloth/gemma-4-26B-A4B", ] # More models at https://huggingface.co/unsloth model, tokenizer = FastVisionModel.from_pretrained( model_name="unsloth/gemma-4-E2B-it", max_seq_length=max_seq_length, # Can increase for longer reasoning traces load_in_4bit=False, # False for LoRA 16bit fast_inference=False, # Enable vllm fast inference ) return FastVisionModel, lora_rank, max_seq_length, model, tokenizer @app.cell(hide_code=True) def _(mo): mo.md(r""" To do efficient RL, we will use [LoRA](https://arxiv.org/abs/2106.09685), which allows us to only add 1 to 5% of extra weights to the model for finetuning purposes. This allows us to save memory usage by over 60%, and yet it retains good accuracy. """) return @app.cell def _(FastVisionModel, lora_rank, model): model_1 = FastVisionModel.get_peft_model( model, r=lora_rank, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128 target_modules=[ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", ], lora_alpha=lora_rank * 2, # *2 speeds up training use_gradient_checkpointing="unsloth", # Reduces memory usage random_state=3407, ) return (model_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" # Sudoku Game Implementation We use GPT-5 to create a clean Sudoku solver environment. The strategy outputs "row,col,value" to fill cells. """) return @app.cell def _(): # @title Sudoku Game Implementation from dataclasses import dataclass, field from typing import List, Tuple, Optional import random import copy def _is_valid_placement( board: List[List[int]], row: int, col: int, num: int ) -> bool: """Check if placing num at (row, col) is valid.""" if num in board[row]: # Check row return False if num in [board[r][col] for r in range(9)]: return False box_row, box_col = (3 * (row // 3), 3 * (col // 3)) # Check column for r in range(box_row, box_row + 3): for c in range(box_col, box_col + 3): if board[r][c] == num: return False # Check 3x3 box return True def _solve_sudoku(board: List[List[int]]) -> bool: """Solve sudoku using backtracking (for puzzle generation).""" for row in range(9): for col in range(9): if board[row][col] == 0: for num in range(1, 10): if _is_valid_placement(board, row, col, num): board[row][col] = num if _solve_sudoku(board): return True board[row][col] = 0 return False return True def _generate_complete_board(rng: random.Random) -> List[List[int]]: """Generate a complete valid Sudoku board.""" board = [[0 for _ in range(9)] for _ in range(9)] for box in range(3): nums = list(range(1, 10)) rng.shuffle(nums) for i in range(3): for j in range(3): board[box * 3 + i][box * 3 + j] = nums[i * 3 + j] _solve_sudoku(board) return board # Fill diagonal 3x3 boxes first (they don't affect each other) @dataclass class SudokuGame: difficulty: int = 40 seed: Optional[int] = None _rng: random.Random = field(init=False, repr=False) _board: List[List[int]] = field(init=False, repr=False) _solution: List[List[int]] = field(init=False, repr=False) # Solve the rest _initial_board: List[List[int]] = field(init=False, repr=False) _moves: int = field(default=0, init=False, repr=False) _state: str = field(default="ongoing", init=False, repr=False) def __post_init__(self): self._rng = random.Random( self.seed ) # Number of cells to remove (20 = easy, 40 = medium, 50 = hard) complete_board = _generate_complete_board(self._rng) self._solution = copy.deepcopy(complete_board) self._board = copy.deepcopy(complete_board) cells = [(r, c) for r in range(9) for c in range(9)] self._rng.shuffle(cells) for r, c in cells[: self.difficulty]: self._board[r][c] = 0 self._initial_board = copy.deepcopy(self._board) self._update_state() def board(self) -> List[List[int]]: """Return current board state.""" # Generate complete board return [row[:] for row in self._board] def initial_board(self) -> List[List[int]]: """Return initial puzzle state.""" # Remove cells to create puzzle return [row[:] for row in self._initial_board] def state(self) -> str: """Return game state: 'ongoing', 'success', or 'failed'.""" return self._state def moves(self) -> int: """Return number of moves made.""" return self._moves def place_number(self, row: int, col: int, num: int) -> bool: """Place a number on the board. Returns True if valid move.""" if not (0 <= row < 9 and 0 <= col < 9): self._state = "failed" return False if not 1 <= num <= 9: self._state = "failed" return False if self._initial_board[row][col] != 0: self._state = "failed" return False if self._board[row][col] != 0: self._state = "failed" return False if not _is_valid_placement(self._board, row, col, num): self._state = "failed" return False self._board[row][col] = num self._moves = self._moves + 1 # Validate input self._update_state() return True def _update_state(self) -> None: """Update game state based on current board.""" if all((self._board[r][c] != 0 for r in range(9) for c in range(9))): if self._board == self._solution: self._state = "success" else: # Can't modify initial cells self._state = "failed" else: self._state = "ongoing" def pretty(self, colors: bool = True) -> str: """Pretty print the Sudoku board.""" RESET = "\x1b[0m" # Check if placement is valid INITIAL = "\x1b[38;5;45m" # Cyan for initial numbers PLACED = "\x1b[38;5;226m" # Yellow for placed numbers EMPTY = "\x1b[38;5;239m" # Gray for empty cells lines = [] lines.append("┌───────┬───────┬───────┐") # Place number for row in range(9): row_str = "│ " for col in range(9): num = self._board[row][col] if colors: if num == 0: row_str = row_str + f"{EMPTY}.{RESET}" elif ( self._initial_board[row][col] != 0 ): # Check if puzzle is complete row_str = row_str + f"{INITIAL}{num}{RESET}" else: # Verify solution is correct row_str = row_str + f"{PLACED}{num}{RESET}" else: row_str = row_str + (str(num) if num != 0 else ".") if col % 3 == 2: row_str = row_str + " │ " else: row_str = row_str + " " lines.append(row_str.rstrip()) if row == 8: lines.append("└───────┴───────┴───────┘") elif row % 3 == 2: # Cyan for initial numbers lines.append( "├───────┼───────┼───────┤" ) # Yellow for placed numbers return "\n".join(lines) # Gray for empty cells return (SudokuGame,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Test the Sudoku environment: """) return @app.cell def _(SudokuGame): # Create an easy puzzle game = SudokuGame(difficulty=30, seed=42) print("Initial puzzle:") print(game.pretty()) print(f"\nState: {game.state()}, Moves: {game.moves()}") return (game,) @app.cell def _(game): game return @app.cell(hide_code=True) def _(mo): mo.md(r""" Try making some moves: """) return @app.cell def _(game): # Make a valid move game.place_number(0, 1, 7) print("\nAfter placing 7 at (1,0):") print(game.pretty()) print(f"State: {game.state()}, Moves: {game.moves()}") return @app.cell(hide_code=True) def _(mo): mo.md(r""" If we do some other action that's not part of the action space, we will get an error, and the game will not accept anymore actions. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" # RL Environment Setup Execute strategies with time limits to prevent infinite loops. """) return @app.cell def _(SudokuGame): from typing import Callable from unsloth import execute_with_time_limit def _execute_strategy(strategy: Callable, game: SudokuGame): """Execute a strategy function on a Sudoku game.""" assert callable(strategy) max_moves = 100 valid_moves = 0 # Track successful moves while ( game.state() == "ongoing" and valid_moves < max_moves ): # Track successful moves try: board = game.board() initial = game.initial_board() result = strategy(board, initial) if not isinstance(result, (tuple, list)) or len(result) != 3: return (valid_moves, "failed") row, col, num = result if not all( (isinstance(x, int) for x in [row, col, num]) ): # Validate result format return (valid_moves, "failed") success = game.place_number( row, col, num ) # Invalid format = immediate fail, but return valid moves made if success: valid_moves = valid_moves + 1 # Track successful moves else: return (valid_moves, "failed") except Exception: # Validate types return (valid_moves, "failed") if valid_moves >= max_moves and game.state() == "ongoing": return (valid_moves, "failed") return ( valid_moves, game.state(), ) return Callable, execute_with_time_limit @app.cell(hide_code=True) def _(mo): mo.md(r""" To allow longer strategies for Reinforcement Learning, we shall allow a 10 second timer. """) return @app.cell def _(Callable, SudokuGame, execute_with_time_limit): @execute_with_time_limit(10) def execute_strategy(strategy: Callable, game: SudokuGame): """Execute strategy with 10 second time limit.""" return _execute_strategy(strategy, game) return (execute_strategy,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Test with a simple strategy: """) return @app.cell def _(SudokuGame, execute_strategy): def simple_strategy(board, initial): """Simple strategy: fill first empty cell with 1.""" for r in range(9): for c in range(9): if board[r][c] == 0 and initial[r][c] == 0: return (r, c, 7) return (0, 0, 7) game_1 = SudokuGame(difficulty=30, seed=42) try: moves, state = execute_strategy(simple_strategy, game_1) print(f"Moves: {moves}, State: {state}") except TimeoutError as e: print(f"Timed out: {e}") return (game_1,) @app.cell def _(game_1): print(game_1.pretty()) return @app.cell(hide_code=True) def _(mo): mo.md(r""" # Code Execution To execute and create a new Python function, we first have to check if the function does not call other global variables or cheat. This is called `countering reward hacking` since we don't want the function to cheat. For example the below piece of code is fine, since it only imports Python level functions. We use `check_python_modules`: """) return @app.cell def _(): from unsloth import check_python_modules, create_locked_down_function # Test safe code sample = """ def strategy(board, initial): for r in range(9): for c in range(9): if board[r][c] == 0: return (r, c, 1) return (0, 0, 1) """ ok, info = check_python_modules(sample) print("Safe Python code?", ok) print(info) return check_python_modules, create_locked_down_function @app.cell(hide_code=True) def _(mo): mo.md(r""" For the below piece of code, since we import `numpy`, we should not allow the execution: """) return @app.cell def _(check_python_modules): sample_1 = "\ndef strategy(board, initial):\n import numpy as np\n return (0, 0, 1)\n" ok_1, info_1 = check_python_modules(sample_1) print("Safe Python code?", ok_1) print(info_1) return @app.cell(hide_code=True) def _(mo): mo.md(r""" # Data & RL task setup Create the prompt that instructs the model to generate a Sudoku solving strategy. You can customize this to some other task for another RL task. """) return @app.cell def _(): prompt = """ Create a Sudoku solving strategy using only native Python built-in functions without any import statements. You are given two lists of lists (9x9 grids): - board: current state (0 means empty) - initial: starting puzzle (0 means was empty, numbers are fixed) Return a tuple (row, col, number) for the next move. - row: 0-8 (row index) - col: 0-8 (column index) - number: 1-9 (digit to place) Only place numbers in cells that are BOTH empty in initial AND empty in board (initial[row][col] == 0 AND board[row][col] == 0) Use Sudoku rules: no duplicates in rows, columns, or 3x3 boxes. Output your function in backticks: ```python def strategy(board, initial): # Your logic here return (row, col, number) ``` All helper functions must be inside def strategy. Output only the function. """.strip() print(prompt) return (prompt,) @app.cell(hide_code=True) def _(mo): mo.md(r""" First, let's prompt the model without RL and see how it goes: """) return @app.cell def _(model_1, prompt, tokenizer): text = tokenizer.apply_chat_template( [{"role": "user", "content": prompt.strip()}], tokenize=False, add_generation_prompt=True, ) from transformers import TextStreamer print("=" * 50) print("BASE MODEL OUTPUT (before RL training):") print("=" * 50) inputs = tokenizer(text=text, add_special_tokens=False, return_tensors="pt").to( "cuda" ) text_streamer = TextStreamer(tokenizer, skip_prompt=True) result = model_1.generate( **inputs, streamer=text_streamer, max_new_tokens=128, use_cache=True, temperature=1.0, top_p=0.95, top_k=64, ) return (TextStreamer,) @app.cell(hide_code=True) def _(mo): mo.md(r""" # Reward functions We now design a `extract_function` function which simply extracts the function wrapped in 3 back ticks. And 3 reward functions: 1. `function_works` which rewards the model if the strategy is a valid Python function. 2. `no_cheating` which checks if the function imported other modules, and if it did, we penalize it. 3. `strategy_succeeds` which checks if the game strategy actually succeeds in attaining Sudoku after running the auto-generated strategy. """) return @app.function def extract_function(text): """Extract Python function from markdown code blocks.""" if text.count("```") >= 2: first = text.find("```") + 3 second = text.find("```", first) fx = text[first:second].strip() fx = fx.removeprefix("python\n") fx = fx[fx.find("def") :] if fx.startswith("def strategy(board, initial):"): return fx return None @app.cell(hide_code=True) def _(mo): mo.md(r""" **Reward 1: Function Works** Checks if the generated code is valid Python and can be executed. """) return @app.cell def _(check_python_modules, create_locked_down_function): def function_works(completions, **kwargs): """Reward for generating valid executable Python code.""" scores = [] for completion in completions: score = 0 # Invalid function response = completion[0]["content"] function = extract_function(response) if function is not None: ok, info = check_python_modules(function) if function is None or "error" in info: score = -2.0 # Invalid function else: try: new_strategy = create_locked_down_function(function) score = 1.0 # Valid function except: score = -1.0 # Function has errors scores.append(score) return scores return (function_works,) @app.cell(hide_code=True) def _(mo): mo.md(r""" **Reward 2: No Cheating** Penalizes functions that import external libraries. """) return @app.cell def _(check_python_modules): def no_cheating(completions, **kwargs): """Penalize use of external imports.""" scores = [] for completion in completions: response = completion[0]["content"] function = extract_function(response) if function is not None: ok, info = check_python_modules(function) scores.append(1.0 if ok else -20.0) # Heavy penalty for cheating else: scores.append(-1.0) # Failed to create function return scores return (no_cheating,) @app.cell(hide_code=True) def _(mo): mo.md(r""" **Reward 3: Strategy Succeeds** Rewards strategies that successfully solve Sudoku puzzles. """) return @app.cell def _( SudokuGame, check_python_modules, create_locked_down_function, execute_strategy, ): import numpy as np global PRINTER PRINTER = 0 def strategy_succeeds(completions, **kwargs): """Reward valid moves even if strategy eventually fails.""" global PRINTER scores = [] seed = np.random.randint(10000) difficulty = 40 for completion in completions: printed = False response = completion[0]["content"] function = extract_function(response) if PRINTER % 5 == 0: printed = True print("\n" + "=" * 60) print(function) print("=" * 60) PRINTER = PRINTER + 1 if function is not None: ok, info = check_python_modules(function) if function is None or "error" in info: scores.append(0) continue try: new_strategy = create_locked_down_function(function) except: scores.append(0) continue try: game = SudokuGame(difficulty=difficulty, seed=seed) valid_moves, game_state = execute_strategy(new_strategy, game) if valid_moves == difficulty: game_state = "success" print(f"\n Valid moves: {valid_moves}, Final state: {game_state}") if not printed: print("Strategy:") print(function[:200] + "..." if len(function) > 200 else function) print("\nFinal board:") print(game.pretty()) if game_state == "success": scores.append(30.0) elif valid_moves > 0: reward = valid_moves * 0.2 scores.append(reward) else: scores.append(-2.0) except TimeoutError: print("Timeout") scores.append(-1.0) except Exception as e: print(f"Exception: {str(e)[:100]}") scores.append(-3.0) # Solved the puzzle! return scores # Reward based on valid moves made before failure # Each valid move is worth 0.2 points # Failed immediately with no valid moves return PRINTER, strategy_succeeds @app.cell(hide_code=True) def _(mo): mo.md(r""" # Dataset Preparation Create the training dataset. """) return @app.cell def _(prompt, tokenizer): from datasets import Dataset dataset = Dataset.from_list( [ { "prompt": [{"role": "user", "content": prompt.strip()}], "answer": 0, } ] * 1000 ) maximum_length = len( tokenizer.apply_chat_template( [{"role": "user", "content": prompt.strip()}], add_generation_prompt=True ) ) print(f"Maximum prompt length: {maximum_length}") print("\nDataset sample:") print(dataset[0]) return dataset, maximum_length @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Train the model Now set up GRPO Trainer and all configurations! We also support GSPO, GAPO, Dr GRPO and more! Go the Unsloth [Reinforcement Learning Docs](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide) for more options. """) return @app.cell def _(max_seq_length, maximum_length): # Leave room for the prompt (plus 1 token safety margin) max_completion_length = max_seq_length - (maximum_length + 1) from trl import GRPOConfig, GRPOTrainer training_args = GRPOConfig( temperature=1.0, learning_rate=5e-5, weight_decay=0.001, warmup_ratio=0.1, lr_scheduler_type="linear", optim="adamw_8bit", logging_steps=1, per_device_train_batch_size=1, gradient_accumulation_steps=2, # Increase to 4 for smoother training num_generations=2, # Decrease if out of memory max_completion_length=max_completion_length, # num_train_epochs = 1, # Set to 1 for a full training run max_steps=60, save_steps=100, report_to="none", # Can use Weights & Biases, TrackIO output_dir="outputs", epsilon=0.2, epsilon_high=0.28, # one sided delta=1.5, # two sided loss_type="bnpo", mask_truncated_completions=True, # For optional training + evaluation # fp16_full_eval = True, # per_device_eval_batch_size = 4, # eval_accumulation_steps = 1, # eval_strategy = "steps", # eval_steps = 1, ) return GRPOTrainer, training_args @app.cell(hide_code=True) def _(mo): mo.md(r""" And let's run the trainer! If you scroll up, you'll see a table of rewards. The goal is to see the `reward` column increase! You might have to wait 150 to 200 steps for any action. You'll probably get low reward for the first 100 steps. Please be patient! | Step | Training Loss | reward | reward_std | completion_length | kl | |------|---------------|-----------|------------|-------------------|----------| | 1 | 0.000000 | 0.125000 | 0.000000 | 200.000000 | 0.000000 | | 2 | 0.000000 | 0.072375 | 0.248112 | 200.000000 | 0.000000 | | 3 | 0.000000 | -0.079000 | 0.163776 | 182.500000 | 0.000005 | """) return @app.cell def _( GRPOTrainer, dataset, function_works, model_1, no_cheating, strategy_succeeds, tokenizer, training_args, ): # For optional training + evaluation # new_dataset = dataset.train_test_split(test_size = 0.01) trainer = GRPOTrainer( model=model_1, processing_class=tokenizer, reward_funcs=[function_works, no_cheating, strategy_succeeds], args=training_args, train_dataset=dataset, ) return (trainer,) @app.cell(hide_code=True) def _(mo): mo.md(r""" And let's train the model! **NOTE** A T4 free GPU might take 5 minutes for one generation sadly since it's an old GPU - A100 or H100 will be much faster! """) return @app.cell def _(trainer): trainer.train() return @app.cell(hide_code=True) def _(mo): mo.md(r""" And now with the LoRA we just trained with GRPO - we first save the LoRA first! """) return @app.cell def _(model_1, tokenizer): model_1.save_pretrained("gemma_4_lora") # Local saving tokenizer.save_pretrained("gemma_4_lora") return @app.cell(hide_code=True) def _(mo): mo.md(r""" Verify LoRA is actually trained! """) return @app.cell def _(): from safetensors import safe_open tensors = {} with safe_open("grpo_saved_lora/adapter_model.safetensors", framework="pt") as f: # Verify both A and B are non zero for key in f.keys(): tensor = f.get_tensor(key) n_zeros = (tensor == 0).sum() / tensor.numel() assert n_zeros.item() != tensor.numel() return @app.cell(hide_code=True) def _(mo): mo.md(r""" # Inference Now let's try the model we just trained! """) return @app.cell def _(TextStreamer, model_1, prompt, tokenizer): text_1 = tokenizer.apply_chat_template( [{"role": "user", "content": prompt.strip()}], tokenize=False, add_generation_prompt=True, ) _ = model_1.generate( **tokenizer(images=None, text=text_1, return_tensors="pt").to("cuda"), temperature=1.0, max_new_tokens=512, streamer=TextStreamer(tokenizer, skip_prompt=False), ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Saving to float16 for VLLM We also support saving to `float16` directly. Select `merged_16bit` for float16 or `merged_4bit` for int4. We also allow `lora` adapters as a fallback. Use `push_to_hub_merged` to upload to your Hugging Face account! You can go to https://huggingface.co/settings/tokens for your personal tokens. See [our docs](https://unsloth.ai/docs/basics/inference-and-deployment) for more deployment options. """) return @app.cell def _(model_1, tokenizer): # Merge to 16bit if False: model_1.save_pretrained_merged( "gemma_4_finetune_16bit", tokenizer, save_method="merged_16bit" ) if False: # Merge to 4bit model_1.push_to_hub_merged( "HF_USERNAME/gemma_4_finetune_16bit", tokenizer, save_method="merged_16bit", token="YOUR_HF_TOKEN", ) if False: model_1.save_pretrained_merged( "gemma_4_finetune_4bit", tokenizer, save_method="merged_4bit" ) # Just LoRA adapters if False: model_1.push_to_hub_merged( "HF_USERNAME/gemma_4_finetune_4bit", tokenizer, save_method="merged_4bit", token="YOUR_HF_TOKEN", ) if False: model_1.save_pretrained("gemma_4_lora") tokenizer.save_pretrained("gemma_4_lora") if False: model_1.push_to_hub("HF_USERNAME/gemma_4_lora", token="YOUR_HF_TOKEN") tokenizer.push_to_hub("HF_USERNAME/gemma_4_lora", token="YOUR_HF_TOKEN") return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### GGUF / llama.cpp Conversion To save to `GGUF` / `llama.cpp`, we support it natively now! We clone `llama.cpp` and we default save it to `q8_0`. We allow all methods like `q4_k_m`. Use `save_pretrained_gguf` for local saving and `push_to_hub_gguf` for uploading to HF. Some supported quant methods (full list on our [docs page](https://unsloth.ai/docs/basics/inference-and-deployment/saving-to-gguf)): * `q8_0` - Fast conversion. High resource use, but generally acceptable. * `q4_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_K. * `q5_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_K. [**NEW**] To finetune and auto export to Ollama, try our [Ollama notebook](https://github.com/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Ollama.ipynb) """) return @app.cell def _(model_1, tokenizer): # Save to 8bit Q8_0 if False: # Remember to go to https://huggingface.co/settings/tokens for a token! # And change hf to your username! model_1.save_pretrained_gguf("gemma_4_finetune", tokenizer) if False: # Save to 16bit GGUF model_1.push_to_hub_gguf( "HF_USERNAME/gemma_4_finetune", tokenizer, token="YOUR_HF_TOKEN" ) if False: model_1.save_pretrained_gguf( "gemma_4_finetune", tokenizer, quantization_method="f16" ) # Save to q4_k_m GGUF if False: model_1.push_to_hub_gguf( "HF_USERNAME/gemma_4_finetune", tokenizer, quantization_method="f16", token="YOUR_HF_TOKEN", ) if False: # Save to multiple GGUF options - much faster if you want multiple! model_1.save_pretrained_gguf( "gemma_4_finetune", tokenizer, quantization_method="q4_k_m" ) if False: model_1.push_to_hub_gguf( "HF_USERNAME/gemma_4_finetune", tokenizer, quantization_method="q4_k_m", token="YOUR_HF_TOKEN", ) # Change hf to your username! if False: model_1.push_to_hub_gguf( "HF_USERNAME/gemma_4_finetune", tokenizer, quantization_method=["q4_k_m", "q8_0", "q5_k_m"], token="YOUR_HF_TOKEN", ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Now, use the `gemma_4_finetune.Q8_0.gguf` file or `gemma_4_finetune.Q4_K_M.gguf` file in llama.cpp. And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord! Some other resources: 1. Train your own reasoning model - Llama GRPO notebook [Open in molab](https://github.com/unslothai/notebooks/blob/main/nb/Llama3.1_(8B)-GRPO.ipynb) 2. Saving finetunes to Ollama. [Free notebook](https://github.com/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Ollama.ipynb) 3. Llama 3.2 Vision finetuning - Radiography use case. [Open in molab](https://github.com/unslothai/notebooks/blob/main/nb/Llama3.2_(11B)-Vision.ipynb) 4. See notebooks for DPO, ORPO, Continued pretraining, conversational finetuning and more on our [documentation](https://unsloth.ai/docs/get-started/unsloth-notebooks)!

Join Discord if you need help + ⭐️ Star us on Github ⭐️

This notebook and all Unsloth notebooks are licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme). """) return if __name__ == "__main__": app.run()