# /// script # requires-python = ">=3.10,<3.14" # dependencies = [ # "accelerate", # "bitsandbytes>=0.43.0", # "datasets==4.3.0", # "hf_transfer", # "huggingface_hub>=0.34.0", # "marimo", # "peft", # "protobuf==3.20.3", # "sentencepiece", # "torchao>=0.16.0", # "transformers-cfg", # "transformers==4.56.2", # "triton>=3.2.0", # "trl==0.22.2", # "unsloth @ git+https://github.com/unslothai/unsloth", # "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo", # ] # # [tool.uv] # no-build-package = [ # "bitsandbytes", # "triton", # "vllm", # "xformers", # ] # /// import marimo __generated_with = "0.23.8" app = marimo.App() @app.cell def _(): import marimo as mo return (mo,) @app.cell(hide_code=True) def _(mo): mo.md(r""" To run this notebook, hit the **▶ Run all** button in the bottom-right corner - or use `Ctrl/Cmd + Shift + R`.

Join Discord if you need help + ⭐ Star us on Github ⭐

To install Unsloth on your local device, follow [our guide](https://unsloth.ai/docs/get-started/install). This notebook is licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme). You will learn how to do [data prep](#Data), how to [train](#Train), how to [run the model](#Inference), & how to save it """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### News """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Introducing **Unsloth Studio** - a new open source, no-code web UI to train and run LLMs. [Blog](https://unsloth.ai/docs/new/studio) • [Notebook](https://github.com/unslothai/unsloth/blob/main/studio/Unsloth_Studio_Colab.ipynb)

_{Train models — no code needed}

_{Run GGUF models on Mac, Windows & Linux}

Train MoEs - DeepSeek, GLM, Qwen and gpt-oss 12x faster with 35% less VRAM. [Blog](https://unsloth.ai/docs/new/faster-moe) Ultra Long-Context Reinforcement Learning is here with 7x more context windows! [Blog](https://unsloth.ai/docs/new/grpo-long-context) New in Reinforcement Learning: [FP8 RL](https://unsloth.ai/docs/new/fp8-reinforcement-learning) • [Vision RL](https://unsloth.ai/docs/new/vision-reinforcement-learning-vlm-rl) • [Standby](https://unsloth.ai/docs/basics/memory-efficient-rl) • [gpt-oss RL](https://unsloth.ai/docs/new/gpt-oss-reinforcement-learning) Visit our docs for all our [model uploads](https://unsloth.ai/docs/get-started/unsloth-model-catalog) and [notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks). """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Unsloth """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Tool Calling

Image from [How Function Calling Works](https://huggingface.co/docs/hugs/en/guides/function-calling) """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" By definition, Tool Calling refers to the capability of models to interact with external tools. As you may already know, LLMs are excellent at generating text in the manner of question-and-answer pairs, chat-like conversation messages, and others. When provided useful context, LLMs are good at responding to prompts that can be used as actionable command parameters that another system would handle. This could be a search engine, calculators, company APIs, spreadsheets, or SQL queries. These systems when interacted with are usually done in a single command line or if more complex a runnable scripting language such as Bash or Python. The painful part is how to sequence these commands and assign the correct parameter names and values. What if we would just prompt the LLM to do these for us? For example. "Schedule an appointment for Alice with Bob Monday, 10:00am Eastern time" For this notebook, we'll use a smaller Qwen2.5 model 1.5B. We will guide you on how to prompt a model to respond in JSON format so we can then parse the results and pass those arguments to a Python script. The intuition for using a small model is that we do not want the model to use its pretraining data for the responses when calculating the vector sum. Smaller models have less stored knowledge and it would be possible that our prompt is not on the training data. Our intention is to provide a simple framework for integrating tool calling into your fine-tuning workflow for unsloth. Let us know how we can help you further. """) return @app.cell def _(): from unsloth import FastQwen2Model import torch max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally! dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False. # 4bit pre quantized models we support for 4x faster downloading + no OOMs. fourbit_models = [ "unsloth/Llama-3.1-8B-bnb-4bit", # Llama-3.1 2x faster "unsloth/Llama-3.1-70B-bnb-4bit", "unsloth/Mistral-Small-Instruct-2409", # Mistral 22b 2x faster! "unsloth/mistral-7b-instruct-v0.3-bnb-4bit", "unsloth/Phi-3.5-mini-instruct", # Phi-3.5 2x faster! "unsloth/Phi-3-medium-4k-instruct", "unsloth/gemma-2-27b-bnb-4bit", # Gemma 2x faster! "unsloth/Llama-3.2-1B-bnb-4bit", # NEW! Llama 3.2 models "unsloth/Llama-3.2-1B-Instruct-bnb-4bit", "unsloth/Llama-3.2-3B-Instruct-bnb-4bit", ] # More models at https://huggingface.co/unsloth qwen_models = [ "unsloth/Qwen2.5-Coder-32B-Instruct", # Qwen 2.5 Coder 2x faster "unsloth/Qwen2.5-Coder-7B", "unsloth/Qwen2.5-14B-Instruct", # 14B fits in a 16GB card "unsloth/Qwen2.5-7B", "unsloth/Qwen2.5-72B-Instruct", # 72B fits in a 48GB card ] # More models at https://huggingface.co/unsloth model, tokenizer = FastQwen2Model.from_pretrained( model_name="unsloth/Qwen2.5-Coder-1.5B-Instruct", max_seq_length=None, # Choose any! We auto support RoPE Scaling internally! dtype=None, # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ load_in_4bit=False, # Use 4bit quantization to reduce memory usage. Can be False. fix_tokenizer=False, # token = "YOUR_HF_TOKEN", # HF Token for gated models ) return model, tokenizer @app.cell def _(tokenizer): # save a copy because apply_chat_template() has in-place modifications import copy tokenizer_orig = copy.deepcopy(tokenizer) return copy, tokenizer_orig @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Tool Definitions This is a list of all the functions that we would provide to the model. The standard format is from OpenAI's [Function Calling Definition](https://platform.openai.com/docs/guides/function-calling?api-mode=chat#defining-functions) It is highly possible that the model trained for tool calling was in the OpenAI standard format. Below is an example of two function definitions. The function definitions of `get_vector_sum` and `get_dot_product` will then be added to the prompt as a context for our prompt: > Find the sum of a = [1, -1, 2] and b = [3, 0, -4]. We can just provide the correct one: `get_vector_sum` but to experiment if the model can identify the correct function to call, we will provide both. """) return @app.function def get_tool_definition_list(): return [ { "type": "function", "function": { "name": "get_vector_sum", "description": "Get the sum of two vectors", "parameters": { "type": "object", "properties": { "a": {"type": "list", "description": "First vector"}, "b": {"type": "list", "description": "Second vector"}, }, "required": ["a", "b"], }, }, }, { "type": "function", "function": { "name": "get_dot_product", "description": "Get the dot product of two vectors", "parameters": { "type": "object", "properties": { "a": {"type": "list", "description": "First vector"}, "b": {"type": "list", "description": "Second vector"}, }, "required": ["a", "b"], }, }, }, ] @app.cell(hide_code=True) def _(mo): mo.md(r""" Below is the user prompt declaration """) return @app.cell def _(): user_query = { "role": "user", "content": "Find the sum of a = [1, -1, 2] and b = [3, 0, -4].", } return (user_query,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Python Code Below is the actual code for the function, you may notice that it has Python docstrings. This is because `apply_chat_template()` can accept and translate functions into OpenAI function definitions that are PEP 257 compliant. """) return @app.function def get_vector_sum(a: list[float], b: list[float]) -> list[float]: """ Performs element-wise addition of two numerical vectors. Both vectors must be of the same length and contain numerical values. Args: a: First vector containing numerical values b: Second vector containing numerical values Returns: Resulting vector where each element is the sum of corresponding elements in a and b Raises: ValueError: If vectors have different lengths Example: >>> get_vector_sum([1, 2], [3, 4]) [4, 6] """ if len(a) != len(b): raise ValueError("Vectors must be of the same length") return [x + y for x, y in zip(a, b)] @app.cell(hide_code=True) def _(mo): mo.md(r""" Now let's prompt the model to provide us the arguments in JSON format. You may notice that we passed the actual function `get_vector_sum()` to the `tool` parameter and not the `get_tool_definition_list()`, You may try to change it from `tools=[get_vector_sum],` to `tools=[get_tool_definition_list()` to see if it works with the function definitions. """) return @app.cell def _(copy, tokenizer_orig, user_query): messages = [] messages.append(user_query) tokenizer_1 = copy.deepcopy(tokenizer_orig) input_ids = tokenizer_1.apply_chat_template( messages, tokenize=True, add_generation_prompt=True, add_special_tokens=False, padding=True, tools=[get_vector_sum], # pass the tools return_tensors="pt", ).to("cuda") print(tokenizer_1.decode(input_ids[0])) return input_ids, tokenizer_1 @app.cell(hide_code=True) def _(mo): mo.md(r""" Below is where we call the unsloth function `generate_with_grammar()`. This function uses Grammar-Constrained Decoding. Meaning it will only respond in JSON. It uses a library fork of [transformers-CFG](https://github.com/Saibo-creator/transformers-CFG) by [Saibo-creator](https://github.com/Saibo-creator) the output is very similar to the llama-cpp-python output. We decided to use this library so that our code would be portable to `llama-cpp-python` later during production. If successful, the model should output a single valid JSON response with the following result: ``` [ { "name": "get_vector_sum", "arguments": { "a": [1, -1, 2], "b": [3, 0, -4] } } ] ``` """) return @app.cell def _(AutoTokenizer): # @title Function for Generation Constraint { display-mode: "form" } from functools import partial from transformers_cfg.grammar_utils import IncrementalGrammarConstraint from transformers_cfg.generation.logits_process import ( GrammarConstrainedLogitsProcessor, ) JSON_ARR_GBNF = r""" # This is the same as json.gbnf but we restrict whitespaces at the end of the root array # Useful for generating JSON arrays root ::= arr value ::= object | array | string | number | ("true" | "false" | "null") ws arr ::= "[\n" ws ( value (",\n" ws value)* )? "]" object ::= "{" ws ( string ":" ws value ("," ws string ":" ws value)* )? "}" ws array ::= "[" ws ( value ("," ws value)* )? "]" ws string ::= "\"" ( [^"\\\x7F\x00-\x1F] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes )* "\"" ws number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws # Optional space: by convention, applied in this grammar after literal chars when allowed ws ::= ([ \t\n] ws)? """ def generate_with_grammar(model, input_ids, **kwargs): tokenizer = AutoTokenizer.from_pretrained(model.config.name_or_path) grammar = IncrementalGrammarConstraint( JSON_ARR_GBNF, start_rule_name="root", tokenizer=tokenizer ) grammar_processor = GrammarConstrainedLogitsProcessor(grammar) partial_generate = partial( model.generate, do_sample=False, repetition_penalty=1.1, num_return_sequences=1, logits_processor=[ # Ensure grammar_processor is accessible grammar_processor ], # Ensure grammar_processor is accessible temperature=None, top_p=None, top_k=None, sliding_window=None, ) # Execute generation with merged parameters return partial_generate(input_ids=input_ids, **kwargs) return (generate_with_grammar,) @app.cell def _(generate_with_grammar, input_ids, model, tokenizer_1): output = generate_with_grammar(model=model, input_ids=input_ids) generated_tokens = output[:, input_ids.shape[1] :] decoded_output = tokenizer_1.batch_decode( generated_tokens, skip_special_tokens=True ) for i, message in enumerate(decoded_output): print(f"{message}") return (decoded_output,) @app.cell(hide_code=True) def _(mo): mo.md(r""" From here we can now parse the arguments provided to us by the model """) return @app.cell def _(decoded_output): import json content = json.loads(decoded_output[0]) arguments = content[0]["arguments"] vector_a = arguments["a"] vector_b = arguments["b"] print(f"args a: {vector_a}, b: {vector_b}") return arguments, json, vector_a, vector_b @app.cell(hide_code=True) def _(mo): mo.md(r""" Here we actually call `get_vector_sum()` and capture the result """) return @app.cell def _(vector_a, vector_b): result = get_vector_sum(vector_a, vector_b) print(f"result: {result}") return (result,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Below is the final prompt to the model in the form of a chat message. To ensure that the model responds with the actual answer prompted the model's answer with the following: > You are a super helpful AI assistant. You are asked to answer a question based on the following context information. > > Question: > > Answer: Then we set `continue_final_message=True` for the tokenizer """) return @app.cell def _(arguments, copy, model, result, tokenizer_orig, user_query): import random import string def generate_alphanumeric(): characters = string.ascii_letters + string.digits result = "".join((random.choice(characters) for _ in range(9))) return result messages_1 = [] original_prompt = user_query["content"] prompt_with_context = f"You are a super helpful AI assistant.\nYou are asked to answer a question based on the following context information.\nQuestion:\n{original_prompt}" messages_1.append({"role": "user", "content": prompt_with_context}) tool_call_id = generate_alphanumeric() tool_calls = [ { "id": tool_call_id, "type": "function", "function": {"name": "get_vector_sum", "arguments": arguments}, } ] messages_1.append({"role": "assistant", "tool_calls": tool_calls}) messages_1.append({"role": "tool", "name": "get_vector_sum", "content": result}) messages_1.append({"role": "assistant", "content": "Answer:\n"}) tokenizer_2 = copy.deepcopy(tokenizer_orig) tool_prompt = tokenizer_2.apply_chat_template( messages_1, continue_final_message=True, add_special_tokens=True, return_tensors="pt", return_dict=True, tools=None, # pass the tools ) tool_prompt = tool_prompt.to(model.device) print(tokenizer_2.decode(tool_prompt["input_ids"][0])) return generate_alphanumeric, tokenizer_2, tool_prompt @app.cell def _(model, tokenizer_2, tool_prompt): out = model.generate(**tool_prompt, max_new_tokens=128) generated_text = out[0, tool_prompt["input_ids"].shape[1] :] print(tokenizer_2.decode(generated_text, skip_special_tokens=True)) return @app.cell(hide_code=True) def _(mo): mo.md(r""" For comparison, if we would prompt the model without tool calling: """) return @app.cell def _(copy, tokenizer_orig, user_query): tokenizer_3 = copy.deepcopy(tokenizer_orig) input_ids_1 = tokenizer_3.apply_chat_template( [user_query], tokenize=True, add_generation_prompt=True, add_special_tokens=False, padding=True, tools=None, # pass the tools return_tensors="pt", ).to("cuda") print(tokenizer_3.decode(input_ids_1[0])) return input_ids_1, tokenizer_3 @app.cell def _(input_ids_1, model, tokenizer_3): output_1 = model.generate(input_ids=input_ids_1, max_new_tokens=1024) generated_tokens_1 = output_1[:, input_ids_1.shape[1] :] decoded_output_1 = tokenizer_3.batch_decode( generated_tokens_1, skip_special_tokens=True ) for i_1, message_1 in enumerate(decoded_output_1): print(f"{message_1}") return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Example currency calculation with API call For the example below, we are going to ask the model to compute a total list of items and convert it to Euro using a public API. Pydantic is used below because it will help us with type safety and validation, and also helps us with clear function schemas. For the prompt, we will ask the model "How much is the total cost of all inventory items in Euros?" Based on the prompt we need to define three tools 1. Get a list of all items in the inventory 2. The conversion rate of the Euro currency 2. Compute the total inventory cost in Euros """) return @app.cell def _(): user_query_1 = { "role": "user", "content": "How much is the total cost of all inventory items in Euros?", } return (user_query_1,) @app.cell def _(): from typing import List, Optional, AnyStr from pydantic import BaseModel, Field import requests class Item(BaseModel): id: int | None = Field( default=None, description="Unique identifier for the item (auto-generated by database)", ) item_code: str = Field( ..., min_length=3, max_length=20, description="Unique SKU or product code for the item", ) name: str = Field( ..., min_length=2, max_length=50, description="Human-readable name of the item", ) cost: float = Field( ..., gt=0, description="Unit cost in local currency (must be positive)" ) quantity: int = Field( ..., ge=0, description="Current inventory quantity (non-negative integer)" ) def inventory_check( item_codes: Optional[List[str]], conversion_rate: float ) -> float: """ Calculates the total value of inventory items in the target conversion rate. When item_codes = None, calculates total value for all items. Args: item_codes: List of item codes to include. (None for all items) conversion_rate: Exchange rate to convert costs to target currency Returns: Total value of matching items in target currency, rounded to 2 decimals """ all_items = get_all_items() # Process all items if None is passed if item_codes is None or len(item_codes) == 0: items_to_process = all_items else: # Convert to set for faster lookups target_codes = set(item_codes) items_to_process = [ item for item in all_items if item.item_code in target_codes ] # Calculate total value with conversion total = sum( item.cost * item.quantity * conversion_rate for item in items_to_process ) return round(total, 2) def get_all_items() -> List[Item]: """Fetches all the inventory items""" return [ Item(item_code="ITEM-001", name="Apple", cost=1.13, quantity=4), Item(item_code="ITEM-002", name="Bottled Water", cost=1.04, quantity=20), Item(item_code="ITEM-003", name="Instant Ramen", cost=10.13, quantity=4), ] def get_usd_to_euro_conversion_rate() -> float: """Gets the conversion rate from USD to EURO""" response = requests.get("https://api.frankfurter.app/latest?from=USD") response.raise_for_status() rate = response.json()["rates"]["EUR"] return rate return ( Item, Optional, get_all_items, get_usd_to_euro_conversion_rate, inventory_check, ) @app.cell(hide_code=True) def _(mo): mo.md(r""" Below we validate the `tools` before passing it to the tokenizer """) return @app.cell def _(copy, get_all_items, get_usd_to_euro_conversion_rate, inventory_check): from transformers.utils import chat_template_utils tools = [get_all_items, inventory_check, get_usd_to_euro_conversion_rate] # pass the tools orig_tools = copy.deepcopy(tools) # save a copy for later for tool in tools: _ = chat_template_utils.get_json_schema(tool) return chat_template_utils, orig_tools, tools @app.cell(hide_code=True) def _(mo): mo.md(r""" After ensuring the functions are valid we pass it to the `tools` param """) return @app.cell def _(copy, tokenizer_orig, tools, user_query_1): messages_2 = [] messages_2.append(user_query_1) tokenizer_4 = copy.deepcopy(tokenizer_orig) input_ids_2 = tokenizer_4.apply_chat_template( messages_2, tokenize=True, add_generation_prompt=True, add_special_tokens=False, padding=True, tools=tools, # pass the tools return_tensors="pt", ).to("cuda") print(tokenizer_4.decode(input_ids_2[0])) # pass the tools return input_ids_2, tokenizer_4 @app.cell def _(generate_with_grammar, input_ids_2, model, tokenizer_4): output_2 = generate_with_grammar(model=model, input_ids=input_ids_2) generated_tokens_2 = output_2[:, input_ids_2.shape[1] :] decoded_output_2 = tokenizer_4.batch_decode( generated_tokens_2, skip_special_tokens=True ) for i_2, message_2 in enumerate(decoded_output_2): print(f"{message_2}") return (decoded_output_2,) @app.cell def _(decoded_output_2, json): content_1 = json.loads(decoded_output_2[0]) arguments_1 = content_1[0]["arguments"] item_codes = arguments_1["item_codes"] conversion_rate = arguments_1["conversion_rate"] print(f"item_codes: {item_codes}, conversion_rate: {conversion_rate}") return arguments_1, conversion_rate, item_codes @app.cell(hide_code=True) def _(mo): mo.md(r""" Here we actually call `inventory_check()` and capture the result """) return @app.cell def _(conversion_rate, inventory_check, item_codes): result_total = inventory_check(item_codes, conversion_rate) result_total return (result_total,) @app.cell def _( arguments_1, copy, generate_alphanumeric, model, result_total, tokenizer_orig, user_query_1, ): messages_3 = [] original_prompt_1 = user_query_1["content"] prompt_with_context_1 = f"You are a super helpful AI assistant.\nYou are asked to answer a question based on the following context information.\nQuestion:\n{original_prompt_1}" messages_3.append({"role": "user", "content": prompt_with_context_1}) tool_call_id_1 = generate_alphanumeric() tool_calls_1 = [ { "id": tool_call_id_1, "type": "function", "function": {"name": "inventory_check", "arguments": arguments_1}, } ] messages_3.append({"role": "assistant", "tool_calls": tool_calls_1}) messages_3.append( {"role": "tool", "name": "inventory_check", "content": result_total} ) messages_3.append({"role": "assistant", "content": "Answer:\n"}) tokenizer_5 = copy.deepcopy(tokenizer_orig) tool_prompt_1 = tokenizer_5.apply_chat_template( messages_3, continue_final_message=True, add_special_tokens=True, return_tensors="pt", return_dict=True, tools=None, # pass the tools ) tool_prompt_1 = tool_prompt_1.to(model.device) print(tokenizer_5.decode(tool_prompt_1["input_ids"][0])) return tokenizer_5, tool_prompt_1 @app.cell def _(model, tokenizer_5, tool_prompt_1): out_1 = model.generate(**tool_prompt_1, max_new_tokens=128) generated_text_1 = out_1[0, tool_prompt_1["input_ids"].shape[1] :] print(tokenizer_5.decode(generated_text_1, skip_special_tokens=True)) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's try if the model can use the correct tools for fetching item names. Note that we added a prompt "Ensure to use fetch_item_by_name first for fetching the item code" """) return @app.cell def _(): user_query_2 = { "role": "user", "content": f"How much is the total inventory cost of item name: Bottled Water in Euros? Ensure to use fetch_item_by_name first for fetching the item code", } return (user_query_2,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Below we declare a new function `fetch_item_by_name` which fetches a single item based on the item name. Next we append the new function to our tool list """) return @app.cell def _(Item, Optional, copy, get_all_items, orig_tools): def fetch_item_by_name(item_name: str) -> Optional[Item]: """ Fetch an item by name and returns the Item object. Args: item_name: The human-readable name of the item to fetch Returns: Optional[Item]: The Item with the given name, or None if not found """ all_items = get_all_items() return next((item for item in all_items if item.name == item_name), None) tools_1 = copy.deepcopy(orig_tools) tools_1.insert(0, fetch_item_by_name) return fetch_item_by_name, tools_1 @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's make sure that we have a valid tools definition """) return @app.cell def _(chat_template_utils, tools_1): for tool_1 in tools_1: _ = chat_template_utils.get_json_schema(tool_1) return @app.cell def _(copy, tokenizer_orig, tools_1, user_query_2): messages_4 = [] messages_4.append(user_query_2) tokenizer_6 = copy.deepcopy(tokenizer_orig) input_ids_3 = tokenizer_6.apply_chat_template( messages_4, tokenize=True, add_generation_prompt=True, add_special_tokens=False, padding=True, tools=tools_1, # pass the tools return_tensors="pt", ).to("cuda") print(tokenizer_6.decode(input_ids_3[0])) return input_ids_3, tokenizer_6 @app.cell def _(generate_with_grammar, input_ids_3, model, tokenizer_6): output_3 = generate_with_grammar(model=model, input_ids=input_ids_3) generated_tokens_3 = output_3[:, input_ids_3.shape[1] :] decoded_output_3 = tokenizer_6.batch_decode( generated_tokens_3, skip_special_tokens=True ) for i_3, message_3 in enumerate(decoded_output_3): print(f"{message_3}") return (decoded_output_3,) @app.cell(hide_code=True) def _(mo): mo.md(r""" The model should return the values for the arguments of `fetch_item_by_name` and `inventory_check` """) return @app.cell def _(decoded_output_3, fetch_item_by_name, inventory_check, json): content_2 = json.loads(decoded_output_3[0]) arguments_for_item_name = content_2[0]["arguments"] item_name = arguments_for_item_name["item_name"] item_code = fetch_item_by_name(item_name).item_code print(f"item_name: {item_name}, item_code: {item_code}") arguments_for_inventory_check = content_2[1]["arguments"] conversion_rate_1 = arguments_for_inventory_check["conversion_rate"] print(f"conversion_rate: {conversion_rate_1}") result_total_1 = inventory_check([item_code], conversion_rate_1) result_total_1 return (result_total_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" After computing (actually call the inventory total of Bottled Water, we prompt the model again. Note that we've added the prompt below for for better accuracy. "Answer:\n" """) return @app.cell def _( arguments_1, copy, generate_alphanumeric, model, result_total_1, tokenizer_orig, user_query_2, ): messages_5 = [] original_prompt_2 = user_query_2["content"] prompt_with_context_2 = f"You are a super helpful AI assistant.\nYou are asked to answer a question based on the following context information.\nQuestion:\n{original_prompt_2}" messages_5.append({"role": "user", "content": prompt_with_context_2}) tool_call_id_2 = generate_alphanumeric() tool_calls_2 = [ { "id": tool_call_id_2, "type": "function", "function": {"name": "inventory_check", "arguments": arguments_1}, } ] messages_5.append({"role": "assistant", "tool_calls": tool_calls_2}) messages_5.append( {"role": "tool", "name": "inventory_check", "content": result_total_1} ) messages_5.append({"role": "assistant", "content": "Answer:\n"}) tokenizer_7 = copy.deepcopy(tokenizer_orig) tool_prompt_2 = tokenizer_7.apply_chat_template( messages_5, continue_final_message=True, add_special_tokens=True, return_tensors="pt", return_dict=True, tools=None, # pass the tools ) tool_prompt_2 = tool_prompt_2.to(model.device) print(tokenizer_7.decode(tool_prompt_2["input_ids"][0])) # pass the result total return tokenizer_7, tool_prompt_2 @app.cell def _(model, tokenizer_7, tool_prompt_2): out_2 = model.generate(**tool_prompt_2, max_new_tokens=128) generated_text_2 = out_2[0, tool_prompt_2["input_ids"].shape[1] :] print(tokenizer_7.decode(generated_text_2, skip_special_tokens=True)) return @app.cell(hide_code=True) def _(mo): mo.md(r""" And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord! Some other resources: 1. Looking to use Unsloth locally? Read our [Installation Guide](https://unsloth.ai/docs/get-started/install) for details on installing Unsloth on Windows, Docker, AMD, Intel GPUs. 2. Learn how to do Reinforcement Learning with our [RL Guide and notebooks](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide). 3. Read our guides and notebooks for [Text-to-speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning) and [vision](https://unsloth.ai/docs/basics/vision-fine-tuning) model support. 4. Explore our [LLM Tutorials Directory](https://unsloth.ai/docs/models/tutorials-how-to-fine-tune-and-run-llms) to find dedicated guides for each model. 5. Need help with Inference? Read our [Inference & Deployment page](https://unsloth.ai/docs/basics/inference-and-deployment) for details on using vLLM, llama.cpp, Ollama etc.

Join Discord if you need help + ⭐️ Star us on Github ⭐️ This notebook and all Unsloth notebooks are licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme)

""") return if __name__ == "__main__": app.run()