ailia LLM Tool Use Guide

Tool Use Flow

Set the tools, input a JSON history and generate. After generation, retrieve the response as JSON; if it has tool_calls, execute the tools, append the results to the history and generate the next turn. Clear the tools once the final answer is obtained. The function names in the diagram are C API names; see "API for each language" below for the names in each language.

The response is buffered in the SDK, so the application does not need to concatenate the deltas during generation. Retrieving it with ailiaLLMGetResponseJson does not clear the buffer; it is reset when a new prompt is set.

API for each language

Python, Flutter and Kotlin raise exceptions on error. SetPromptJson in Unity returns a bool. For tool use, use the JSON history from the very first turn.

JSON history

[
  {"role":"user","content":"What is the weather in Tokyo?"},
  {"role":"assistant","content":"","tool_calls":[
    {"id":"call_0","type":"function","function":{
      "name":"get_weather","arguments":"{\"city\":\"Tokyo\"}"}}
  ]},
  {"role":"tool","tool_call_id":"call_0","content":"Snow, -3 C"}
]

The messages array uses the OpenAI-compatible format. Append the assistant message with tool_calls and the resulting tool messages to the history.

• Assistant messages: append the object retrieved with ailiaLLMGetResponseJson as is. Thinking is given in reasoning_content. The content text is not re-parsed as tool calls, so specify tool calls in tool_calls.
• tool_calls: id and function.name cannot be empty. function.arguments is a string that holds a JSON object. IDs are unique within an assistant message; the same ID can be reused in a different turn.
• Tool results: match each result to a call of the immediately preceding assistant message with tool_call_id. The order of the results does not matter. content is a string; convert objects into a JSON string.
• After clearing the tools: after clearing the tools with ailiaLLMSetTools, a history that contains past tool_calls and tool results can still be passed to ailiaLLMSetPromptJson to generate a summary or a final answer. ID matching is still validated. Clearing the tools disables the output parser, so retrieve the response JSON before clearing them.

Images and audio

The content array of a user message can contain images and audio alongside text. While tools are set, ailiaLLMSetMultimodalPrompt cannot be used, so images and audio are also passed in the JSON history. See the Multimodal Guide for how to specify them.

Incomplete output and errors

Calling ailiaLLMGetResponseJson after setting a prompt and before generating returns an empty assistant message.

C++ sample

The full flow: opening the model, setting the tools with ailiaLLMSetTools, generating, returning tool results, and clearing the tools. The history is edited with nlohmann/json. Weather values are mock data, not live weather information.

#include "ailia_llm.h"
#include <nlohmann/json.hpp>
#include <cstdio>
#include <string>
#include <vector>
using json = nlohmann::json;

// Runs one turn: SetPromptJson → Generate → GetResponseJson.
static int generate_turn(AILIALLM* llm, const json& messages, json& response) {
    std::string input = messages.dump();
    int status = ailiaLLMSetPromptJson(llm, input.c_str());
    if (status != AILIA_LLM_STATUS_SUCCESS) return status;
    unsigned int done = 0;
    while (!done) {
        status = ailiaLLMGenerate(llm, &done);
        if (status != AILIA_LLM_STATUS_SUCCESS) return status;
        // Preview with GetDeltaText only if needed. No concatenation for the history.
    }
    unsigned int size = 0;
    status = ailiaLLMGetResponseJsonSize(llm, &size);
    if (status != AILIA_LLM_STATUS_SUCCESS) return status;
    std::vector<char> output(size);
    status = ailiaLLMGetResponseJson(llm, output.data(), size);
    if (status != AILIA_LLM_STATUS_SUCCESS) return status;
    try {
        response = json::parse(output.data());
    } catch (const json::exception&) {
        return AILIA_LLM_STATUS_INVALID_ARGUMENT;
    }
    return AILIA_LLM_STATUS_SUCCESS;
}

static int run_tool_use(AILIALLM* llm) {
    const char* tools = R"([{"type":"function","function":{
        "name":"get_weather","description":"Get the current weather.",
        "parameters":{"type":"object","properties":{"city":{"type":"string"}},
                      "required":["city"]}}}])";
    int status = ailiaLLMSetTools(llm, tools);
    if (status != AILIA_LLM_STATUS_SUCCESS) return status;

    json messages = json::array({{{"role", "user"}, {"content", "What is the weather in Tokyo?"}}});
    for (int turn = 0; turn < 8; turn++) {
        json response;
        status = generate_turn(llm, messages, response);
        // Returns PARSE_ERROR on incomplete output. Decide on a retry without running tools.
        if (status != AILIA_LLM_STATUS_SUCCESS) return status;
        try {
            if (!response.is_object() || response.at("role") != "assistant") {
                return AILIA_LLM_STATUS_INVALID_ARGUMENT;
            }
            messages.push_back(response);
            if (!response.contains("tool_calls") || response["tool_calls"].is_null()) {
                if (!response.contains("content") || !response["content"].is_string() ||
                    response["content"].get<std::string>().empty()) {
                    return AILIA_LLM_STATUS_INVALID_ARGUMENT;
                }
                printf("%s\n", response["content"].get<std::string>().c_str());
                return AILIA_LLM_STATUS_SUCCESS;
            }
            if (!response["tool_calls"].is_array() || response["tool_calls"].empty()) {
                return AILIA_LLM_STATUS_INVALID_ARGUMENT;
            }
            for (const auto& call : response["tool_calls"]) {
                const std::string id = call.at("id").get<std::string>();
                const auto& function = call.at("function");
                if (id.empty() || function.at("name") != "get_weather") {
                    return AILIA_LLM_STATUS_INVALID_ARGUMENT;
                }
                json args = json::parse(function.at("arguments").get<std::string>());
                if (!args.is_object() || !args.contains("city") || !args["city"].is_string() ||
                    args["city"].get<std::string>().empty()) {
                    return AILIA_LLM_STATUS_INVALID_ARGUMENT;
                }
                // Execute the tool with validated arguments. Mock result here.
                json result = {{"city", args["city"]}, {"weather", "sunny"}};
                messages.push_back({{"role", "tool"}, {"tool_call_id", id},
                                    {"content", result.dump()}}); // The result is a JSON string
            }
        } catch (const json::exception&) {
            return AILIA_LLM_STATUS_INVALID_ARGUMENT;
        }
    }
    // Reaching the turn limit after tool calls is not a final answer.
    return AILIA_LLM_STATUS_OTHER_ERROR;
}

int main() {
    AILIALLM* llm = nullptr;
    if (ailiaLLMCreate(&llm) != AILIA_LLM_STATUS_SUCCESS) return 1;
    int status = ailiaLLMOpenModelFileA(llm, "gemma-4-E2B-it-Q4_K_M.gguf", 4096);
    if (status == AILIA_LLM_STATUS_SUCCESS) {
        status = run_tool_use(llm);
        ailiaLLMSetTools(llm, nullptr); // Clear the tools even on error.
    }
    if (status != AILIA_LLM_STATUS_SUCCESS) {
        fprintf(stderr, "error %d: %s\n", status, ailiaLLMGetErrorDetail(llm));
    }
    ailiaLLMDestroy(llm);
    return status == AILIA_LLM_STATUS_SUCCESS ? 0 : 1;
}

Kotlin sample

Pass an AiliaLLM instance with the model already loaded. The sample uses org.json from Android. On JVM environments other than Android, add an equivalent JSON library.

import axip.ailia_llm.AiliaLLM
import org.json.JSONArray
import org.json.JSONObject

// llm has the model loaded. JSON is handled with org.json from Android.
fun runToolUse(llm: AiliaLLM) {
    try {
        llm.setTools("""[{"type":"function","function":{
            "name":"get_weather","description":"Get the current weather.",
            "parameters":{"type":"object","properties":{"city":{"type":"string"}},
                          "required":["city"]}
        }}]""")
        val messages = JSONArray().put(
            JSONObject().put("role", "user").put("content", "What is the weather in Tokyo?")
        )
        for (turn in 0 until 8) {
            llm.setPromptJson(messages.toString())
            while (!llm.generate()) {
                print(llm.getDeltaText()) // Optional preview. No concatenation needed.
            }
            // Throws on incomplete output. Decide on a retry without executing any tool.
            val response = JSONObject(llm.getResponseJson())
            messages.put(response)
            val calls = response.optJSONArray("tool_calls")
            if (calls == null || calls.length() == 0) {
                println(response.optString("content"))
                return
            }
            for (i in 0 until calls.length()) {
                val call = calls.getJSONObject(i)
                val function = call.getJSONObject("function")
                require(function.getString("name") == "get_weather")
                val args = JSONObject(function.getString("arguments"))
                val city = args.getString("city")
                require(city.isNotBlank())
                // Validate the arguments and execute the tool. Mock result here.
                val result = JSONObject().put("city", city).put("weather", "sunny")
                messages.put(JSONObject()
                    .put("role", "tool")
                    .put("tool_call_id", call.getString("id"))
                    .put("content", result.toString())) // The result is a JSON string
            }
        }
        error("No final answer within 8 turns")
    } finally {
        llm.setTools(null)
    }
}

Flutter sample

Pass an AiliaLLMModel instance with the model already loaded. The content of a tool result must be converted into a JSON string instead of being left as a Map.

import 'dart:convert';
import 'dart:io';
import 'package:ailia_llm/ailia_llm_model.dart';

// llm has the model loaded.
void runToolUse(AiliaLLMModel llm) {
  try {
    llm.setTools([{
      'type': 'function',
      'function': {
        'name': 'get_weather',
        'description': 'Get the current weather.',
        'parameters': {
          'type': 'object',
          'properties': {'city': {'type': 'string'}},
          'required': ['city'],
        },
      },
    }]);
    final messages = <Map<String, dynamic>>[
      {'role': 'user', 'content': 'What is the weather in Tokyo?'},
    ];
    for (var turn = 0; turn < 8; turn++) {
      llm.setPromptJson(messages);
      String? delta;
      while ((delta = llm.generate()) != null) {
        stdout.write(delta); // Optional preview. No concatenation needed.
      }
      // Throws on incomplete output. Decide on a retry without executing any tool.
      final response = llm.getResponseJson();
      messages.add(response);
      final calls = response['tool_calls'] as List? ?? [];
      if (calls.isEmpty) {
        print(response['content']);
        return;
      }
      for (final call in calls) {
        final function = call['function'];
        if (function['name'] != 'get_weather') {
          throw StateError('Unknown tool: ${function['name']}');
        }
        final args = jsonDecode(function['arguments'] as String);
        final city = args['city'] as String;
        if (city.isEmpty) throw FormatException('city must not be empty');
        // Validate the arguments and execute the tool. Mock result here.
        final result = {'city': city, 'weather': 'sunny'};
        messages.add({
          'role': 'tool',
          'tool_call_id': call['id'],
          'content': jsonEncode(result), // The result is a JSON string
        });
      }
    }
    throw StateError('No final answer within 8 turns');
  } finally {
    llm.setTools(null);
  }
}

Python sample

Pass an AiliaLLM instance with the model already loaded. Every sample clears the tools in a finally block, including on exceptions.

import json

# model is an ailia_llm.AiliaLLM with the model already loaded.
def run_tool_use(model):
    try:
        model.set_tools([{"type": "function", "function": {
            "name": "get_weather", "description": "Get the current weather.",
            "parameters": {"type": "object", "properties": {"city": {"type": "string"}},
                           "required": ["city"]}}}])
        messages = [{"role": "user", "content": "What is the weather in Tokyo?"}]
        for turn in range(8):
            for delta in model.generate_json(messages):
                print(delta, end="", flush=True)  # Optional preview; no accumulation is needed.
            # Raises on incomplete output. Decide on a retry without running tools.
            response = model.get_response_json()
            messages.append(response)
            calls = response.get("tool_calls") or []
            if not calls:
                print(response["content"])
                return
            for call in calls:
                function = call["function"]
                if function["name"] != "get_weather":
                    raise ValueError("Unknown tool: " + function["name"])
                args = json.loads(function["arguments"])
                city = args.get("city") if isinstance(args, dict) else None
                if not isinstance(city, str) or not city:
                    raise ValueError("Invalid city argument")
                # Validate and execute the tool in the application. Mock result here.
                result = {"city": city, "weather": "sunny"}
                messages.append({"role": "tool", "tool_call_id": call["id"],
                                 "content": json.dumps(result)})  # The result is a JSON string
        raise RuntimeError("No final answer within 8 turns")
    finally:
        model.set_tools(None)

Unity sample

Pass an AiliaLLMModel with a model loaded. Newtonsoft.Json (Unity package com.unity.nuget.newtonsoft-json) is used for JSON. The Unity API returns false or an empty string on failure, so get the details with GetErrorDetail().

using System;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using ailiaLLM;

public static class ToolUseSample
{
    // llm has a model loaded. Newtonsoft.Json is used for JSON.
    public static string RunToolUse(AiliaLLMModel llm)
    {
        try
        {
            var tools = new JArray(new JObject
            {
                ["type"] = "function",
                ["function"] = new JObject
                {
                    ["name"] = "get_weather",
                    ["description"] = "Get the current weather.",
                    ["parameters"] = new JObject
                    {
                        ["type"] = "object",
                        ["properties"] = new JObject { ["city"] = new JObject { ["type"] = "string" } },
                        ["required"] = new JArray("city")
                    }
                }
            });
            if (!llm.SetTools(tools.ToString(Formatting.None))) throw new Exception(llm.GetErrorDetail());
            var messages = new JArray(new JObject { ["role"] = "user", ["content"] = "What is the weather in Tokyo?" });
            for (int turn = 0; turn < 8; turn++)
            {
                if (!llm.SetPromptJson(messages.ToString(Formatting.None))) throw new Exception(llm.GetErrorDetail());
                bool done = false;
                while (true)
                {
                    if (!llm.Generate(ref done)) throw new Exception(llm.GetErrorDetail());
                    if (done) break;
                    // GetDeltaText() can be used for an optional preview; no accumulation is needed.
                }
                // Returns an empty string on error such as incomplete output. Decide on a retry without running tools.
                string responseJson = llm.GetResponseJson();
                if (string.IsNullOrEmpty(responseJson)) throw new Exception(llm.GetErrorDetail());
                var response = JObject.Parse(responseJson);
                messages.Add(response);
                var calls = response["tool_calls"] as JArray;
                if (calls == null || calls.Count == 0)
                {
                    return (string)response["content"];
                }
                foreach (JObject call in calls)
                {
                    var function = (JObject)call["function"];
                    if ((string)function["name"] != "get_weather") throw new Exception("Unknown tool");
                    var args = JObject.Parse((string)function["arguments"]);
                    string city = (string)args["city"];
                    if (string.IsNullOrEmpty(city)) throw new Exception("Invalid city argument");
                    // Validate and execute the tool in the application. Mock result here.
                    var result = new JObject { ["city"] = city, ["weather"] = "sunny" };
                    messages.Add(new JObject
                    {
                        ["role"] = "tool",
                        ["tool_call_id"] = (string)call["id"],
                        ["content"] = result.ToString(Formatting.None) // The result is a JSON string
                    });
                }
            }
            throw new Exception("No final answer within 8 turns");
        }
        finally
        {
            llm.SetTools(null);
        }
    }
}