Buckets:

gijl
/

lv-storage

Files

xet

gijl/lv-storage / llama.cpp /common /chat.h

gijl

15 days ago

download

raw

20.5 kB

// Chat support (incl. tool call grammar constraining & output parsing) w/ generic & custom template handlers.

#pragma once

#include "common.h"
#include "peg-parser.h"
#include "jinja/parser.h"
#include "jinja/runtime.h"
#include "jinja/caps.h"

#include "json.h"

#include <chrono>
#include <functional>
#include <map>
#include <string>
#include <vector>

using chat_template_caps = jinja::caps;

struct common_chat_templates;

namespace autoparser {
struct generation_params;
struct autoparser;
}  // namespace autoparser

struct common_chat_params;
struct common_chat_template;

// Builds the prompt and parser for a template that has a dedicated handler (see common/parsers)
using common_chat_params_init_fn = common_chat_params (*)(const common_chat_template & tmpl,
                                                          const autoparser::generation_params & inputs);

struct common_chat_tool_call {
    std::string name;
    std::string arguments;
    std::string id;

    bool operator==(const common_chat_tool_call & other) const {
        return name == other.name && arguments == other.arguments && id == other.id;
    }
};

struct common_chat_msg_content_part {
    std::string type;
    std::string text;
    std::string name; // tool_reference: name of a tool from the request tools

    // TODO @ngxson : no known chat templates support reasoning_content in content parts yet
    //                this can be useful for models with interleaved thinking (like Kimi-K2)
    //                if you see any templates explicitly support this, please ping me
    // std::string reasoning_content;

    bool operator==(const common_chat_msg_content_part & other) const {
        return type == other.type && text == other.text && name == other.name;
    }
};

struct common_chat_template {
    jinja::program prog;
    std::string bos_tok;
    std::string eos_tok;
    std::string src;
    chat_template_caps caps;
    // Dedicated handler picked once from the source, null when the differential autoparser is used
    common_chat_params_init_fn params_init = nullptr;

    // Differential analysis, run once here when there is no dedicated handler. Null when there
    // is one, or when the analysis failed, in which case analysis_error says why.
    std::unique_ptr<autoparser::autoparser> analysis;
    std::string analysis_error;

    common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token);

    // autoparser is incomplete here, so these are defined where it is complete
    ~common_chat_template();
    common_chat_template(common_chat_template &&);
    common_chat_template & operator=(common_chat_template &&);

    const std::string & source() const { return src; }
    const std::string & bos_token() const { return bos_tok; }
    const std::string & eos_token() const { return eos_tok; }

    chat_template_caps original_caps() const {
        return caps;
    }
};

struct common_chat_msg {
    std::string                               role;
    std::string                               content;
    std::vector<common_chat_msg_content_part> content_parts;
    std::vector<common_chat_tool_call>        tool_calls;
    std::string                               reasoning_content;
    std::string                               tool_name;
    std::string                               tool_call_id;
    // non-OpenAI message keys as a JSON object string, only for templates that read them (DeepSeek V4.1)
    std::string                               extra_json;

    common_json to_json_oaicompat(bool concat_typed_text = false) const;

    std::string render_content(const std::string & delimiter = "\n\n") const;

    bool empty() const {
        return content.empty() && content_parts.empty() && tool_calls.empty() && reasoning_content.empty() &&
               tool_name.empty() && tool_call_id.empty();
    }

    bool contains_media() const {
        for (const auto & part : content_parts) {
            if (part.type == "media_marker") {
                return true;
            }
        }
        return false;
    }

    void set_tool_call_ids(std::vector<std::string> &           ids_cache,
                           const std::function<std::string()> & gen_tool_call_id) {
        for (auto i = 0u; i < tool_calls.size(); i++) {
            if (ids_cache.size() <= i) {
                auto id = tool_calls[i].id;
                if (id.empty()) {
                    id = gen_tool_call_id();
                }
                ids_cache.push_back(id);
            }
            tool_calls[i].id = ids_cache[i];
        }
    }

    bool operator==(const common_chat_msg & other) const {
        return role == other.role && content == other.content && content_parts == other.content_parts &&
               tool_calls == other.tool_calls && reasoning_content == other.reasoning_content &&
               tool_name == other.tool_name && tool_call_id == other.tool_call_id;
    }

    bool operator!=(const common_chat_msg & other) const { return !(*this == other); }
};

struct common_chat_msg_diff {
    std::string           reasoning_content_delta;
    std::string           content_delta;
    size_t                tool_call_index = std::string::npos;
    common_chat_tool_call tool_call_delta;

    static std::vector<common_chat_msg_diff> compute_diffs(const common_chat_msg & msg_prv,
                                                           const common_chat_msg & msg_new);

    bool operator==(const common_chat_msg_diff & other) const {
        return content_delta == other.content_delta && tool_call_index == other.tool_call_index &&
               tool_call_delta == other.tool_call_delta;
    }
};

enum common_chat_role {
    COMMON_CHAT_ROLE_UNKNOWN,
    COMMON_CHAT_ROLE_SYSTEM,
    COMMON_CHAT_ROLE_ASSISTANT,
    COMMON_CHAT_ROLE_USER,
    COMMON_CHAT_ROLE_TOOL
};

common_chat_role common_chat_role_from_string(const std::string & role);
const char *     common_chat_role_to_string(common_chat_role role);

struct common_chat_msg_span {
    common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN;
    std::size_t pos = 0;
    std::size_t len = 0;

    bool valid() const {
        return role != COMMON_CHAT_ROLE_UNKNOWN;
    }
};

struct common_chat_msg_spans {
    std::vector<common_chat_msg_span> spans;

    void add(common_chat_role role, size_t pos, size_t len) {
        spans.push_back({ role, pos, len });
    }

    bool is_user_start(int32_t pos) const {
        for (auto it = spans.begin(); it != spans.end(); ++it) {
            if (it->role == COMMON_CHAT_ROLE_USER && pos == (int32_t) it->pos) {
                return true;
            }
        }
        return false;
    }

    int32_t last_user_message_pos() const {
        for (auto it = spans.rbegin(); it != spans.rend(); ++it) {
            if (it->role == COMMON_CHAT_ROLE_USER) {
                return (int32_t) it->pos;
            }
        }
        return -1;
    }
};

struct common_chat_msg_delimiter {
    common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN;
    std::string      delimiter;
    llama_tokens     tokens = {};
};

struct common_chat_msg_delimiters {
    std::vector<common_chat_msg_delimiter> delimiters;

    common_chat_msg_delimiters() = default;
    common_chat_msg_delimiters(std::initializer_list<common_chat_msg_delimiter> delims) : delimiters(delims) {}

    void add(common_chat_role role, const std::string & delimiter) {
        delimiters.push_back({ role, delimiter });
    }

    void tokenize(const llama_vocab * vocab);

    // split tokens into message spans. skips maps a start index to a length of a region to jump over without matching
    common_chat_msg_spans split(const llama_tokens & tokens, const std::map<size_t, size_t> & skips = {}) const;
};

struct common_chat_tool {
    std::string name;
    std::string description;
    std::string parameters;

    // withheld from the system prompt, a tool_reference reveals it later in the conversation
    bool defer_loading = false;
};

enum common_chat_tool_choice {
    COMMON_CHAT_TOOL_CHOICE_AUTO,
    COMMON_CHAT_TOOL_CHOICE_REQUIRED,
    COMMON_CHAT_TOOL_CHOICE_NONE,
};

enum common_chat_format {
    COMMON_CHAT_FORMAT_CONTENT_ONLY,

    // These are intended to be parsed by the PEG parser
    COMMON_CHAT_FORMAT_PEG_SIMPLE,
    COMMON_CHAT_FORMAT_PEG_NATIVE,
    COMMON_CHAT_FORMAT_PEG_GEMMA4,
    COMMON_CHAT_FORMAT_PEG_MINIMAX_M3,

    COMMON_CHAT_FORMAT_COUNT,  // Not a format, just the # formats
};


// Continuation method provided via `continue_final_message`
enum common_chat_continuation {
    COMMON_CHAT_CONTINUATION_NONE,
    COMMON_CHAT_CONTINUATION_AUTO,
    COMMON_CHAT_CONTINUATION_REASONING,
    COMMON_CHAT_CONTINUATION_CONTENT,
};

struct common_chat_templates_inputs {
    std::vector<common_chat_msg>          messages;
    std::string                           grammar;
    std::string                           json_schema;
    bool                                  add_generation_prompt  = true;
    common_chat_continuation              continue_final_message = COMMON_CHAT_CONTINUATION_NONE;
    bool                                  use_jinja              = true;
    // Parameters below only supported when use_jinja is true
    std::vector<common_chat_tool>         tools;
    common_chat_tool_choice               tool_choice         = COMMON_CHAT_TOOL_CHOICE_AUTO;
    bool                                  parallel_tool_calls = false;
    common_reasoning_format               reasoning_format    = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool enable_thinking"
    bool                                  enable_thinking     = true;
    std::chrono::system_clock::time_point now                 = std::chrono::system_clock::now();
    std::map<std::string, std::string>    chat_template_kwargs;
    bool                                  add_bos = false;
    bool                                  add_eos = false;
    bool                                  force_pure_content = false;
};

struct common_chat_prompt_part {
    std::string text;
    bool        is_input = false; // from user input, special tokens inside must not be parsed
};

// rendered chat template, split into template parts and user input parts
struct common_chat_prompt {
    std::vector<common_chat_prompt_part> parts;

    common_chat_prompt() = default;

    // single part of template text
    explicit common_chat_prompt(std::string text) { parts.push_back({std::move(text), false}); }

    std::string str() const;

    void append(const std::string & text, bool is_input = false);

    // replace the last occurrence of `from`, user input parts are skipped
    void replace_last(const std::string & from, const std::string & to);
};

struct common_chat_params {
    common_chat_format                  format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
    common_chat_prompt                  prompt;
    std::string                         grammar;
    bool                                grammar_lazy         = false;
    std::string                         generation_prompt;
    bool                                supports_thinking    = false;
    std::string                         thinking_start_tag;  // e.g., "<think>"
    std::vector<std::string>            thinking_end_tags;   // e.g., "</think>"
    std::vector<common_grammar_trigger> grammar_triggers;
    std::vector<std::string>            preserved_tokens;
    std::vector<std::string>            additional_stops;
    common_peg_arena                    parser;
    common_chat_msg_delimiters          message_delimiters;
};

struct common_chat_input {
    std::string              text;
    std::vector<llama_token> tokens;

    common_chat_input() = default;

    // plain text, with no tokens
    explicit common_chat_input(std::string text) : text(std::move(text)), tokens(this->text.size(), LLAMA_TOKEN_NULL) {}

    size_t size() const { return text.size(); }
    bool empty() const { return text.empty(); }

    void append(const std::string & piece, llama_token token);
    void append(const common_chat_input & chunk);

    void prepend(const std::string & prefix);
    void prepend(const common_chat_input & prefix);

    void truncate(size_t pos);

    common_chat_input substr(size_t pos, size_t n = std::string::npos) const;
};

common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const std::string & text);

// per-message parsing syntax
// should be derived from common_chat_params
struct common_chat_parser_params {
    common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
    common_chat_input  generation_prompt;
    bool               debug  = false; // Enable debug output for PEG parser
    common_peg_arena   parser = {};
    common_chat_parser_params() = default;
    common_chat_parser_params(const common_chat_params & chat_params) {
        format  = chat_params.format;
        generation_prompt = common_chat_input(chat_params.generation_prompt);
    }
};

// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid
bool common_chat_verify_template(const std::string & tmpl, bool use_jinja);

void common_chat_templates_free(struct common_chat_templates * tmpls);

struct common_chat_templates_deleter {
    void operator()(common_chat_templates * tmpls) { common_chat_templates_free(tmpls); }
};

typedef std::unique_ptr<struct common_chat_templates, common_chat_templates_deleter> common_chat_templates_ptr;

common_chat_templates_ptr common_chat_templates_init(const struct llama_model * model,
                                                     const std::string &        chat_template_override,
                                                     const std::string &        bos_token_override = "",
                                                     const std::string &        eos_token_override = "");

bool        common_chat_templates_was_explicit(const struct common_chat_templates * tmpls);
std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant = "");

struct common_chat_params common_chat_templates_apply(const struct common_chat_templates *        tmpls,
                                                      const struct common_chat_templates_inputs & inputs);

// Format single message, while taking into account the position of that message in chat history
std::string common_chat_format_single(const struct common_chat_templates * tmpls,
                                      const std::vector<common_chat_msg> & past_msg,
                                      const common_chat_msg &              new_msg,
                                      bool                                 add_ass,
                                      bool                                 use_jinja);

// Returns an example of formatted chat
std::string common_chat_format_example(const struct common_chat_templates *       tmpls,
                                       bool                                       use_jinja,
                                       const std::map<std::string, std::string> & chat_template_kwargs);

const char *    common_chat_format_name(common_chat_format format);
common_chat_msg common_chat_parse(const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);

struct common_chat_session_params {
    bool echo  = false; // include the assistant prefill in the output when continuing a message
    bool debug = false; // enable debug output for the PEG parser
};

class common_chat_session {
  public:
    common_chat_session() { result.role = "assistant"; }

    common_chat_session(const common_chat_templates *        tmpls,
                        const llama_vocab *                  vocab,
                        const common_chat_templates_inputs & inputs,
                        const common_chat_session_params &   params = {});

    const common_chat_prompt & prompt() const { return prompt_parts; }
    common_chat_format       format()   const { return parser_params.format; }
    const common_chat_msg &  msg()      const { return result; }
    const common_peg_arena & parser()   const { return parser_params.parser; }

    const std::string &              grammar()            const { return grammar_text; }
    const std::string &              generation_prompt()  const { return generation_prompt_text; }
    const std::string &              thinking_start_tag() const { return thinking_start; }
    const std::vector<std::string> & thinking_end_tags()  const { return thinking_ends; }
    const std::vector<std::string> & additional_stops()   const { return stops; }

    const common_chat_msg_delimiters & message_delimiters() const { return delimiters; }

    void apply_sampling(common_params_sampling & sampling) const;

    bool has_template() const { return templated; }

    const common_chat_msg & feed(const common_chat_input & chunk);

    const common_chat_msg & finish(const common_chat_input & chunk = {});

  private:
    common_chat_prompt                  prompt_parts;
    std::string                         grammar_text;
    bool                                grammar_lazy = false;
    std::vector<common_grammar_trigger> grammar_triggers;
    std::set<llama_token>               preserved_tokens;
    std::vector<std::string>            stops;
    std::string                         generation_prompt_text;
    std::string                         thinking_start;
    std::vector<std::string>            thinking_ends;

    common_chat_parser_params  parser_params;
    common_chat_msg_delimiters delimiters;
    common_chat_input          input;
    common_chat_msg            result;
    bool                       templated = false;
    bool                       finished  = false;
};

// used by arg and server
const char *            common_reasoning_format_name(common_reasoning_format format);
common_reasoning_format common_reasoning_format_from_name(const std::string & format);

common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice);

bool common_chat_templates_support_enable_thinking(const common_chat_templates * chat_templates);

// Parses a JSON array of messages in OpenAI's chat completion API format.
std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const common_json & messages);

std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const common_json & tools);

common_chat_continuation common_chat_continuation_parse(const common_json & value);

// DEPRECATED: only used in tests
common_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text = false);

common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);

// The parameters schema of a function tool. A tool without parameters, or with an empty {}, takes zero arguments.
common_json common_chat_tool_parameters(const common_json & function);

// get template caps, useful for reporting to server /props endpoint
common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates);

common_chat_prompt common_chat_template_direct_apply(
    const common_chat_template & tmpl,
    const autoparser::generation_params & inputs);

std::string common_chat_template_generation_prompt(
    const common_chat_template &          tmpl,
    const autoparser::generation_params & inputs);

std::optional<common_chat_params> common_chat_try_specialized_template(
        const common_chat_template &          tmpl,
        const autoparser::generation_params & params);


// specialized per-task preset
struct common_chat_prompt_preset {
    std::string system;
    std::string user;
    bool has_language = false; // the preset already names the requested language
};

common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "");

// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
// language and voice are optional, the template defines their defaults
std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
                                       const std::string & language = "", const std::string & voice = "");

Xet Storage Details

Size:: 20.5 kB
Xet hash:: 4d4f0caab7849c30902cb6cd99fc3e4616186cef36ca85326566b4fd9f3473fd

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	// Chat support (incl. tool call grammar constraining & output parsing) w/ generic & custom template handlers.

	#pragma once

	#include "common.h"
	#include "peg-parser.h"
	#include "jinja/parser.h"
	#include "jinja/runtime.h"
	#include "jinja/caps.h"

	#include "json.h"

	#include <chrono>
	#include <functional>
	#include <map>
	#include <string>
	#include <vector>

	using chat_template_caps = jinja::caps;

	struct common_chat_templates;

	namespace autoparser {
	struct generation_params;
	struct autoparser;
	} // namespace autoparser

	struct common_chat_params;
	struct common_chat_template;

	// Builds the prompt and parser for a template that has a dedicated handler (see common/parsers)
	using common_chat_params_init_fn = common_chat_params (*)(const common_chat_template & tmpl,
	const autoparser::generation_params & inputs);

	struct common_chat_tool_call {
	std::string name;
	std::string arguments;
	std::string id;

	bool operator==(const common_chat_tool_call & other) const {
	return name == other.name && arguments == other.arguments && id == other.id;
	}
	};

	struct common_chat_msg_content_part {
	std::string type;
	std::string text;
	std::string name; // tool_reference: name of a tool from the request tools

	// TODO @ngxson : no known chat templates support reasoning_content in content parts yet
	// this can be useful for models with interleaved thinking (like Kimi-K2)
	// if you see any templates explicitly support this, please ping me
	// std::string reasoning_content;

	bool operator==(const common_chat_msg_content_part & other) const {
	return type == other.type && text == other.text && name == other.name;
	}
	};

	struct common_chat_template {
	jinja::program prog;
	std::string bos_tok;
	std::string eos_tok;
	std::string src;
	chat_template_caps caps;
	// Dedicated handler picked once from the source, null when the differential autoparser is used
	common_chat_params_init_fn params_init = nullptr;

	// Differential analysis, run once here when there is no dedicated handler. Null when there
	// is one, or when the analysis failed, in which case analysis_error says why.
	std::unique_ptr<autoparser::autoparser> analysis;
	std::string analysis_error;

	common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token);

	// autoparser is incomplete here, so these are defined where it is complete
	~common_chat_template();
	common_chat_template(common_chat_template &&);
	common_chat_template & operator=(common_chat_template &&);

	const std::string & source() const { return src; }
	const std::string & bos_token() const { return bos_tok; }
	const std::string & eos_token() const { return eos_tok; }

	chat_template_caps original_caps() const {
	return caps;
	}
	};

	struct common_chat_msg {
	std::string role;
	std::string content;
	std::vector<common_chat_msg_content_part> content_parts;
	std::vector<common_chat_tool_call> tool_calls;
	std::string reasoning_content;
	std::string tool_name;
	std::string tool_call_id;
	// non-OpenAI message keys as a JSON object string, only for templates that read them (DeepSeek V4.1)
	std::string extra_json;

	common_json to_json_oaicompat(bool concat_typed_text = false) const;

	std::string render_content(const std::string & delimiter = "\n\n") const;

	bool empty() const {
	return content.empty() && content_parts.empty() && tool_calls.empty() && reasoning_content.empty() &&
	tool_name.empty() && tool_call_id.empty();
	}

	bool contains_media() const {
	for (const auto & part : content_parts) {
	if (part.type == "media_marker") {
	return true;
	}
	}
	return false;
	}

	void set_tool_call_ids(std::vector<std::string> & ids_cache,
	const std::function<std::string()> & gen_tool_call_id) {
	for (auto i = 0u; i < tool_calls.size(); i++) {
	if (ids_cache.size() <= i) {
	auto id = tool_calls[i].id;
	if (id.empty()) {
	id = gen_tool_call_id();
	}
	ids_cache.push_back(id);
	}
	tool_calls[i].id = ids_cache[i];
	}
	}

	bool operator==(const common_chat_msg & other) const {
	return role == other.role && content == other.content && content_parts == other.content_parts &&
	tool_calls == other.tool_calls && reasoning_content == other.reasoning_content &&
	tool_name == other.tool_name && tool_call_id == other.tool_call_id;
	}

	bool operator!=(const common_chat_msg & other) const { return !(*this == other); }
	};

	struct common_chat_msg_diff {
	std::string reasoning_content_delta;
	std::string content_delta;
	size_t tool_call_index = std::string::npos;
	common_chat_tool_call tool_call_delta;

	static std::vector<common_chat_msg_diff> compute_diffs(const common_chat_msg & msg_prv,
	const common_chat_msg & msg_new);

	bool operator==(const common_chat_msg_diff & other) const {
	return content_delta == other.content_delta && tool_call_index == other.tool_call_index &&
	tool_call_delta == other.tool_call_delta;
	}
	};

	enum common_chat_role {
	COMMON_CHAT_ROLE_UNKNOWN,
	COMMON_CHAT_ROLE_SYSTEM,
	COMMON_CHAT_ROLE_ASSISTANT,
	COMMON_CHAT_ROLE_USER,
	COMMON_CHAT_ROLE_TOOL
	};

	common_chat_role common_chat_role_from_string(const std::string & role);
	const char * common_chat_role_to_string(common_chat_role role);

	struct common_chat_msg_span {
	common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN;
	std::size_t pos = 0;
	std::size_t len = 0;

	bool valid() const {
	return role != COMMON_CHAT_ROLE_UNKNOWN;
	}
	};

	struct common_chat_msg_spans {
	std::vector<common_chat_msg_span> spans;

	void add(common_chat_role role, size_t pos, size_t len) {
	spans.push_back({ role, pos, len });
	}

	bool is_user_start(int32_t pos) const {
	for (auto it = spans.begin(); it != spans.end(); ++it) {
	if (it->role == COMMON_CHAT_ROLE_USER && pos == (int32_t) it->pos) {
	return true;
	}
	}
	return false;
	}

	int32_t last_user_message_pos() const {
	for (auto it = spans.rbegin(); it != spans.rend(); ++it) {
	if (it->role == COMMON_CHAT_ROLE_USER) {
	return (int32_t) it->pos;
	}
	}
	return -1;
	}
	};

	struct common_chat_msg_delimiter {
	common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN;
	std::string delimiter;
	llama_tokens tokens = {};
	};

	struct common_chat_msg_delimiters {
	std::vector<common_chat_msg_delimiter> delimiters;

	common_chat_msg_delimiters() = default;
	common_chat_msg_delimiters(std::initializer_list<common_chat_msg_delimiter> delims) : delimiters(delims) {}

	void add(common_chat_role role, const std::string & delimiter) {
	delimiters.push_back({ role, delimiter });
	}

	void tokenize(const llama_vocab * vocab);

	// split tokens into message spans. skips maps a start index to a length of a region to jump over without matching
	common_chat_msg_spans split(const llama_tokens & tokens, const std::map<size_t, size_t> & skips = {}) const;
	};

	struct common_chat_tool {
	std::string name;
	std::string description;
	std::string parameters;

	// withheld from the system prompt, a tool_reference reveals it later in the conversation
	bool defer_loading = false;
	};

	enum common_chat_tool_choice {
	COMMON_CHAT_TOOL_CHOICE_AUTO,
	COMMON_CHAT_TOOL_CHOICE_REQUIRED,
	COMMON_CHAT_TOOL_CHOICE_NONE,
	};

	enum common_chat_format {
	COMMON_CHAT_FORMAT_CONTENT_ONLY,

	// These are intended to be parsed by the PEG parser
	COMMON_CHAT_FORMAT_PEG_SIMPLE,
	COMMON_CHAT_FORMAT_PEG_NATIVE,
	COMMON_CHAT_FORMAT_PEG_GEMMA4,
	COMMON_CHAT_FORMAT_PEG_MINIMAX_M3,

	COMMON_CHAT_FORMAT_COUNT, // Not a format, just the # formats
	};


	// Continuation method provided via `continue_final_message`
	enum common_chat_continuation {
	COMMON_CHAT_CONTINUATION_NONE,
	COMMON_CHAT_CONTINUATION_AUTO,
	COMMON_CHAT_CONTINUATION_REASONING,
	COMMON_CHAT_CONTINUATION_CONTENT,
	};

	struct common_chat_templates_inputs {
	std::vector<common_chat_msg> messages;
	std::string grammar;
	std::string json_schema;
	bool add_generation_prompt = true;
	common_chat_continuation continue_final_message = COMMON_CHAT_CONTINUATION_NONE;
	bool use_jinja = true;
	// Parameters below only supported when use_jinja is true
	std::vector<common_chat_tool> tools;
	common_chat_tool_choice tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO;
	bool parallel_tool_calls = false;
	common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool enable_thinking"
	bool enable_thinking = true;
	std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
	std::map<std::string, std::string> chat_template_kwargs;
	bool add_bos = false;
	bool add_eos = false;
	bool force_pure_content = false;
	};

	struct common_chat_prompt_part {
	std::string text;
	bool is_input = false; // from user input, special tokens inside must not be parsed
	};

	// rendered chat template, split into template parts and user input parts
	struct common_chat_prompt {
	std::vector<common_chat_prompt_part> parts;

	common_chat_prompt() = default;

	// single part of template text
	explicit common_chat_prompt(std::string text) { parts.push_back({std::move(text), false}); }

	std::string str() const;

	void append(const std::string & text, bool is_input = false);

	// replace the last occurrence of `from`, user input parts are skipped
	void replace_last(const std::string & from, const std::string & to);
	};

	struct common_chat_params {
	common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
	common_chat_prompt prompt;
	std::string grammar;
	bool grammar_lazy = false;
	std::string generation_prompt;
	bool supports_thinking = false;
	std::string thinking_start_tag; // e.g., "<think>"
	std::vector<std::string> thinking_end_tags; // e.g., "</think>"
	std::vector<common_grammar_trigger> grammar_triggers;
	std::vector<std::string> preserved_tokens;
	std::vector<std::string> additional_stops;
	common_peg_arena parser;
	common_chat_msg_delimiters message_delimiters;
	};

	struct common_chat_input {
	std::string text;
	std::vector<llama_token> tokens;

	common_chat_input() = default;

	// plain text, with no tokens
	explicit common_chat_input(std::string text) : text(std::move(text)), tokens(this->text.size(), LLAMA_TOKEN_NULL) {}

	size_t size() const { return text.size(); }
	bool empty() const { return text.empty(); }

	void append(const std::string & piece, llama_token token);
	void append(const common_chat_input & chunk);

	void prepend(const std::string & prefix);
	void prepend(const common_chat_input & prefix);

	void truncate(size_t pos);

	common_chat_input substr(size_t pos, size_t n = std::string::npos) const;
	};

	common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const std::string & text);

	// per-message parsing syntax
	// should be derived from common_chat_params
	struct common_chat_parser_params {
	common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
	common_chat_input generation_prompt;
	bool debug = false; // Enable debug output for PEG parser
	common_peg_arena parser = {};
	common_chat_parser_params() = default;
	common_chat_parser_params(const common_chat_params & chat_params) {
	format = chat_params.format;
	generation_prompt = common_chat_input(chat_params.generation_prompt);
	}
	};

	// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid
	bool common_chat_verify_template(const std::string & tmpl, bool use_jinja);

	void common_chat_templates_free(struct common_chat_templates * tmpls);

	struct common_chat_templates_deleter {
	void operator()(common_chat_templates * tmpls) { common_chat_templates_free(tmpls); }
	};

	typedef std::unique_ptr<struct common_chat_templates, common_chat_templates_deleter> common_chat_templates_ptr;

	common_chat_templates_ptr common_chat_templates_init(const struct llama_model * model,
	const std::string & chat_template_override,
	const std::string & bos_token_override = "",
	const std::string & eos_token_override = "");

	bool common_chat_templates_was_explicit(const struct common_chat_templates * tmpls);
	std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant = "");

	struct common_chat_params common_chat_templates_apply(const struct common_chat_templates * tmpls,
	const struct common_chat_templates_inputs & inputs);

	// Format single message, while taking into account the position of that message in chat history
	std::string common_chat_format_single(const struct common_chat_templates * tmpls,
	const std::vector<common_chat_msg> & past_msg,
	const common_chat_msg & new_msg,
	bool add_ass,
	bool use_jinja);

	// Returns an example of formatted chat
	std::string common_chat_format_example(const struct common_chat_templates * tmpls,
	bool use_jinja,
	const std::map<std::string, std::string> & chat_template_kwargs);

	const char * common_chat_format_name(common_chat_format format);
	common_chat_msg common_chat_parse(const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
	common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);

	struct common_chat_session_params {
	bool echo = false; // include the assistant prefill in the output when continuing a message
	bool debug = false; // enable debug output for the PEG parser
	};

	class common_chat_session {
	public:
	common_chat_session() { result.role = "assistant"; }

	common_chat_session(const common_chat_templates * tmpls,
	const llama_vocab * vocab,
	const common_chat_templates_inputs & inputs,
	const common_chat_session_params & params = {});

	const common_chat_prompt & prompt() const { return prompt_parts; }
	common_chat_format format() const { return parser_params.format; }
	const common_chat_msg & msg() const { return result; }
	const common_peg_arena & parser() const { return parser_params.parser; }

	const std::string & grammar() const { return grammar_text; }
	const std::string & generation_prompt() const { return generation_prompt_text; }
	const std::string & thinking_start_tag() const { return thinking_start; }
	const std::vector<std::string> & thinking_end_tags() const { return thinking_ends; }
	const std::vector<std::string> & additional_stops() const { return stops; }

	const common_chat_msg_delimiters & message_delimiters() const { return delimiters; }

	void apply_sampling(common_params_sampling & sampling) const;

	bool has_template() const { return templated; }

	const common_chat_msg & feed(const common_chat_input & chunk);

	const common_chat_msg & finish(const common_chat_input & chunk = {});

	private:
	common_chat_prompt prompt_parts;
	std::string grammar_text;
	bool grammar_lazy = false;
	std::vector<common_grammar_trigger> grammar_triggers;
	std::set<llama_token> preserved_tokens;
	std::vector<std::string> stops;
	std::string generation_prompt_text;
	std::string thinking_start;
	std::vector<std::string> thinking_ends;

	common_chat_parser_params parser_params;
	common_chat_msg_delimiters delimiters;
	common_chat_input input;
	common_chat_msg result;
	bool templated = false;
	bool finished = false;
	};

	// used by arg and server
	const char * common_reasoning_format_name(common_reasoning_format format);
	common_reasoning_format common_reasoning_format_from_name(const std::string & format);

	common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice);

	bool common_chat_templates_support_enable_thinking(const common_chat_templates * chat_templates);

	// Parses a JSON array of messages in OpenAI's chat completion API format.
	std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const common_json & messages);

	std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const common_json & tools);

	common_chat_continuation common_chat_continuation_parse(const common_json & value);

	// DEPRECATED: only used in tests
	common_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text = false);

	common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);

	// The parameters schema of a function tool. A tool without parameters, or with an empty {}, takes zero arguments.
	common_json common_chat_tool_parameters(const common_json & function);

	// get template caps, useful for reporting to server /props endpoint
	common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates);

	common_chat_prompt common_chat_template_direct_apply(
	const common_chat_template & tmpl,
	const autoparser::generation_params & inputs);

	std::string common_chat_template_generation_prompt(
	const common_chat_template & tmpl,
	const autoparser::generation_params & inputs);

	std::optional<common_chat_params> common_chat_try_specialized_template(
	const common_chat_template & tmpl,
	const autoparser::generation_params & params);


	// specialized per-task preset
	struct common_chat_prompt_preset {
	std::string system;
	std::string user;
	bool has_language = false; // the preset already names the requested language
	};

	common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "");

	// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
	// language and voice are optional, the template defines their defaults
	std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
	const std::string & language = "", const std::string & voice = "");