Something went wrong. Try again.
A single-model (Bonsai-27B) inference engine for Intel Arc GPUs
Something went wrong. Try again.
C++
at extra-debug
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333#include "qxmx_chat_template.h"
#include <algorithm>#include <cstdio>#include <sstream>#include <string>#include <vector>
#include "qxmx_json.h"#include "qxmx_tokenizer.h"
namespace qx {
namespace {
/* ---- special token ids (verified via marker_probe on the real GGUF) ---- * id 248044 is bos/pad (endoftext); not emitted by the renderer (add_bos=0). * The thinking tags (248068/248069) are ALSO single special tokens (type=USER); * llama.cpp tokenizes them atomically, so the renderer must emit them by id * rather than byte-encoding the text (which would split them wrong). */constexpr int32_t ID_IM_START = 248045;constexpr int32_t ID_IM_END = 248046; /* == eos */constexpr int32_t ID_TC_OPEN = CHAT_ID_TC_OPEN;constexpr int32_t ID_TC_CLOSE = CHAT_ID_TC_CLOSE;constexpr int32_t ID_TR_OPEN = 248066; /* tool-response open */constexpr int32_t ID_TR_CLOSE = 248067; /* tool-response close */constexpr int32_t ID_THINK_O = 248068; /* thinking open tag */constexpr int32_t ID_THINK_C = 248069; /* thinking close tag */
/* Build an angle-bracketed marker from a fragment that contains no '<' or '>'. * All literal markers in the model's template are constructed this way so the * renderer's source never embeds a raw special-tag sequence. */inline std::string mk(const char* body, bool closing = false) { std::string s; s += '<'; if (closing) s += '/'; s += body; s += '>'; return s;}
/* im_start / im_end are the only markers with the <|...|> shape. */inline std::string im_marker(const char* body) { return std::string("<|") + body + "|>";}
/* Serialize an OpenAI tool object the way Jinja's tojson would (HF/llama.cpp * default spacing ", "/": "): * {"type": "function", "function": {"name": ..., "description": ..., "parameters": ...}} * parameters_json is parsed then re-serialized so the spacing is normalized * regardless of how the caller spaced it; "" / missing -> null. */std::string tool_tojson(const chat_tool& t) { std::string params = "null"; if (!t.parameters_json.empty()) { jval pv; if (jparse(t.parameters_json, pv)) { params.clear(); jserialize(params, pv); } } std::string out = "{\"type\": \"function\", \"function\": {\"name\": \""; out += json_escape(t.name); out += "\", \"description\": \""; out += json_escape(t.description); out += "\", \"parameters\": "; out += params; out += "}}"; return out;}
/* ---- segment list: text runs + single special-token ids ---- * Building the whole render as segments lets text-render and tokenize share * one structure. Text segments are byte-BPE-encoded independently; specials * are emitted by id. This mirrors llama.cpp's "split on specials, then BPE * each piece" semantics, so no BPE merge crosses a special boundary. */struct seg { bool is_special; int32_t special_id; /* valid when is_special */ std::string text; /* valid when !is_special */};
struct builder { std::vector<seg> segs;
/* Append a text run. Adjacent text runs (with no special between them) * are coalesced into a single segment so BPE merges across the original * call boundaries happen exactly as llama.cpp would produce for the * concatenated string (merges must not be artificially broken at a * boundary where no special token actually sits). */ void text(const std::string& s) { if (s.empty()) return; if (!segs.empty() && !segs.back().is_special) { segs.back().text += s; } else { segs.push_back({false, 0, s}); } } void special(int32_t id) { segs.push_back({true, id, {}}); }};
/* ---- the synthetic "# Tools" system message (template lines 45-60) ---- */void emit_tools_system(builder& b, const std::vector<chat_tool>& tools, const chat_message* sys) { b.special(ID_IM_START); b.text("system\n# Tools\n\nYou have access to the following functions:\n\n"); b.text(mk("tools")); for (const chat_tool& t : tools) { b.text("\n"); b.text(tool_tojson(t)); } b.text("\n"); b.text(mk("tools", true)); b.text("\n\nIf you choose to call a function ONLY reply in the following " "format with NO suffix:\n\n"); /* the example block: tc_open, function example, two parameters, tc_close */ b.special(ID_TC_OPEN); b.text("\n<function=example_function_name>\n" "<parameter=example_parameter_1>\nvalue_1\n</parameter>\n" "<parameter=example_parameter_2>\nThis is the value for the second " "parameter\nthat can span\nmultiple lines\n</parameter>\n" "</function>\n"); b.special(ID_TC_CLOSE); b.text("\n\n"); b.text(mk("IMPORTANT")); b.text("\nReminder:\n" "- Function calls MUST follow the specified format: an inner " "<function=...></function> block must be nested within "); b.special(ID_TC_OPEN); b.special(ID_TC_CLOSE); b.text(" XML tags\n" "- Required parameters MUST be specified\n" "- You may provide optional reasoning for your function call in " "natural language BEFORE the function call, but NOT after\n" "- If there is no function call available, answer the question like " "normal with your current knowledge and do not tell the user about " "function calls\n"); b.text(mk("IMPORTANT", true)); /* append user-supplied system content (if any) */ if (sys && !sys->content.empty()) { b.text("\n\n"); b.text(sys->content); } b.special(ID_IM_END); b.text("\n");}
/* ---- one assistant turn's tool_calls (template lines 105-129) ---- */void emit_assistant_tool_calls(builder& b, const chat_message& m) { bool first = true; bool have_content = !m.content.empty(); /* Each tool_call is: [maybe \n or \n\n][tc_open] BODY [tc_close], where * BODY is a single text run so BPE merges across the function/parameter * markup boundaries (e.g. =get) happen exactly as llama.cpp would produce * for the whole string. Splitting BODY into per-piece text segments would * break those cross-boundary merges. */ for (const chat_tool_call& tc : m.tool_calls) { if (first) { if (have_content) b.text("\n\n"); b.special(ID_TC_OPEN); } else { b.text("\n"); b.special(ID_TC_OPEN); } std::string body; body += "\n<function="; body += tc.name; body += ">\n"; if (!tc.arguments_json.empty()) { jval av; if (jparse(tc.arguments_json, av) && av.t == jval::OBJ) { for (const auto& kv : av.obj) { body += "<parameter="; body += kv.first; body += ">\n"; if (kv.second.t == jval::STR) { body += kv.second.s; } else { jserialize(body, kv.second); } body += "\n</parameter>\n"; } } } body += "</function>\n"; b.text(body); b.special(ID_TC_CLOSE); first = false; }}
/* ---- the generation prompt (template lines 147-153) ---- */void emit_generation_prompt(builder& b, bool enable_thinking) { b.special(ID_IM_START); b.text("assistant\n"); if (!enable_thinking) { /* think-open + blank line + think-close (template L150). The thinking * tags are single special tokens (248068/248069), emitted by id. */ b.special(ID_THINK_O); b.text("\n\n"); b.special(ID_THINK_C); b.text("\n\n"); } else { b.special(ID_THINK_O); b.text("\n"); }}
/* ---- the full render (shared by text + tokenize) ---- */void build_render(builder& b, const std::vector<chat_message>& messages, const std::vector<chat_tool>& tools, bool add_generation_prompt, bool enable_thinking) { const chat_message* sys = (!messages.empty() && messages[0].role == "system") ? &messages[0] : nullptr;
if (!tools.empty()) { emit_tools_system(b, tools, sys); } else if (sys) { b.special(ID_IM_START); b.text("system\n"); b.text(sys->content); b.special(ID_IM_END); b.text("\n"); }
/* message body: skip the system message if we already emitted it above * (either as the synthetic "# Tools" system msg, when tools present, or * as a plain system turn otherwise). */ size_t start = sys ? 1 : 0;
for (size_t i = start; i < messages.size(); i++) { const chat_message& m = messages[i]; if (m.role == "user") { b.special(ID_IM_START); b.text("user\n"); b.text(m.content); b.special(ID_IM_END); b.text("\n"); } else if (m.role == "assistant") { b.special(ID_IM_START); b.text("assistant\n"); b.text(m.content); if (!m.tool_calls.empty()) emit_assistant_tool_calls(b, m); b.special(ID_IM_END); b.text("\n"); } else if (m.role == "tool") { /* tool result: wrapped as a user turn with the tool-response * markers (single special tokens 248066/248067). <|im_start|>user * is emitted only if the previous message was not itself a tool * message. */ bool prev_is_tool = (i > 0) && (messages[i-1].role == "tool"); if (!prev_is_tool) { b.special(ID_IM_START); b.text("user"); } b.text("\n"); b.special(ID_TR_OPEN); b.text("\n"); b.text(m.content); b.text("\n"); b.special(ID_TR_CLOSE); bool next_is_tool = (i + 1 < messages.size()) && (messages[i+1].role == "tool"); if (!next_is_tool) { b.special(ID_IM_END); b.text("\n"); } } /* system messages beyond index 0 raise in the real template; we * silently ignore them (defensive). */ }
if (add_generation_prompt) emit_generation_prompt(b, enable_thinking);}
} // namespace
std::string render_chat_template_text( const std::vector<chat_message>& messages, const std::vector<chat_tool>& tools, bool add_generation_prompt, bool enable_thinking) { static const std::string im_start_text = im_marker("im_start"); static const std::string im_end_text = im_marker("im_end"); static const std::string tc_open_text = mk("tool_call"); static const std::string tc_close_text = mk("tool_call", true); static const std::string tr_open_text = mk("tool_response"); static const std::string tr_close_text = mk("tool_response", true); static const std::string think_o_text = mk("think"); static const std::string think_c_text = mk("think", true); builder b; build_render(b, messages, tools, add_generation_prompt, enable_thinking); std::string out; for (const seg& s : b.segs) { if (s.is_special) { switch (s.special_id) { case ID_IM_START: out += im_start_text; break; case ID_IM_END: out += im_end_text; break; case ID_TC_OPEN: out += tc_open_text; break; case ID_TC_CLOSE: out += tc_close_text; break; case ID_TR_OPEN: out += tr_open_text; break; case ID_TR_CLOSE: out += tr_close_text; break; case ID_THINK_O: out += think_o_text; break; case ID_THINK_C: out += think_c_text; break; default: break; } } else { out += s.text; } } return out;}
std::vector<int32_t> render_chat_template( const tokenizer& tok, const std::vector<chat_message>& messages, const std::vector<chat_tool>& tools, bool add_generation_prompt, bool enable_thinking) { builder b; build_render(b, messages, tools, add_generation_prompt, enable_thinking); std::vector<int32_t> ids; std::vector<int32_t> tmp; for (const seg& s : b.segs) { if (s.is_special) { ids.push_back(s.special_id); } else { tmp.clear(); tok.encode(s.text, &tmp, /*add_special=*/false); for (int32_t id : tmp) ids.push_back(id); } } return ids;}
} // namespace qx