Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
JavaScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242#!/usr/bin/env node// Renderer checks that need the real Tinker Python environment: structured// thinking must survive the renderer a model actually uses, and the generated// SFT script must fail closed before any training client is created.//// These live outside `unit-test` because they cannot run without// `tinker-cookbook` installed in a project or package `.venv`. Keeping them// here lets Tangled CI stay hermetic while `npm test` still runs them locally.// This is a local-only lane, like `local-integration-test`, which needs a real// `pi` binary: it reports a missing environment plainly instead of failing with// a renderer error that looks like a code defect.import { execFileSync } from "node:child_process";import { existsSync, mkdtempSync, writeFileSync, readFileSync } from "node:fs";import os from "node:os";import path from "node:path";import { probeInklingThinkingRenderer } from "../lib/reasoning-machine.ts";import { formatThinkingValidationFailure, validateThinkingDatasetForModel } from "../lib/thinking-renderer.ts";import { pythonCandidates } from "../lib/python.ts";
let checks = 0;function assert(condition, message) { if (!condition) throw new Error(message); checks++;}
const repoRoot = path.resolve(import.meta.dirname, "..");const tmp = mkdtempSync(path.join(os.tmpdir(), "machine-thinking-"));
// Resolve a venv interpreter explicitly. `pythonBin` falls back to bare// `python3`, which silently turns "deps missing" into a confusing failure.const venvPython = pythonCandidates(repoRoot, repoRoot).find((candidate) => existsSync(candidate));if (!venvPython) { console.error([ "thinking-test needs a Python environment with tinker-cookbook installed.", "", "No .venv found. From the repository root:", " uv venv --python 3.12 .venv", " uv pip install -p .venv/bin/python -U tinker-cookbook tml-renderers", "", "This lane is local-only; Tangled CI does not run it.", ].join("\n")); process.exit(1);}try { execFileSync(venvPython, ["-c", "import tinker_cookbook"], { stdio: "pipe" });} catch (error) { console.error([ `thinking-test found ${venvPython} but it cannot import tinker_cookbook.`, "", " uv pip install -p .venv/bin/python -U tinker-cookbook tml-renderers", "", `Underlying error: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}`, ].join("\n")); process.exit(1);}
const thinkingJsonl = path.join(tmp, "thinking-pool.jsonl");writeFileSync(thinkingJsonl, `${JSON.stringify({ tag: "good", messages: [ { role: "user", content: "Good morning." }, { role: "assistant", content: [{ type: "thinking", thinking: "This is a casual greeting rather than a substantive request." }, { type: "text", text: "Good morning." }] }, ],})}\n`);
// A thinking row late in a long file must still be found.const lateThinkingJsonl = path.join(tmp, "late-thinking.jsonl");const lateLines = [];for (let i = 0; i < 201; i++) { lateLines.push(JSON.stringify({ tag: "good", messages: [{ role: "user", content: `plain ${i}` }, { role: "assistant", content: `ok ${i}` }] }));}lateLines.push(JSON.stringify({ tag: "good", messages: [ { role: "user", content: "late" }, { role: "assistant", content: [{ type: "thinking", thinking: "Gather the required facts before answering." }, { type: "text", text: "done" }] }, ],}));writeFileSync(lateThinkingJsonl, `${lateLines.join("\n")}\n`);
const inklingGate = validateThinkingDatasetForModel({ repoRoot, cwd: repoRoot, dataFile: thinkingJsonl, model: "thinkingmachines/Inkling-Small", maxExamples: 0 });assert(inklingGate.ok, `thinking dataset should validate for Inkling: ${inklingGate.reason ?? inklingGate.error ?? "unknown"}`);
const lateGate = validateThinkingDatasetForModel({ repoRoot, cwd: repoRoot, dataFile: lateThinkingJsonl, model: "thinkingmachines/Inkling-Small", maxExamples: 0 });assert(lateGate.hasThinkingDataset, "late thinking after 200 plain rows must be detected");assert(lateGate.totalThinkingRows === 1, "late-thinking fixture reports one thinking row");assert(lateGate.ok, `late thinking must validate on Inkling: ${lateGate.reason ?? lateGate.error ?? "unknown"}`);
const roleColonGate = validateThinkingDatasetForModel({ repoRoot, cwd: repoRoot, dataFile: thinkingJsonl, model: "Qwen/Qwen3.5-9B-Base", maxExamples: 0 });assert(roleColonGate.hasThinkingDataset, "role_colon rejection still sees thinking dataset");assert(!roleColonGate.ok, "role_colon / non-thinking renderer must reject structured thinking");assert(formatThinkingValidationFailure(roleColonGate, "Qwen/Qwen3.5-9B-Base").includes("structured thinking"), "failure message names thinking incompatibility");
const qwenGate = validateThinkingDatasetForModel({ repoRoot, cwd: repoRoot, dataFile: thinkingJsonl, model: "Qwen/Qwen3-8B", maxExamples: 0 });assert(qwenGate.ok, `Qwen3-8B <think> channel should accept structured thinking: ${qwenGate.reason ?? qwenGate.error ?? "unknown"}`);assert(qwenGate.renderer === "qwen3" || qwenGate.receipts?.[0]?.hasQwenThink || qwenGate.receipts?.[0]?.hasThinkingChannel, "Qwen receipt notes thinking channel");
// The generated SFT script must refuse an unsupported thinking renderer before// it initializes a training client, so a bad run costs nothing.const template = readFileSync(path.join(repoRoot, "templates", "train_sft.py"), "utf8");const trainProbePath = path.join(tmp, "train_sft_probe.py");writeFileSync(trainProbePath, template .replaceAll("__PACKAGE_ROOT__", JSON.stringify(repoRoot)) .replaceAll("__MODEL__", JSON.stringify("Qwen/Qwen3.5-9B-Base")) .replaceAll("__MAX_LENGTH__", "32768") .replaceAll("__BATCH_SIZE__", "8") .replaceAll("__DATA_FILE__", JSON.stringify(thinkingJsonl)) .replaceAll("__TEST_SIZE__", "0") .replaceAll("__LOG_PATH__", JSON.stringify(path.join(tmp, "logs"))) .replaceAll("__LEARNING_RATE__", "2e-4") .replaceAll("__NUM_EPOCHS__", "1") .replaceAll("__MAX_STEPS__", "1"));try { execFileSync(venvPython, [trainProbePath], { stdio: "pipe" }); assert(false, "generated train_sft should fail before client init for unsupported thinking renderer");} catch (error) { const output = `${error.stdout ?? ""}${error.stderr ?? ""}`; assert(/thinking-renderer|structured thinking|SystemExit|blocked/i.test(output), "generated train_sft fails before training client for thinking dataset on unsupported model"); assert(/"hasThinkingDataset": true|"blocked": true|thinking-renderer/i.test(output), "generated script emits fail-closed thinking receipt");}
// Renderer receipts. Counts are observed evidence, not fixtures, so assert the// invariants (a thinking channel exists, targets are weighted) not the numbers.const probe = probeInklingThinkingRenderer({ repoRoot, cwd: repoRoot });assert(probe.ok, `inkling thinking renderer probe failed: ${probe.error ?? "unknown"}`);assert(!probe.extensionPropertyWarning, "Inkling probe must not emit extension-property ALL warning");assert(probe.answerFirst?.hasContentThinking || probe.answerFirst?.hasThinkingChannel, "answer-first probe renders a thinking channel");assert(Number(probe.answerFirst?.weightedTokens) > 0, "answer-first probe has positively weighted tokens");assert(probe.answerFirst?.targetCount === 1, "answer-first expands to one LAST target");assert(probe.toolCallFirst?.hasContentThinking || probe.toolCallFirst?.hasThinkingChannel, "tool-call-first probe renders a thinking channel");assert(probe.toolCallFirst?.toolPreserved, "tool-call-first probe preserves tool calls");assert(Number(probe.toolCallFirst?.weightedTokens) > 0, "tool-call-first probe has positively weighted tokens");assert(probe.toolCallFirst?.targetCount === 2, "tool-call-first expands to two LAST targets");const toolTargets = probe.toolCallFirst?.targetReceipts;assert(Array.isArray(toolTargets) && toolTargets.length === 2, "tool-call-first reports two target receipts");assert(toolTargets[0]?.hasThinking && (Array.isArray(toolTargets[0]?.toolNames) ? toolTargets[0].toolNames.includes("bash") : toolTargets[0]?.toolPreserved), "first tool target keeps thinking+tool call");assert(toolTargets[1]?.ok && toolTargets[1]?.toolContextPreserved !== false, "second tool target keeps prior tool context");
// The dataset builder must produce weighted LAST targets without a Tinker client.const builderSmoke = path.join(tmp, "builder_smoke.py");writeFileSync(builderSmoke, `import json, sys, tempfilefrom pathlib import Pathrepo = Path(${JSON.stringify(repoRoot)})sys.path.insert(0, str(repo / "scripts"))template = (repo / "templates" / "train_sft.py").read_text(encoding="utf-8")rendered = templatefor key, value in { "__PACKAGE_ROOT__": json.dumps(str(repo)), "__MODEL__": json.dumps("thinkingmachines/Inkling-Small"), "__MAX_LENGTH__": "2048", "__BATCH_SIZE__": "2", "__DATA_FILE__": json.dumps("/tmp/unused.jsonl"), "__TEST_SIZE__": "0", "__LOG_PATH__": json.dumps(str(Path(tempfile.mkdtemp()) / "logs")), "__LEARNING_RATE__": "2e-4", "__NUM_EPOCHS__": "1", "__MAX_STEPS__": "1",}.items(): rendered = rendered.replace(key, value)mod_path = Path(tempfile.mkdtemp()) / "train_sft_smoke.py"mod_path.write_text(rendered, encoding="utf-8")import importlib.utilspec = importlib.util.spec_from_file_location("train_sft_smoke", mod_path)mod = importlib.util.module_from_spec(spec)sys.modules["train_sft_smoke"] = modspec.loader.exec_module(mod)
fixture = Path(tempfile.mkdtemp()) / "sft-input.jsonl"tool_trace = [ {"role": "user", "content": "Write a Turing model"}, {"role": "assistant", "content": [{"type": "thinking", "thinking": "Check whether Julia is installed before drafting code."}], "tool_calls": [{"id": "call_julia", "type": "function", "function": {"name": "bash", "arguments": "{\\"command\\":\\"which julia\\"}"}}]}, {"role": "tool", "content": "/usr/bin/julia", "tool_call_id": "call_julia"}, {"role": "assistant", "content": "Julia is installed."},]from assistant_targets import expand_assistant_targetstargets = expand_assistant_targets(tool_trace)rows = [{"messages": target, "source_line": 1, "target_index": i} for i, target in enumerate(targets)]fixture.write_text("".join(json.dumps(row) + "\\n" for row in rows), encoding="utf-8")
from tinker_cookbook import model_infofrom tinker_cookbook.supervised.types import ChatDatasetBuilderCommonConfigfrom tinker_cookbook.renderers import TrainOnWhat
def smoke_model(model_name): renderer_name = model_info.get_recommended_renderer_name(model_name) common = ChatDatasetBuilderCommonConfig( model_name_for_tokenizer=model_name, renderer_name=renderer_name, max_length=2048, batch_size=2, train_on_what=TrainOnWhat.LAST_ASSISTANT_MESSAGE, ) builder = mod.MachineExpandedDatasetBuilder( common_config=common, file_path=str(fixture), test_size=0, shuffle_seed=0, ) train_ds, test_ds = builder() assert test_ds is None assert len(train_ds.original_conversations) == 2 batch = train_ds.get_batch(0) assert len(batch) == 2 weighted = [] for datum in batch: loss = datum.loss_fn_inputs["weights"] values = loss.tolist() if hasattr(loss, "tolist") else list(loss) flat = [] def walk(x): if isinstance(x, (list, tuple)): for item in x: walk(item) else: flat.append(float(x)) walk(values) weighted.append(sum(1 for v in flat if v > 0)) assert all(w > 0 for w in weighted), weighted return {"model": model_name, "renderer": renderer_name, "targets": 2, "weighted": weighted, "clientInit": False}
receipts = [smoke_model("thinkingmachines/Inkling-Small")]try: receipts.append(smoke_model("Qwen/Qwen3-8B"))except Exception as e: receipts.append({"model": "Qwen/Qwen3-8B", "skipped": True, "error": f"{type(e).__name__}: {e}"})print(json.dumps({"ok": True, "receipts": receipts, "tinkerClientInitialized": False}))`);const builderSmokeOut = JSON.parse(execFileSync(venvPython, [builderSmoke], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"],}).trim().split("\n").filter(Boolean).slice(-1)[0]);assert(builderSmokeOut.ok, "no-client builder smoke should succeed");assert(builderSmokeOut.tinkerClientInitialized === false, "builder smoke must not initialize a Tinker client");const inklingReceipt = builderSmokeOut.receipts?.find((row) => row.model === "thinkingmachines/Inkling-Small");assert(inklingReceipt?.targets === 2 && inklingReceipt.weighted?.every((n) => n > 0), "Inkling builder smoke builds two positively weighted targets");const qwenReceipt = builderSmokeOut.receipts?.find((row) => row.model === "Qwen/Qwen3-8B");if (qwenReceipt && !qwenReceipt.skipped) { assert(qwenReceipt.targets === 2 && qwenReceipt.weighted?.every((n) => n > 0), "Qwen builder smoke builds two positively weighted targets");}
console.log(`thinking renderer tests passed (${checks} checks, ${venvPython})`);