Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
139 kB · 2769 lines
TypeScript
at letta-code
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";import { Box, Text } from "@earendil-works/pi-tui";import { execFile as execFileCb } from "node:child_process";import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";import { mkdir, readFile, rm, writeFile } from "node:fs/promises";import os from "node:os";import path from "node:path";import { promisify } from "node:util";import { CHECKPOINT_STATE_PATH, INKLING_MODEL, QWEN_38_27B_256K_MODEL, QWEN_38_27B_MODEL, TINKER_ANTHROPIC_BASE_URL, TINKER_OAI_BASE_URL, isInklingModel, listTinkerChatModels,} from "../lib/tinker-catalog.ts";
const execFile = promisify(execFileCb);
const DEFAULT_MODEL = INKLING_MODEL;const STATE_PATH = CHECKPOINT_STATE_PATH;const MESSAGE_TYPE = "tinker-report";
type ReportLevel = "info" | "success" | "warning" | "error";
type CheckpointModel = { id: string; name: string; contextWindow: number; maxTokens: number; addedAt: number; baseModel?: string; reasoning?: boolean; vision?: boolean;};
type TinkerState = { checkpoints: CheckpointModel[];};
function checkpointRegistration(options: { id: string; name: string; baseModel?: string; contextWindow?: number; maxTokens?: number;}): CheckpointModel { const inkling = isInklingModel(options.baseModel); return { id: options.id, name: options.name, baseModel: options.baseModel, reasoning: inkling, vision: inkling, contextWindow: options.contextWindow ?? (inkling ? 65_536 : 32_768), maxTokens: options.maxTokens ?? (inkling ? 16_384 : 4_096), addedAt: Date.now(), };}
function shellSplit(input: string): string[] { const out: string[] = []; let current = ""; let quote: "'" | '"' | undefined; let escaping = false;
for (const ch of input.trim()) { if (escaping) { current += ch; escaping = false; continue; } if (ch === "\\" && quote !== "'") { escaping = true; continue; } if ((ch === "'" || ch === '"') && !quote) { quote = ch; continue; } if (ch === quote) { quote = undefined; continue; } if (/\s/.test(ch) && !quote) { if (current) out.push(current); current = ""; continue; } current += ch; } if (current) out.push(current); return out;}
function parseOptions(tokens: string[]): { positional: string[]; options: Record<string, string | boolean> } { const positional: string[] = []; const options: Record<string, string | boolean> = {}; for (let i = 0; i < tokens.length; i++) { const token = tokens[i]!; if (!token.startsWith("--")) { positional.push(token); continue; } const raw = token.slice(2); const eq = raw.indexOf("="); if (eq >= 0) { options[raw.slice(0, eq)] = raw.slice(eq + 1); continue; } const next = tokens[i + 1]; if (next && !next.startsWith("--")) { options[raw] = next; i++; } else { options[raw] = true; } } return { positional, options };}
async function loadState(): Promise<TinkerState> { try { const raw = await readFile(STATE_PATH, "utf8"); const parsed = JSON.parse(raw) as Partial<TinkerState>; return { checkpoints: Array.isArray(parsed.checkpoints) ? parsed.checkpoints : [] }; } catch { return { checkpoints: [] }; }}
async function saveState(state: TinkerState): Promise<void> { await mkdir(path.dirname(STATE_PATH), { recursive: true }); await writeFile(STATE_PATH, JSON.stringify(state, null, 2) + "\n");}
function readJsonl(filePath: string): Record<string, unknown>[] { if (!existsSync(filePath)) return []; return readFileSync(filePath, "utf8") .split(/\n/) .filter((line) => line.trim()) .map((line) => JSON.parse(line));}
function compactMetrics(row: Record<string, unknown>, limit = 28): string { const interesting = Object.entries(row) .filter(([k, v]) => typeof v === "number" && (k.includes("loss") || k.includes("score") || k.includes("reward") || k.includes("kl") || k.includes("nll") || k.startsWith("progress/") || k.startsWith("time/total")) ) .slice(0, limit); if (interesting.length === 0) return JSON.stringify(row, null, 2).slice(0, 2000); return interesting.map(([k, v]) => `- ${k}: ${typeof v === "number" ? Number(v.toFixed(6)) : v}`).join("\n");}
function latestMetrics(metricsPath: string): string | undefined { if (!existsSync(metricsPath)) return undefined; try { const rows = readJsonl(metricsPath); const last = rows.slice(-1)[0]; if (!last) return undefined; return compactMetrics(last); } catch (error) { return `Could not parse ${metricsPath}: ${error instanceof Error ? error.message : String(error)}`; }}
function monitorSummary(logDir: string): string[] { const metricsPath = path.join(logDir, "metrics.jsonl"); const lines: string[] = [`Tinker monitor: ${path.basename(logDir)}`]; try { const rows = readJsonl(metricsPath); const last = rows.slice(-1)[0]; if (last) { const step = last["progress/batch"] ?? last["batch"] ?? rows.length; lines.push(`step: ${step}`); const lossEntry = Object.entries(last).find(([k, v]) => typeof v === "number" && k.includes("loss")); const rewardEntry = Object.entries(last).find(([k, v]) => typeof v === "number" && k.includes("reward")); const scoreEntry = Object.entries(last).find(([k, v]) => typeof v === "number" && k.includes("score")); if (lossEntry) lines.push(`${lossEntry[0]}: ${lossEntry[1]}`); if (rewardEntry) lines.push(`${rewardEntry[0]}: ${rewardEntry[1]}`); if (scoreEntry) lines.push(`${scoreEntry[0]}: ${scoreEntry[1]}`); } else { lines.push("metrics: waiting"); } } catch (error) { lines.push(`metrics error: ${error instanceof Error ? error.message : String(error)}`); } try { const checkpoints = readCheckpoints(logDir); const last = checkpoints.filter((c) => c.sampler_path || c.state_path).slice(-1)[0]; if (last) lines.push(`checkpoint: ${last.name}${last.final ? " (final)" : ""}`); } catch {} return lines.slice(0, 8);}
type CheckpointRecord = { name?: string; batch?: number; epoch?: number; final?: boolean; state_path?: string; sampler_path?: string;};
function readCheckpoints(logDir: string): CheckpointRecord[] { return readJsonl(path.join(logDir, "checkpoints.jsonl")) as CheckpointRecord[];}
async function commandExists(cmd: string): Promise<boolean> { try { await execFile("which", [cmd], { timeout: 10_000 }); return true; } catch { return false; }}
function validateDataset(filePath: string): string { if (!existsSync(filePath)) return `❌ File not found: ${filePath}`; const lines = readFileSync(filePath, "utf8").split(/\n/).filter((line: string) => line.trim()); let ok = 0; let assistantMessages = 0; let userMessages = 0; const issues: string[] = []; const samples: string[] = [];
lines.forEach((line: string, idx: number) => { try { const obj = JSON.parse(line); if (!Array.isArray(obj.messages)) { issues.push(`line ${idx + 1}: missing messages[]`); return; } if (obj.messages.length < 2) issues.push(`line ${idx + 1}: fewer than 2 messages`); for (const [j, msg] of obj.messages.entries()) { if (!msg || typeof msg !== "object") { issues.push(`line ${idx + 1} message ${j}: not an object`); continue; } if (!["system", "user", "assistant", "tool"].includes(msg.role)) { issues.push(`line ${idx + 1} message ${j}: unexpected role ${JSON.stringify(msg.role)}`); } if (msg.role === "assistant") assistantMessages++; if (msg.role === "user") userMessages++; if (!(typeof msg.content === "string" || Array.isArray(msg.content))) { issues.push(`line ${idx + 1} message ${j}: content should be string or content-part array`); } } ok++; if (samples.length < 3) { const preview = obj.messages .map((m: any) => `${m.role}: ${typeof m.content === "string" ? m.content : JSON.stringify(m.content)}`) .join("\n") .slice(0, 1200); samples.push(`### Example ${idx + 1}\n\n\`\`\`text\n${preview}\n\`\`\``); } } catch (error) { issues.push(`line ${idx + 1}: invalid JSON (${error instanceof Error ? error.message : String(error)})`); } });
const warnings: string[] = []; if (assistantMessages === 0) warnings.push("No assistant messages found; SFT will have nothing obvious to train on."); if (userMessages === 0) warnings.push("No user messages found; expected chat JSONL may be malformed."); if (lines.length < 20) warnings.push("Very small dataset; use this only for smoke tests.");
return [ `# Dataset validation: ${filePath}`, `- JSONL rows: ${lines.length}`, `- Valid rows: ${ok}`, `- User messages: ${userMessages}`, `- Assistant messages: ${assistantMessages}`, issues.length ? `\n## Issues\n${issues.slice(0, 50).map((x) => `- ${x}`).join("\n")}${issues.length > 50 ? `\n- … ${issues.length - 50} more` : ""}` : "\n## Issues\nNone found by the lightweight validator.", warnings.length ? `\n## Warnings\n${warnings.map((x) => `- ${x}`).join("\n")}` : "", `\n## Samples\n${samples.join("\n\n")}`, "\nNext: run the Python-backed validator (`/tinker validate ... --model ...`) before spending real compute.", ].filter(Boolean).join("\n");}
async function validateDatasetWithPython(filePath: string, model: string, maxExamples: number, maxLength: number): Promise<string> { const code = String.raw`import json, sys, statisticsfile_path, model_name, max_examples_s, max_length_s = sys.argv[1:5]max_examples = int(max_examples_s)max_length = int(max_length_s)
try: from tinker_cookbook import model_info from tinker_cookbook.renderers import TrainOnWhat, get_renderer from tinker_cookbook.tokenizer_utils import get_tokenizerexcept Exception as e: print(json.dumps({"ok": False, "stage": "import", "error": f"{type(e).__name__}: {e}"})) raise SystemExit(0)
try: renderer_name = model_info.get_recommended_renderer_name(model_name) tokenizer = get_tokenizer(model_name) renderer = get_renderer(renderer_name, tokenizer)except Exception as e: print(json.dumps({"ok": False, "stage": "renderer", "error": f"{type(e).__name__}: {e}"})) raise SystemExit(0)
BUCKETS = [512, 2048, 8192, 16384, 32768, 65536]
def flatten_values(x): if hasattr(x, "detach"): x = x.detach().cpu() if hasattr(x, "tolist"): x = x.tolist() if isinstance(x, (list, tuple)): out = [] for item in x: out.extend(flatten_values(item)) return out try: return [float(x)] except Exception: return []
def content_text(content): if isinstance(content, str): return content if isinstance(content, list): parts = [] for part in content: if isinstance(part, dict): if isinstance(part.get("text"), str): parts.append(part["text"]) elif isinstance(part.get("content"), str): parts.append(part["content"]) elif isinstance(part, str): parts.append(part) return "\n".join(parts) return ""
def stats(xs): if not xs: return {"count": 0} xs_sorted = sorted(xs) return { "count": len(xs), "min": xs_sorted[0], "p25": xs_sorted[max(0, int((len(xs_sorted)-1)*0.25))], "median": statistics.median(xs_sorted), "p90": xs_sorted[max(0, int((len(xs_sorted)-1)*0.90))], "p95": xs_sorted[max(0, int((len(xs_sorted)-1)*0.95))], "max": xs_sorted[-1], "mean": sum(xs_sorted) / len(xs_sorted), }
def histogram(xs): counts = {f"<= {b}": 0 for b in BUCKETS} counts[f"> {BUCKETS[-1]}"] = 0 for x in xs: placed = False for b in BUCKETS: if x <= b: counts[f"<= {b}"] += 1 placed = True break if not placed: counts[f"> {BUCKETS[-1]}"] += 1 return counts
def contiguous_ranges(indices): if not indices: return [] ranges = [] start = prev = indices[0] for idx in indices[1:]: if idx == prev + 1: prev = idx else: ranges.append((start, prev + 1)) start = prev = idx ranges.append((start, prev + 1)) return ranges
rows = 0valid_rows = 0roles = {}bad = []no_user = []no_assistant = []empty_assistant = []assistant_char_lengths = []
checked = 0token_lengths = []trainable_tokens = []trainable_ratios = []zero_trainable = []over_max_length = []top_longest = []previews = []trainable_previews = []length_mismatch = []
try: with open(file_path) as f: for line_number, line in enumerate(f, start=1): line = line.strip() if not line: continue rows += 1 try: obj = json.loads(line) except Exception as e: bad.append({"line": line_number, "issue": f"invalid JSON: {type(e).__name__}: {e}"}) continue
messages = obj.get("messages") if not isinstance(messages, list): bad.append({"line": line_number, "issue": "missing messages[]"}) continue
valid_rows += 1 seen_user = False seen_assistant = False assistant_chars_this_row = 0
for j, msg in enumerate(messages): if not isinstance(msg, dict): bad.append({"line": line_number, "issue": f"message {j} is not an object"}) continue role = msg.get("role", "<missing>") roles[role] = roles.get(role, 0) + 1 if role not in {"system", "user", "assistant", "tool"}: bad.append({"line": line_number, "issue": f"message {j} has unexpected role {role!r}"}) if role == "user": seen_user = True if role == "assistant": seen_assistant = True txt = content_text(msg.get("content")) assistant_chars_this_row += len(txt.strip()) if not txt.strip(): empty_assistant.append(line_number) content = msg.get("content") if not isinstance(content, (str, list)): bad.append({"line": line_number, "issue": f"message {j} content should be string or content-part array"})
if not seen_user: no_user.append(line_number) if not seen_assistant: no_assistant.append(line_number) assistant_char_lengths.append(assistant_chars_this_row)
if checked >= max_examples: continue
try: model_input, weights = renderer.build_supervised_example( messages, train_on_what=TrainOnWhat.ALL_ASSISTANT_MESSAGES ) toks = model_input.to_ints() if hasattr(model_input, "to_ints") else [] w = flatten_values(weights) positive = [i for i, weight in enumerate(w) if weight > 0] checked += 1 token_lengths.append(len(toks)) trainable_tokens.append(len(positive)) trainable_ratios.append((len(positive) / len(toks)) if toks else 0.0)
if len(w) not in {len(toks), max(0, len(toks) - 1)}: length_mismatch.append({"line": line_number, "tokens": len(toks), "weights": len(w)}) if len(positive) == 0: zero_trainable.append(line_number) if len(toks) > max_length: over_max_length.append({"line": line_number, "tokens": len(toks)})
top_longest.append({"line": line_number, "tokens": len(toks), "trainable": len(positive)}) top_longest = sorted(top_longest, key=lambda x: x["tokens"], reverse=True)[:8]
if len(previews) < 3: previews.append({"line": line_number, "text": tokenizer.decode(toks[: min(len(toks), 500)])}) if len(trainable_previews) < 5 and positive: snippets = [] for start, end in contiguous_ranges(positive)[:4]: snippet_tokens = toks[start:min(end, start + 160)] snippets.append({"range": [start, end], "text": tokenizer.decode(snippet_tokens)}) trainable_previews.append({"line": line_number, "snippets": snippets}) except Exception as e: bad.append({"line": line_number, "issue": f"renderer/tokenization failed: {type(e).__name__}: {e}"})except Exception as e: print(json.dumps({"ok": False, "stage": "file", "error": f"{type(e).__name__}: {e}"})) raise SystemExit(0)
serious = []if bad: serious.append(f"{len(bad)} data/renderer issue(s)")if no_assistant: serious.append(f"{len(no_assistant)} row(s) with no assistant message")if empty_assistant: serious.append(f"{len(empty_assistant)} empty assistant message(s)")if zero_trainable: serious.append(f"{len(zero_trainable)} checked row(s) with zero trainable tokens")if length_mismatch: serious.append(f"{len(length_mismatch)} checked row(s) with token/weight length mismatch")
warnings = []if rows < 20: warnings.append("very small dataset; good for smoke tests only")if over_max_length: warnings.append(f"{len(over_max_length)} checked row(s) exceed max_length={max_length}")if checked < min(max_examples, valid_rows): warnings.append("not all rows were tokenized; increase --examples for a fuller audit")if token_lengths and max(token_lengths) > max_length * 0.9: warnings.append("some examples are close to the max length; inspect truncation behavior")if assistant_char_lengths and statistics.median(assistant_char_lengths) < 20: warnings.append("median assistant completion is very short; confirm this is intentional")
if serious: readiness = "FIX DATA FIRST"elif warnings: readiness = "SMOKE ONLY"else: readiness = "READY"
mean_tokens = (sum(token_lengths) / len(token_lengths)) if token_lengths else 0mean_trainable = (sum(trainable_tokens) / len(trainable_tokens)) if trainable_tokens else 0estimate = { "estimated_total_input_tokens_per_epoch": int(mean_tokens * valid_rows) if checked else 0, "estimated_trainable_tokens_per_epoch": int(mean_trainable * valid_rows) if checked else 0, "basis": f"estimated from {checked} checked row(s)",}
print(json.dumps({ "ok": not serious, "stage": "done", "readiness": readiness, "serious": serious, "warnings": warnings, "model": model_name, "renderer": renderer_name, "rows": rows, "valid_rows": valid_rows, "checked_examples": checked, "roles": roles, "token_lengths": stats(token_lengths), "trainable_tokens": stats(trainable_tokens), "trainable_ratios": stats(trainable_ratios), "assistant_chars": stats(assistant_char_lengths), "token_length_histogram": histogram(token_lengths), "trainable_token_histogram": histogram(trainable_tokens), "estimate": estimate, "bad": bad[:80], "bad_count": len(bad), "no_user": no_user[:20], "no_user_count": len(no_user), "no_assistant": no_assistant[:20], "no_assistant_count": len(no_assistant), "empty_assistant": empty_assistant[:20], "empty_assistant_count": len(empty_assistant), "zero_trainable": zero_trainable[:20], "zero_trainable_count": len(zero_trainable), "over_max_length": over_max_length[:20], "over_max_length_count": len(over_max_length), "length_mismatch": length_mismatch[:20], "length_mismatch_count": len(length_mismatch), "top_longest": top_longest, "previews": previews, "trainable_previews": trainable_previews,}, ensure_ascii=False))`; const { stdout } = await execFile("python3", ["-c", code, filePath, model, String(maxExamples), String(maxLength)], { timeout: 240_000, maxBuffer: 12 * 1024 * 1024, }); const result = JSON.parse(stdout.trim().split(/\n/).slice(-1)[0] ?? "{}"); if (!result.ok && result.stage !== "done") { const installCommand = isInklingModel(model) ? "uv pip install -U 'tinker-cookbook[inkling]'" : "uv pip install -U tinker-cookbook"; return [ `# Python-backed validation could not run`, `- Stage: ${result.stage ?? "unknown"}`, `- Error: ${result.error ?? "unknown"}`, "", "Install/upgrade dependencies with:", "```bash", installCommand, "```", "", "Falling back to lightweight JSONL checks is still useful, but renderer/token-mask validation requires Python dependencies.", ].join("\n"); }
const readiness = String(result.readiness ?? "UNKNOWN"); const icon = readiness === "READY" ? "✅" : readiness === "SMOKE ONLY" ? "⚠️" : "❌"; const fmtStats = (s: any) => s?.count ? `count=${s.count}, min=${s.min}, p50=${s.median}, p90=${s.p90}, p95=${s.p95}, max=${s.max}, mean=${Number(s.mean).toFixed(1)}` : "n/a"; const fmtHist = (h: Record<string, number> | undefined) => h ? Object.entries(h).filter(([, v]) => v > 0).map(([k, v]) => `- ${k}: ${v}`).join("\n") || "- empty" : "- n/a"; const issueLines = [ ...(result.bad ?? []).map((x: any) => `- line ${x.line}: ${x.issue}`), result.no_assistant_count ? `- rows with no assistant message: ${result.no_assistant.join(", ")}${result.no_assistant_count > result.no_assistant.length ? " …" : ""}` : "", result.empty_assistant_count ? `- empty assistant rows: ${result.empty_assistant.join(", ")}${result.empty_assistant_count > result.empty_assistant.length ? " …" : ""}` : "", result.zero_trainable_count ? `- zero-trainable checked rows: ${result.zero_trainable.join(", ")}${result.zero_trainable_count > result.zero_trainable.length ? " …" : ""}` : "", result.length_mismatch_count ? `- token/weight length mismatches: ${(result.length_mismatch ?? []).map((x: any) => `line ${x.line} (${x.tokens} tokens/${x.weights} weights)`).join(", ")}` : "", ].filter(Boolean); const topLongest = (result.top_longest ?? []).map((x: any) => `- line ${x.line}: ${x.tokens} tokens, ${x.trainable} trainable`).join("\n") || "- n/a"; const trainablePreviews = (result.trainable_previews ?? []).map((p: any) => { const snippets = (p.snippets ?? []).map((s: any) => ` - tokens ${s.range[0]}–${s.range[1]}:\n\n ${String(s.text).replace(/\n/g, "\n ").slice(0, 1000)}`).join("\n"); return `### Line ${p.line}\n${snippets}`; }).join("\n\n") || "No trainable token previews available."; const decodedPreviews = (result.previews ?? []).map((p: any) => `### Line ${p.line}\n\n\`\`\`text\n${String(p.text).slice(0, 1600)}\n\`\`\``).join("\n\n") || "No decoded previews available.";
const recommendation = readiness === "READY" ? "Run `/tinker smoke train_sft.py --yes`, inspect metrics, then scale only after defining an eval." : readiness === "SMOKE ONLY" ? "Run only a tiny smoke test for now. Inspect warnings, decoded previews, and trainable snippets before larger runs." : "Fix the data issues above before creating a training run.";
return [ `# Tinker data readiness report`, `## ${icon} ${readiness}`, `- File: \`${filePath}\``, `- Model: \`${result.model}\``, `- Recommended renderer: \`${result.renderer}\``, `- Rows: ${result.rows} (${result.valid_rows} valid JSONL conversation rows)`, `- Checked with tokenizer/renderer: ${result.checked_examples}`, `- Roles: ${Object.entries(result.roles ?? {}).map(([k, v]) => `${k}=${v}`).join(", ") || "none"}`, "", `## Recommendation\n${recommendation}`, (result.serious ?? []).length ? `\n## Must fix\n${(result.serious ?? []).map((x: string) => `- ${x}`).join("\n")}` : "", (result.warnings ?? []).length ? `\n## Warnings\n${(result.warnings ?? []).map((x: string) => `- ${x}`).join("\n")}` : "", `\n## Token stats\n- Input tokens: ${fmtStats(result.token_lengths)}\n- Trainable assistant tokens: ${fmtStats(result.trainable_tokens)}\n- Trainable ratio: ${fmtStats(result.trainable_ratios)}\n- Assistant text chars: ${fmtStats(result.assistant_chars)}`, `\n## Token length histogram\n${fmtHist(result.token_length_histogram)}`, `\n## Trainable-token histogram\n${fmtHist(result.trainable_token_histogram)}`, `\n## Token-volume estimate\n- Total input tokens / epoch: ~${result.estimate?.estimated_total_input_tokens_per_epoch ?? 0}\n- Trainable tokens / epoch: ~${result.estimate?.estimated_trainable_tokens_per_epoch ?? 0}\n- Basis: ${result.estimate?.basis ?? "n/a"}`, issueLines.length ? `\n## Detailed issues\n${issueLines.slice(0, 100).join("\n")}` : "\n## Detailed issues\nNo serious data/rendering issues found in checked examples.", `\n## Longest checked examples\n${topLongest}`, (result.over_max_length_count ?? 0) > 0 ? `\n## Over max length (${maxLength})\n${(result.over_max_length ?? []).map((x: any) => `- line ${x.line}: ${x.tokens} tokens`).join("\n")}` : "", `\n## Trainable token snippets\n${trainablePreviews}`, `\n## Decoded input previews\n${decodedPreviews}`, ].filter(Boolean).join("\n");}
function makeSftScript(options: { dataFile: string; model: string; logPath: string; maxSteps: string; batchSize: string; learningRate: string; testSize: string; maxLength: string;}) { return `import asyncio\nimport sys\n\nimport chz\n\nfrom tinker_cookbook import cli_utils, model_info\nfrom tinker_cookbook.renderers import TrainOnWhat\nfrom tinker_cookbook.supervised import train\nfrom tinker_cookbook.supervised.data import FromConversationFileBuilder\nfrom tinker_cookbook.supervised.types import ChatDatasetBuilderCommonConfig\n\n\ndef build_config_blueprint() -> chz.Blueprint[train.Config]:\n model_name = ${JSON.stringify(options.model)}\n renderer_name = model_info.get_recommended_renderer_name(model_name)\n\n common_config = ChatDatasetBuilderCommonConfig(\n model_name_for_tokenizer=model_name,\n renderer_name=renderer_name,\n max_length=${options.maxLength},\n batch_size=${options.batchSize},\n train_on_what=TrainOnWhat.ALL_ASSISTANT_MESSAGES,\n )\n\n dataset = FromConversationFileBuilder(\n common_config=common_config,\n file_path=${JSON.stringify(options.dataFile)},\n test_size=${options.testSize},\n shuffle_seed=0,\n )\n\n return chz.Blueprint(train.Config).apply({\n "log_path": ${JSON.stringify(options.logPath)},\n "model_name": model_name,\n "recipe_name": "pi_tinker_sft",\n "renderer_name": renderer_name,\n "dataset_builder": dataset,\n "learning_rate": ${options.learningRate},\n "lr_schedule": "linear",\n "num_epochs": 1,\n "lora_rank": 32,\n "save_every": 20,\n "eval_every": 10,\n "max_steps": ${options.maxSteps},\n })\n\n\ndef main(config: train.Config):\n print("Resolved Tinker SFT config:")\n print(config)\n cli_utils.check_log_dir(config.log_path, behavior_if_exists="ask")\n asyncio.run(train.main(config))\n\n\nif __name__ == "__main__":\n blueprint = build_config_blueprint()\n blueprint.make_from_argv(sys.argv[1:])\n main(blueprint.make())\n`;}
function makeEvalScript() { return `"""\nOptional quick checkpoint smoke test.\n\nUsage:\n python eval_checkpoint.py 'tinker://.../sampler_weights/...'\n"""\nimport asyncio\nimport sys\n\nimport tinker\n\n\nasync def main(model_path: str):\n svc = tinker.ServiceClient()\n sc = await svc.create_sampling_client_async(model_path=model_path)\n tok = sc.get_tokenizer()\n prompt = tinker.ModelInput.from_ints(tok.encode("The best way to test a fine-tuned model is"))\n result = await sc.sample_async(\n prompt=prompt,\n num_samples=1,\n sampling_params=tinker.SamplingParams(max_tokens=80, temperature=0.7),\n )\n seq = result.sequences[0] if hasattr(result, "sequences") else result.samples[0]\n print(tok.decode(seq.tokens))\n\n\nif __name__ == "__main__":\n if len(sys.argv) != 2:\n raise SystemExit("Usage: python eval_checkpoint.py 'tinker://.../sampler_weights/...'" )\n asyncio.run(main(sys.argv[1]))\n`;}
function makeProjectReadme(options: { model: string; dataFile: string; logPath: string; successMetric: string }) { return `# Tinker fine-tuning project\n\nThis project was scaffolded by Machine. The important files are normal editable Python, not hidden framework state.\n\n## Goal\n\nFine-tune \`${options.model}\` on:\n\n\`${options.dataFile}\`\n\n## Success metric\n\n${options.successMetric || "Define this before scaling beyond a smoke test."}\n\n## Smoke test\n\n\`\`\`bash\n${isInklingModel(options.model) ? "uv pip install -U 'tinker-cookbook[inkling]'" : "uv pip install -U tinker-cookbook"}\npython train_sft.py max_steps=2\n\`\`\`\n\n${isInklingModel(options.model) ? "Inkling uses its recommended TMLv0 renderer automatically. This generic SFT scaffold trains at the renderer default effort, 0.9 (high). Run `/tinker inkling sweep` before training and use the same effort for baseline and checkpoint evals.\n\n" : ""}## Monitor\n\nInside Pi:\n\n\`\`\`text\n/tinker monitor ${options.logPath}\n/tinker status ${options.logPath}\n/tinker checkpoints ${options.logPath}\n\`\`\`\n\n## Scale up\n\nOnly scale after checking:\n\n- JSONL validation passed\n- renderer/token validation passed\n- smoke test produced metrics\n- decoded examples look correct\n- success metric/eval is defined\n\n\`\`\`bash\npython train_sft.py max_steps=100\n\`\`\`\n\n## Chat with a checkpoint in Pi\n\nAfter a sampler checkpoint appears in \`checkpoints.jsonl\`:\n\n\`\`\`text\n/tinker checkpoints ${options.logPath}\n/model\n\`\`\`\n`;}
function makeExampleEvalJsonl() { return [ { messages: [{ role: "user", content: "Rewrite this to be concise: We are extremely sorry for the inconvenience and are investigating." }], expected: "Sorry for the inconvenience — we’re investigating.", match: "contains", notes: "Replace with examples from your real target distribution.", }, { messages: [{ role: "user", content: "Classify sentiment: The setup was fast and the docs were clear." }], expected: "positive", match: "contains", }, ].map((row) => JSON.stringify(row)).join("\n") + "\n";}
type PreparedDataset = { inputPath: string; outFile: string; format: string; rows: number; warnings: string[]; samples: string[];};
type ExampleTemplate = { slug: string; title: string; goal: string; trainRows: Array<Record<string, unknown>>; evalRows: Array<Record<string, unknown>>;};
const EXAMPLE_TEMPLATES: ExampleTemplate[] = [ { slug: "customer-support", title: "Customer support tone + answer quality", goal: "More accurate, concise, empathetic customer support answers", trainRows: [ { messages: [{ role: "user", content: "My package says delivered but it isn't here. What do I do?" }, { role: "assistant", content: "I’m sorry — that’s frustrating. Please check nearby doors or your building mail area first. If it still hasn’t arrived, send us your order number and we’ll open a carrier trace or arrange a replacement." }] }, { messages: [{ role: "user", content: "Can I cancel my subscription?" }, { role: "assistant", content: "Yes. Go to Settings → Billing → Cancel subscription. Your access stays active until the end of the current billing period." }] }, { messages: [{ role: "user", content: "The app keeps crashing after login." }, { role: "assistant", content: "Sorry about that. Please update to the latest version, restart the app, and try again. If it still crashes, send us your device type and app version so we can investigate." }] }, ], evalRows: [ { messages: [{ role: "user", content: "I was charged twice this month." }], expected: "sorry", match: "contains" }, { messages: [{ role: "user", content: "Where do I cancel?" }], expected: "Settings", match: "contains" }, ], }, { slug: "structured-extraction", title: "Structured JSON extraction", goal: "Higher valid-JSON extraction accuracy on domain text", trainRows: [ { messages: [{ role: "user", content: "Extract JSON: Jane Doe works at Acme as VP Sales." }, { role: "assistant", content: '{"name":"Jane Doe","company":"Acme","title":"VP Sales"}' }] }, { messages: [{ role: "user", content: "Extract JSON: Omar Khan, CTO, Northstar Labs." }, { role: "assistant", content: '{"name":"Omar Khan","company":"Northstar Labs","title":"CTO"}' }] }, { messages: [{ role: "user", content: "Extract JSON: No company listed for Priya Shah." }, { role: "assistant", content: '{"name":"Priya Shah","company":null,"title":null}' }] }, ], evalRows: [ { messages: [{ role: "user", content: "Extract JSON: Alex Lee is Head of Ops at Meridian." }], expected: '"company":"Meridian"', match: "contains" }, { messages: [{ role: "user", content: "Extract JSON: Taylor Smith, no role, Orbit." }], expected: '"company":"Orbit"', match: "contains" }, ], }, { slug: "concise-writing", title: "Concise rewriting", goal: "Shorter, clearer rewrites that preserve meaning", trainRows: [ { messages: [{ role: "user", content: "Rewrite concisely: We are reaching out today in order to inform you that your requested refund has now been successfully processed." }, { role: "assistant", content: "Your refund has been processed." }] }, { messages: [{ role: "user", content: "Rewrite concisely: Due to the fact that the meeting was moved, we will need to make an adjustment to the current schedule." }, { role: "assistant", content: "Because the meeting moved, we need to adjust the schedule." }] }, { messages: [{ role: "user", content: "Rewrite concisely: At this point in time, we do not have sufficient information to make a final decision." }, { role: "assistant", content: "We don’t have enough information to decide yet." }] }, ], evalRows: [ { messages: [{ role: "user", content: "Rewrite concisely: We wanted to let you know that your account has been approved." }], expected: "approved", match: "contains" }, { messages: [{ role: "user", content: "Rewrite concisely: In the event that you need assistance, please contact support." }], expected: "support", match: "contains" }, ], },];
function jsonl(rows: Array<Record<string, unknown>>): string { return rows.map((row) => JSON.stringify(row)).join("\n") + "\n";}
function parseCsv(text: string): string[][] { const rows: string[][] = []; let row: string[] = []; let field = ""; let quoted = false; for (let i = 0; i < text.length; i++) { const ch = text[i]!; const next = text[i + 1]; if (quoted && ch === '"' && next === '"') { field += '"'; i++; continue; } if (ch === '"') { quoted = !quoted; continue; } if (ch === "," && !quoted) { row.push(field); field = ""; continue; } if ((ch === "\n" || ch === "\r") && !quoted) { if (ch === "\r" && next === "\n") i++; row.push(field); if (row.some((x) => x.trim())) rows.push(row); row = []; field = ""; continue; } field += ch; } row.push(field); if (row.some((x) => x.trim())) rows.push(row); return rows;}
function normalizeHeader(header: string): string { return header.trim().toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");}
function pickColumn(headers: string[], candidates: string[]): number { const normalized = headers.map(normalizeHeader); for (const candidate of candidates) { const idx = normalized.indexOf(candidate); if (idx >= 0) return idx; } for (const candidate of candidates) { const idx = normalized.findIndex((h) => h.includes(candidate)); if (idx >= 0) return idx; } return -1;}
function rowToTrainingExample(row: Record<string, unknown>): Record<string, unknown> | undefined { if (Array.isArray(row.messages)) return { messages: row.messages }; const system = typeof row.system === "string" ? row.system : typeof row.instruction === "string" ? row.instruction : undefined; const input = String(row.user ?? row.question ?? row.prompt ?? row.input ?? row.query ?? "").trim(); const output = String(row.assistant ?? row.answer ?? row.completion ?? row.output ?? row.response ?? "").trim(); if (!input || !output) return undefined; const messages: Array<Record<string, string>> = []; if (system) messages.push({ role: "system", content: system }); messages.push({ role: "user", content: input }); messages.push({ role: "assistant", content: output }); return { messages };}
function collectTextFiles(dir: string): string[] { const out: string[] = []; for (const name of readdirSync(dir)) { const target = path.join(dir, name); const stat = statSync(target); if (stat.isDirectory()) out.push(...collectTextFiles(target)); else if (/\.(md|mdx|txt)$/i.test(name)) out.push(target); } return out;}
async function prepareDataset(inputPath: string, outFile: string): Promise<PreparedDataset> { const warnings: string[] = []; const rows: Array<Record<string, unknown>> = []; const stat = statSync(inputPath); let format = "unknown";
if (stat.isDirectory()) { format = "docs-directory"; for (const file of collectTextFiles(inputPath).slice(0, 500)) { const text = readFileSync(file, "utf8").replace(/\s+/g, " ").trim(); if (!text) continue; const chunks = text.match(/.{1,3500}(?:\s|$)/g) ?? [text.slice(0, 3500)]; for (const [idx, chunk] of chunks.slice(0, 20).entries()) { rows.push({ messages: [ { role: "user", content: `Summarize the useful facts from ${path.basename(file)}${chunks.length > 1 ? ` part ${idx + 1}` : ""}:\n\n${chunk.trim()}` }, { role: "assistant", content: chunk.trim().slice(0, 900) }, ], source: path.relative(path.dirname(inputPath), file), }); } } warnings.push("Docs were converted into starter summarization examples. For best results, replace assistant outputs with the exact behavior you want the model to learn."); } else if (/\.csv$/i.test(inputPath)) { format = "csv"; const parsed = parseCsv(readFileSync(inputPath, "utf8")); if (parsed.length < 2) throw new Error("CSV needs a header row and at least one data row."); const headers = parsed[0]!.map((h) => h.trim()); const userIdx = pickColumn(headers, ["user", "question", "prompt", "input", "query"]); const assistantIdx = pickColumn(headers, ["assistant", "answer", "completion", "output", "response"]); const systemIdx = pickColumn(headers, ["system", "instruction"]); const messagesIdx = pickColumn(headers, ["messages"]); for (const values of parsed.slice(1)) { if (messagesIdx >= 0) { try { const messages = JSON.parse(values[messagesIdx] ?? ""); if (Array.isArray(messages)) rows.push({ messages }); else warnings.push("Skipped a CSV row whose messages column was not a JSON array."); } catch { warnings.push("Skipped a CSV row whose messages column was not valid JSON."); } continue; } if (userIdx < 0 || assistantIdx < 0) throw new Error(`CSV needs columns like question/prompt/input and answer/response/output. Found: ${headers.join(", ")}`); const messages: Array<Record<string, string>> = []; const system = systemIdx >= 0 ? String(values[systemIdx] ?? "").trim() : ""; const user = String(values[userIdx] ?? "").trim(); const assistant = String(values[assistantIdx] ?? "").trim(); if (!user || !assistant) continue; if (system) messages.push({ role: "system", content: system }); messages.push({ role: "user", content: user }); messages.push({ role: "assistant", content: assistant }); rows.push({ messages }); } } else if (/\.jsonl$/i.test(inputPath)) { format = "jsonl"; for (const line of readFileSync(inputPath, "utf8").split(/\n/).filter((x) => x.trim())) { const converted = rowToTrainingExample(JSON.parse(line)); if (converted) rows.push(converted); else warnings.push("Skipped a JSONL row that did not contain messages[] or recognizable prompt/response fields."); } } else if (/\.json$/i.test(inputPath)) { format = "json"; const parsed = JSON.parse(readFileSync(inputPath, "utf8")); const items = Array.isArray(parsed) ? parsed : Array.isArray(parsed.examples) ? parsed.examples : Array.isArray(parsed.data) ? parsed.data : []; if (!Array.isArray(items) || items.length === 0) throw new Error("JSON should be an array, or contain examples[]/data[]."); for (const item of items) { const converted = item && typeof item === "object" ? rowToTrainingExample(item as Record<string, unknown>) : undefined; if (converted) rows.push(converted); else warnings.push("Skipped a JSON item that did not contain messages[] or recognizable prompt/response fields."); } } else if (/\.(md|mdx|txt)$/i.test(inputPath)) { format = "document"; const text = readFileSync(inputPath, "utf8").replace(/\s+/g, " ").trim(); for (const chunk of (text.match(/.{1,3500}(?:\s|$)/g) ?? [text]).slice(0, 100)) { rows.push({ messages: [{ role: "user", content: `Summarize the useful facts from this document:\n\n${chunk.trim()}` }, { role: "assistant", content: chunk.trim().slice(0, 900) }] }); } warnings.push("A raw document creates weak starter examples. Convert real question/answer or input/output pairs when possible."); } else { throw new Error("Unsupported input. Use CSV, JSON, JSONL, TXT/MD, or a directory of TXT/MD files."); }
if (rows.length === 0) throw new Error("No usable training rows found."); await mkdir(path.dirname(outFile), { recursive: true }); await writeFile(outFile, jsonl(rows)); return { inputPath, outFile, format, rows: rows.length, warnings, samples: rows.slice(0, 3).map((row) => JSON.stringify(row).slice(0, 1000)), };}
function recommendPlan(goal: string, dataRows?: number): string { const g = goal.toLowerCase(); const structured = /json|extract|classif|label|schema|sql|regex|parse/.test(g); const reasoning = /math|reason|proof|solve|logic|agent|tool/.test(g); const writing = /write|tone|support|summar|rewrite|style|email|copy/.test(g); const model = DEFAULT_MODEL; const examples = dataRows ? `${dataRows} detected rows` : "unknown row count"; const rowsAdvice = !dataRows ? "Start with 20–100 high-quality examples for a smoke test, then scale." : dataRows < 20 ? "Good for a smoke test only; add more examples before expecting durable quality gains." : dataRows < 200 ? "Enough for an early SFT run if examples are high quality." : "Enough to try a serious SFT run after baseline eval passes."; const task = structured ? "SFT with strict eval for valid outputs" : reasoning ? "SFT first; consider RL only after strong evals exist" : writing ? "SFT with before/after human or rubric eval" : "SFT golden path"; return [ `# Tinker recommendation`, `- Goal: ${goal || "not provided"}`, `- Data: ${examples}`, `- Recommended first model: \`${model}\``, `- First training method: ${task}`, `- Starter settings: \`max_steps=20\`, \`batch_size=8\`, \`lr=2e-4\`, 2-step smoke test before scale-up`, `- Data advice: ${rowsAdvice}`, "", "## Fastest path", "```text", "/tinker new data/train.jsonl --goal \"" + (goal || "what should improve") + "\" --model " + model, "/tinker doctor", "/tinker validate data/train.jsonl --model " + model, "/tinker eval baseline --model " + model + " --yes", "/tinker smoke train_sft.py --yes", "```", ].join("\n");}
function templateBySlug(slug: string): ExampleTemplate | undefined { return EXAMPLE_TEMPLATES.find((t) => t.slug === slug);}
async function writeExampleTemplate(cwd: string, slug: string, force = false): Promise<{ template: ExampleTemplate; files: string[] }> { const template = templateBySlug(slug); if (!template) throw new Error(`Unknown example ${slug}. Available: ${EXAMPLE_TEMPLATES.map((t) => t.slug).join(", ")}`); const files = [ { rel: path.join("examples", slug, "train.jsonl"), content: jsonl(template.trainRows) }, { rel: path.join("examples", slug, "eval.jsonl"), content: jsonl(template.evalRows) }, { rel: path.join("examples", slug, "README.md"), content: `# ${template.title}\n\nGoal: ${template.goal}\n\nTry:\n\n\`\`\`text\n/tinker new examples/${slug}/train.jsonl --goal "${template.goal}" --force\n/tinker validate examples/${slug}/train.jsonl --quick\n\`\`\`\n` }, ]; const written: string[] = []; for (const file of files) { const target = path.join(cwd, file.rel); if (existsSync(target) && !force) continue; await mkdir(path.dirname(target), { recursive: true }); await writeFile(target, file.content); written.push(file.rel); } return { template, files: written };}
async function buildDoctorReport(cwd: string, dataFileArg?: string): Promise<string> { const wizard = await readWizardState(cwd); const dataFile = dataFileArg ? path.resolve(cwd, dataFileArg) : wizard?.dataFile; const model = wizard?.model ?? DEFAULT_MODEL; const checks: string[] = []; checks.push(process.env.TINKER_API_KEY ? "✅ TINKER_API_KEY is set" : "❌ TINKER_API_KEY is missing"); checks.push((await commandExists("python3")) ? "✅ python3 found" : "❌ python3 not found"); checks.push((await commandExists("uv")) ? "✅ uv found" : "⚠️ uv not found; pip fallback is okay"); checks.push((await commandExists("tinker")) ? "✅ tinker CLI found" : "⚠️ tinker CLI not found"); if (isInklingModel(model)) { try { const { stdout } = await execFile("python3", ["-c", "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"], { timeout: 20_000 }); const [major, minor] = stdout.trim().split(".").map(Number); checks.push(major > 3 || (major === 3 && minor >= 11) ? `✅ Python ${stdout.trim()} supports Inkling` : `❌ Inkling requires Python 3.11+ (found ${stdout.trim()})`); } catch {} } const modules = isInklingModel(model) ? ["tinker", "tinker_cookbook", "chz", "tml_renderers"] : ["tinker", "tinker_cookbook", "chz"]; for (const mod of modules) { try { await execFile("python3", ["-c", `import ${mod}; print('ok')`], { cwd, timeout: 20_000 }); checks.push(`✅ Python import ${mod}`); } catch { checks.push(`⚠️ Python cannot import ${mod}`); } } if (isInklingModel(model)) { try { const { stdout } = await execFile("python3", ["-c", "import importlib.metadata as m; print(m.version('tinker'))"], { timeout: 20_000 }); const [major, minor] = stdout.trim().split(".").map(Number); checks.push(major > 0 || minor >= 23 ? `✅ Tinker SDK ${stdout.trim()} supports Inkling` : `❌ Inkling requires Tinker SDK 0.23+ (found ${stdout.trim()})`); } catch {} } checks.push(existsSync(path.join(cwd, "train_sft.py")) ? "✅ train_sft.py exists" : "⬜ train_sft.py missing; run /tinker new or /tinker init"); checks.push(existsSync(path.join(cwd, "eval.py")) ? "✅ eval.py exists" : "⬜ eval.py missing; run /tinker eval init"); if (dataFile) checks.push(existsSync(dataFile) ? `✅ data file exists: ${rel(cwd, dataFile)}` : `❌ data file missing: ${dataFile}`); else checks.push("⬜ no data file selected yet"); if (existsSync(path.join(cwd, "train_sft.py"))) { try { await execFile("python3", ["-m", "py_compile", "train_sft.py"], { cwd, timeout: 20_000 }); checks.push("✅ train_sft.py compiles"); } catch (error: any) { checks.push(`❌ train_sft.py does not compile: ${error?.message ?? String(error)}`); } } if (existsSync(path.join(cwd, "eval.py"))) { try { await execFile("python3", ["-m", "py_compile", "eval.py"], { cwd, timeout: 20_000 }); checks.push("✅ eval.py compiles"); } catch (error: any) { checks.push(`❌ eval.py does not compile: ${error?.message ?? String(error)}`); } } if (isInklingModel(model)) { checks.push("ℹ️ Inkling uses the TMLv0 renderer and default SFT effort 0.9 (high)"); } const next = wizard ? wizardSteps(cwd, wizard).find((step) => !step.done)?.nextCommand : "/tinker new"; return [ "# Tinker doctor", checks.map((c) => `- ${c}`).join("\n"), "", "## Next recommended command", "```text", next ?? "/tinker next", "```", "", isInklingModel(model) ? "Inkling dependency fix: `uv pip install -U 'tinker-cookbook[inkling]'`." : "", "If anything is confusing, run `/skill:tinker-debug` with this report.", ].join("\n");}
type ImproveBudget = "demo" | "smoke" | "small" | "real";
function parseImproveBudget(raw: unknown): ImproveBudget { const budget = String(raw ?? "demo").toLowerCase(); if (["demo", "smoke", "small", "real"].includes(budget)) return budget as ImproveBudget; throw new Error("--budget must be one of: demo, smoke, small, real");}
function budgetMaxSteps(budget: ImproveBudget, override?: unknown): number { if (override !== undefined) return Number(override); if (budget === "smoke") return 2; if (budget === "small") return 20; if (budget === "real") return 100; return 0;}
const INKLING_EFFORT_PRESETS: Record<string, number> = { none: 0, minimal: 0.1, low: 0.2, medium: 0.7, high: 0.9, xhigh: 0.99,};
function parseInklingEfforts(value: unknown): number[] { const raw = String(value ?? "low,medium,high,xhigh"); const efforts = raw.split(",").map((part) => { const text = part.trim().toLowerCase(); return text in INKLING_EFFORT_PRESETS ? INKLING_EFFORT_PRESETS[text]! : Number(text); }); if (efforts.length === 0 || efforts.some((effort) => !Number.isFinite(effort) || effort < 0 || effort >= 1)) { throw new Error("--efforts must be comma-separated presets or floats in [0, 1), e.g. low,medium,high,xhigh or 0.2,0.7,0.9,0.99"); } return efforts;}
function managedRunPlan(budget: ImproveBudget): string { const lines = [ "# Managed fine-tuning operator", `- Budget: \`${budget}\``, "- Principle: never scale until data, eval, and smoke test are sane.", "", "## Loop", "1. Prepare/locate data", "2. Scaffold editable Tinker Cookbook files", "3. Run project doctor", "4. Validate data/rendering", "5. Ensure eval exists", "6. Run baseline eval before training", "7. Run 2-step smoke training", "8. Scale training only with confirmation", "9. Evaluate checkpoint on the same eval", "10. Compare wins/regressions and register the checkpoint for chat", ]; if (budget === "demo") lines.push("", "Demo budget stops before real Tinker API usage."); if (budget === "smoke") lines.push("", "Smoke budget runs baseline eval + 2-step smoke training, then stops."); if (budget === "small") lines.push("", "Small budget runs a short training job after smoke passes."); if (budget === "real") lines.push("", "Real budget still asks for confirmation before scale-up unless `--yes` is passed."); return lines.join("\n");}
function suggestDataImprovementsFromEval(filePath?: string): string { if (!filePath || !existsSync(filePath)) { return [ "## What data to add next", "After you have eval failures, add training examples that look like the failures — same input style, corrected ideal output, and edge cases.", "Prefer 20 excellent examples over 200 noisy examples.", ].join("\n"); } try { const result = readEvalSummary(filePath); const failures = (result.results ?? []).filter((r) => !r.correct).slice(0, 8); if (failures.length === 0) { return "## What data to add next\nNo eval failures found in the sample. Add harder held-out eval cases before scaling further."; } return [ "## What data to add next", "Add corrected training examples similar to these failures:", ...failures.map((f) => `- Eval #${f.index}: expected ${JSON.stringify(f.expected)} but got ${JSON.stringify(String(f.output).slice(0, 180))}`), "", "For each failure, add 2–5 training rows that show the exact desired behavior and one eval row that stays held out.", ].join("\n"); } catch { return "## What data to add next\nCould not read eval failures. Add examples for the cases your current model gets wrong, then re-run baseline/checkpoint comparison."; }}
function sanitizeName(name: string): string { return name.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "tinker-model";}
function makeDeployFiles(options: { checkpoint: string; alias: string; outDir: string }) { const model = options.checkpoint; const alias = sanitizeName(options.alias); return [ { rel: "README.md", content: `# Deploy ${alias}\n\nThis folder was generated by \`/tinker deploy\`. It gives you copy-paste snippets for using a Tinker sampler checkpoint through Tinker's OpenAI-compatible endpoint.\n\n## Environment\n\n\`\`\`bash\nexport TINKER_API_KEY=...\nexport TINKER_MODEL='${model}'\nexport TINKER_BASE_URL='${TINKER_OAI_BASE_URL}'\n\`\`\`\n\n## Python\n\n\`\`\`bash\npython3 -m pip install openai\npython3 python_client.py\n\`\`\`\n\n## Node\n\n\`\`\`bash\nnpm install openai\nnode node_client.mjs\n\`\`\`\n\n## FastAPI wrapper\n\n\`\`\`bash\npython3 -m pip install fastapi uvicorn openai\nuvicorn fastapi_app:app --reload\n\`\`\`\n\nThis endpoint is best for quick inspection/internal testing. For production serving, export or deploy weights through your preferred serving stack when available.\n`, }, { rel: ".env.example", content: `TINKER_API_KEY=your-api-key\nTINKER_BASE_URL=${TINKER_OAI_BASE_URL}\nTINKER_MODEL=${model}\n`, }, { rel: "python_client.py", content: `import os\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=os.environ["TINKER_API_KEY"],\n base_url=os.environ.get("TINKER_BASE_URL", "${TINKER_OAI_BASE_URL}"),\n)\n\nmodel = os.environ.get("TINKER_MODEL", "${model}")\n\nresponse = client.chat.completions.create(\n model=model,\n messages=[{"role": "user", "content": "Say what you improved in one sentence."}],\n max_tokens=120,\n temperature=0.2,\n)\n\nprint(response.choices[0].message.content)\n`, }, { rel: "node_client.mjs", content: `import OpenAI from "openai";\n\nconst client = new OpenAI({\n apiKey: process.env.TINKER_API_KEY,\n baseURL: process.env.TINKER_BASE_URL ?? "${TINKER_OAI_BASE_URL}",\n});\n\nconst model = process.env.TINKER_MODEL ?? "${model}";\n\nconst response = await client.chat.completions.create({\n model,\n messages: [{ role: "user", content: "Say what you improved in one sentence." }],\n max_tokens: 120,\n temperature: 0.2,\n});\n\nconsole.log(response.choices[0].message.content);\n`, }, { rel: "fastapi_app.py", content: `import os\nfrom fastapi import FastAPI\nfrom pydantic import BaseModel\nfrom openai import OpenAI\n\napp = FastAPI(title="${alias} Tinker checkpoint")\nclient = OpenAI(\n api_key=os.environ["TINKER_API_KEY"],\n base_url=os.environ.get("TINKER_BASE_URL", "${TINKER_OAI_BASE_URL}"),\n)\nMODEL = os.environ.get("TINKER_MODEL", "${model}")\n\nclass ChatRequest(BaseModel):\n message: str\n max_tokens: int = 256\n temperature: float = 0.2\n\n@app.post("/chat")\ndef chat(req: ChatRequest):\n response = client.chat.completions.create(\n model=MODEL,\n messages=[{"role": "user", "content": req.message}],\n max_tokens=req.max_tokens,\n temperature=req.temperature,\n )\n return {"output": response.choices[0].message.content}\n`, }, ];}
async function writeDeployFiles(cwd: string, options: { checkpoint: string; alias: string; outDir: string; force?: boolean }): Promise<string[]> { const dir = path.resolve(cwd, options.outDir); const written: string[] = []; for (const file of makeDeployFiles({ checkpoint: options.checkpoint, alias: options.alias, outDir: options.outDir })) { const target = path.join(dir, file.rel); if (existsSync(target) && !options.force) continue; await mkdir(path.dirname(target), { recursive: true }); await writeFile(target, file.content); written.push(path.relative(cwd, target)); } return written;}
function resolveCheckpointRef(input: string | undefined, state: TinkerState, wizard?: WizardState): { checkpoint?: string; alias: string } { if (!input || input === "latest") { const latest = state.checkpoints.slice(-1)[0]; return { checkpoint: wizard?.checkpointPath ?? latest?.id, alias: wizard?.registeredModel ?? latest?.name ?? "my-finetune" }; } const byAlias = state.checkpoints.find((m) => m.name === input || m.id === input); if (byAlias) return { checkpoint: byAlias.id, alias: byAlias.name }; if (input.startsWith("tinker://")) return { checkpoint: input, alias: "my-finetune" }; return { checkpoint: undefined, alias: input };}
function makeExactEvalScript() { return `"""Simple editable eval for Tinker checkpoints.
Input JSONL format: {"messages": [{"role": "user", "content": "..."}], "expected": "...", "match": "contains"}
Match modes: - exact: normalized output equals normalized expected - contains: normalized expected appears in normalized output - prefix: normalized output starts with normalized expected
Examples: python eval.py --base-model thinkingmachines/Inkling --effort 0.9 --data data/eval.jsonl --out eval_results/baseline.json python eval.py --model-path 'tinker://.../sampler_weights/...' --renderer-model thinkingmachines/Inkling --effort 0.9 --data data/eval.jsonl --out eval_results/step-20.json"""
from __future__ import annotations
import argparseimport asyncioimport jsonimport refrom pathlib import Pathfrom typing import Any
import tinkerfrom tinker_cookbook import model_infofrom tinker_cookbook.renderers import get_renderer, get_text_content
def normalize(text: str) -> str: text = text.strip().lower() text = re.sub(r"\\s+", " ", text) text = text.strip(" \\t\\n\\r.,;:!?") return text
def score_output(output: str, expected: str, match: str) -> bool: out = normalize(output) exp = normalize(expected) if match == "exact": return out == exp if match == "prefix": return out.startswith(exp) if match == "contains": return exp in out raise ValueError(f"unknown match mode: {match!r}")
def load_rows(path: str) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] with open(path) as f: for line_number, line in enumerate(f, start=1): line = line.strip() if not line: continue row = json.loads(line) if not isinstance(row.get("messages"), list): raise ValueError(f"line {line_number}: missing messages[]") if "expected" not in row: raise ValueError(f"line {line_number}: missing expected") rows.append(row) return rows
async def main() -> None: parser = argparse.ArgumentParser(description="Evaluate a base model or Tinker sampler checkpoint.") target = parser.add_mutually_exclusive_group(required=True) target.add_argument("--base-model", help="Tinker base model id for baseline evaluation") target.add_argument("--model-path", help="Tinker sampler checkpoint path") parser.add_argument("--renderer-model", help="Model id used to choose renderer/tokenizer; defaults to --base-model or sampler base model") parser.add_argument("--data", default="data/eval.jsonl") parser.add_argument("--out", default="eval_results/result.json") parser.add_argument("--max-tokens", type=int, default=128) parser.add_argument("--temperature", type=float, default=0.0) parser.add_argument("--effort", type=float, default=0.9, help="Inkling effort in [0, 1); ignored by other renderers") parser.add_argument("--limit", type=int, default=0, help="0 means all rows") args = parser.parse_args()
if not 0.0 <= args.effort < 1.0: raise SystemExit("--effort must be in [0, 1)")
rows = load_rows(args.data) if args.limit: rows = rows[: args.limit] if not rows: raise SystemExit("No eval rows found.")
service = tinker.ServiceClient() if args.model_path: sampling_client = await service.create_sampling_client_async(model_path=args.model_path) target_name = args.model_path renderer_model = args.renderer_model or sampling_client.get_base_model() else: sampling_client = await service.create_sampling_client_async(base_model=args.base_model) target_name = args.base_model renderer_model = args.renderer_model or args.base_model
tokenizer = sampling_client.get_tokenizer() renderer_name = model_info.get_recommended_renderer_name(renderer_model) renderer = get_renderer(renderer_name, tokenizer) stop = renderer.get_stop_sequences()
results: list[dict[str, Any]] = [] correct = 0
for index, row in enumerate(rows, start=1): messages = row["messages"] expected = str(row["expected"]) match = str(row.get("match", "contains")) prompt_kwargs = {"role": "assistant"} if renderer_model.startswith("thinkingmachines/Inkling"): prompt_kwargs["effort"] = args.effort prompt = renderer.build_generation_prompt(messages, **prompt_kwargs) response = await sampling_client.sample_async( prompt=prompt, num_samples=1, sampling_params=tinker.SamplingParams( max_tokens=args.max_tokens, temperature=args.temperature, stop=stop, ), ) sequence = response.sequences[0] if hasattr(response, "sequences") else response.samples[0] parsed_message, termination = renderer.parse_response(sequence.tokens) output = get_text_content(parsed_message) is_correct = score_output(output, expected, match) correct += int(is_correct) results.append({ "index": index, "correct": is_correct, "match": match, "expected": expected, "output": output, "termination": getattr(termination, "value", str(termination)), "messages": messages, }) print(f"[{index}/{len(rows)}] {'✓' if is_correct else '✗'} expected={expected!r} output={output[:120]!r}")
summary = { "target": target_name, "renderer_model": renderer_model, "renderer": renderer_name, "effort": args.effort if renderer_model.startswith("thinkingmachines/Inkling") else None, "data": args.data, "num_examples": len(rows), "num_correct": correct, "accuracy": correct / len(rows), "results": results, } out = Path(args.out) out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\\n") print(f"accuracy={summary['accuracy']:.3f} ({correct}/{len(rows)})") print(f"wrote {out}")
if __name__ == "__main__": asyncio.run(main())`;}
type EvalSummary = { target?: string; renderer_model?: string; renderer?: string; effort?: number | null; data?: string; num_examples?: number; num_correct?: number; accuracy?: number; results?: Array<{ index: number; correct: boolean; expected: string; output: string }>;};
function readEvalSummary(filePath: string): EvalSummary { return JSON.parse(readFileSync(filePath, "utf8")) as EvalSummary;}
function formatEvalSummary(filePath: string): string { const result = readEvalSummary(filePath); const accuracy = typeof result.accuracy === "number" ? `${(result.accuracy * 100).toFixed(1)}%` : "n/a"; const examples = result.num_examples ?? result.results?.length ?? 0; const correct = result.num_correct ?? result.results?.filter((r) => r.correct).length ?? 0; const failures = (result.results ?? []).filter((r) => !r.correct).slice(0, 5); return [ `- File: \`${filePath}\``, `- Target: \`${result.target ?? "unknown"}\``, `- Renderer: \`${result.renderer ?? "unknown"}\`${result.renderer_model ? ` for \`${result.renderer_model}\`` : ""}`, result.effort !== null && result.effort !== undefined ? `- Inkling effort: ${result.effort}` : "", `- Accuracy: ${accuracy} (${correct}/${examples})`, failures.length ? `\n## Sample failures\n${failures.map((f) => `### #${f.index}\n- Expected: ${JSON.stringify(f.expected)}\n- Output: ${JSON.stringify(String(f.output).slice(0, 500))}`).join("\n\n")}` : "\nNo failures recorded.", ].join("\n");}
function compareEvalSummaries(baselinePath: string, candidatePath: string): string { const baseline = readEvalSummary(baselinePath); const candidate = readEvalSummary(candidatePath); const bAcc = baseline.accuracy ?? 0; const cAcc = candidate.accuracy ?? 0; const delta = cAcc - bAcc; const bResults = new Map((baseline.results ?? []).map((r) => [r.index, r])); const effortMismatch = baseline.effort !== null && baseline.effort !== undefined && candidate.effort !== null && candidate.effort !== undefined && baseline.effort !== candidate.effort; const wins: string[] = []; const regressions: string[] = []; for (const r of candidate.results ?? []) { const before = bResults.get(r.index); if (!before) continue; if (!before.correct && r.correct) wins.push(`#${r.index}: ${JSON.stringify(String(r.output).slice(0, 220))}`); if (before.correct && !r.correct) regressions.push(`#${r.index}: expected ${JSON.stringify(r.expected)}, got ${JSON.stringify(String(r.output).slice(0, 220))}`); } return [ `# Eval comparison`, `- Baseline: \`${baselinePath}\` — ${(bAcc * 100).toFixed(1)}% (${baseline.num_correct}/${baseline.num_examples})`, `- Candidate: \`${candidatePath}\` — ${(cAcc * 100).toFixed(1)}% (${candidate.num_correct}/${candidate.num_examples})`, `- Delta: ${delta >= 0 ? "+" : ""}${(delta * 100).toFixed(1)} points`, effortMismatch ? `- ⚠️ Effort mismatch: baseline=${baseline.effort}, candidate=${candidate.effort}. Re-run at identical effort before attributing the delta to fine-tuning.` : (baseline.effort !== null && baseline.effort !== undefined ? `- Inkling effort: ${baseline.effort} (matched)` : ""), wins.length ? `\n## Wins\n${wins.slice(0, 10).map((x) => `- ${x}`).join("\n")}` : "\n## Wins\nNone recorded.", regressions.length ? `\n## Regressions\n${regressions.slice(0, 10).map((x) => `- ${x}`).join("\n")}` : "\n## Regressions\nNone recorded.", ].join("\n");}
type WizardState = { version: 1; createdAt: number; updatedAt: number; dataFile?: string; dataFileInput?: string; model?: string; metric?: string; logPath?: string; validationAt?: number; baselineResult?: string; smokeAt?: number; smokeLogDir?: string; checkpointPath?: string; candidateResult?: string; registeredModel?: string;};
function wizardDir(cwd: string) { return path.join(cwd, ".tinker-pi");}
function wizardStatePath(cwd: string) { return path.join(wizardDir(cwd), "state.json");}
async function readWizardState(cwd: string): Promise<WizardState | undefined> { try { return JSON.parse(await readFile(wizardStatePath(cwd), "utf8")) as WizardState; } catch { return undefined; }}
async function writeWizardState(cwd: string, state: WizardState): Promise<void> { await mkdir(wizardDir(cwd), { recursive: true }); await writeFile(wizardStatePath(cwd), JSON.stringify({ ...state, updatedAt: Date.now() }, null, 2) + "\n");}
async function patchWizardState(cwd: string, patch: Partial<WizardState>): Promise<void> { const existing = await readWizardState(cwd); if (!existing) return; await writeWizardState(cwd, { ...existing, ...patch, version: 1 });}
function rel(cwd: string, filePath?: string) { if (!filePath) return ""; const r = path.relative(cwd, filePath); return r && !r.startsWith("..") ? r : filePath;}
type WizardStep = { key: string; label: string; done: boolean; detail: string; nextCommand?: string; apiUsage?: boolean;};
function findLatestCandidateEval(cwd: string): string | undefined { const dir = path.join(cwd, "eval_results"); if (!existsSync(dir)) return undefined; try { const candidates = readdirSync(dir) .filter((name) => name.endsWith(".json") && name !== "baseline.json") .map((name) => path.join(dir, name)) .sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs); return candidates[0]; } catch { return undefined; }}
function firstSamplerCheckpoint(logDir?: string): string | undefined { if (!logDir || !existsSync(path.join(logDir, "checkpoints.jsonl"))) return undefined; try { return readCheckpoints(logDir).filter((c) => c.sampler_path).slice(-1)[0]?.sampler_path; } catch { return undefined; }}
function wizardSteps(cwd: string, state: WizardState): WizardStep[] { const dataFile = state.dataFile; const logDir = state.logPath ? path.resolve(cwd, state.logPath) : undefined; const baseline = state.baselineResult ?? (existsSync(path.join(cwd, "eval_results", "baseline.json")) ? path.join(cwd, "eval_results", "baseline.json") : undefined); const checkpoint = state.checkpointPath ?? firstSamplerCheckpoint(logDir); const candidate = state.candidateResult ?? findLatestCandidateEval(cwd); const registered = state.registeredModel || (checkpoint ? undefined : state.registeredModel);
return [ { key: "env", label: "Environment ready", done: Boolean(process.env.TINKER_API_KEY) && existsSync(process.execPath), detail: process.env.TINKER_API_KEY ? "TINKER_API_KEY is set" : "Set TINKER_API_KEY first", nextCommand: "/tinker setup", }, { key: "data", label: "Training data selected", done: Boolean(dataFile && existsSync(dataFile)), detail: dataFile ? rel(cwd, dataFile) : "No JSONL selected", nextCommand: "/tinker start path/to/train.jsonl", }, { key: "files", label: "Project files created", done: existsSync(path.join(cwd, "train_sft.py")) && existsSync(path.join(cwd, "eval.py")) && existsSync(path.join(cwd, "data", "eval.jsonl")), detail: "train_sft.py, eval.py, data/eval.jsonl", nextCommand: "/tinker next --create-files", }, { key: "validate", label: "Data validated", done: Boolean(state.validationAt), detail: state.validationAt ? new Date(state.validationAt).toLocaleString() : "Need readiness report", nextCommand: dataFile ? `/tinker validate ${rel(cwd, dataFile)} --model ${state.model ?? DEFAULT_MODEL}` : undefined, }, { key: "baseline", label: "Baseline eval run", done: Boolean(baseline && existsSync(baseline)), detail: baseline ? rel(cwd, baseline) : "Measure base model before training", nextCommand: `/tinker eval baseline --model ${state.model ?? DEFAULT_MODEL}${isInklingModel(state.model ?? DEFAULT_MODEL) ? " --effort 0.9" : ""} --yes`, apiUsage: true, }, { key: "smoke", label: "2-step smoke test run", done: Boolean(state.smokeAt) || Boolean(logDir && existsSync(path.join(logDir, "metrics.jsonl"))), detail: state.smokeAt ? new Date(state.smokeAt).toLocaleString() : "Run tiny training test", nextCommand: "/tinker smoke train_sft.py --yes", apiUsage: true, }, { key: "train", label: "Training/checkpoint available", done: Boolean(checkpoint), detail: checkpoint ? checkpoint : "Run more steps, then save/check checkpoints", nextCommand: state.logPath ? `/tinker monitor ${state.logPath}` : "python train_sft.py max_steps=100", apiUsage: true, }, { key: "checkpoint-eval", label: "Checkpoint eval run", done: Boolean(candidate && existsSync(candidate)), detail: candidate ? rel(cwd, candidate) : "Evaluate trained checkpoint on same eval set", nextCommand: checkpoint ? `/tinker eval checkpoint ${checkpoint} --model ${state.model ?? DEFAULT_MODEL}${isInklingModel(state.model ?? DEFAULT_MODEL) ? " --effort 0.9" : ""} --yes` : undefined, apiUsage: true, }, { key: "compare", label: "Before/after compared", done: Boolean(baseline && candidate && existsSync(baseline) && existsSync(candidate)), detail: baseline && candidate ? `${rel(cwd, baseline)} vs ${rel(cwd, candidate)}` : "Compare baseline and checkpoint evals", nextCommand: baseline && candidate ? `/tinker eval compare ${rel(cwd, baseline)} ${rel(cwd, candidate)}` : undefined, }, { key: "chat", label: "Checkpoint registered for chat", done: Boolean(registered), detail: registered ? registered : "Select checkpoint as a Pi model", nextCommand: checkpoint ? `/tinker use ${checkpoint} my-finetune` : undefined, }, ];}
function renderWizard(cwd: string, state: WizardState): string { const steps = wizardSteps(cwd, state); const next = steps.find((step) => !step.done); const progress = `${steps.filter((s) => s.done).length}/${steps.length}`; return [ `# Fine-tune wizard`, `- Progress: ${progress}`, `- Model: \`${state.model ?? DEFAULT_MODEL}\``, state.dataFile ? `- Training data: \`${rel(cwd, state.dataFile)}\`` : "- Training data: not selected", state.metric ? `- Success metric: ${state.metric}` : "- Success metric: not defined", "", ...steps.map((step) => `${step.done ? "✅" : "⬜"} **${step.label}** — ${step.detail}`), "", next ? `## Next step\n${next.apiUsage ? "⚠️ This step may use the Tinker API.\n\n" : ""}Run:\n\n\`\`\`text\n${next.nextCommand ?? "/tinker next"}\n\`\`\`` : "## Complete\nYou have a validated dataset, before/after evals, and a checkpoint ready to chat with in Pi.", "", "Use `/tinker next` any time to see the next recommended action, or `/tinker reset` to start over.", ].join("\n");}
async function createWizardFiles(cwd: string, state: WizardState, force = false): Promise<string[]> { if (!state.dataFile) throw new Error("Wizard has no dataFile. Run `/tinker start data/train.jsonl` first."); const model = state.model ?? DEFAULT_MODEL; const logPath = state.logPath ?? `logs/sft-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19)}`; const files = [ { rel: "README.md", content: makeProjectReadme({ model, dataFile: state.dataFile, logPath, successMetric: state.metric ?? "Define before scaling." }) }, { rel: "train_sft.py", content: makeSftScript({ dataFile: state.dataFile, model, logPath, maxSteps: "20", batchSize: "8", learningRate: "2e-4", testSize: "0", maxLength: "32768" }) }, { rel: "eval_checkpoint.py", content: makeEvalScript() }, { rel: "eval.py", content: makeExactEvalScript() }, { rel: path.join("data", "eval.jsonl"), content: makeExampleEvalJsonl() }, { rel: "tinker.yaml", content: `task: sft\ndata: ${state.dataFile}\nmodel: ${model}\nlog_path: ${logPath}\nmax_steps: 20\nbatch_size: 8\nlearning_rate: 2e-4\nsuccess_metric: ${state.metric ?? ""}\n` }, { rel: path.join("notes", "plan.md"), content: `# Fine-tuning plan\n\n## Goal\n\n${state.metric ?? "Define what should improve."}\n\n## Wizard\n\nRun \`/tinker next\` for the next step.\n` }, ]; const written: string[] = []; for (const file of files) { const target = path.join(cwd, file.rel); if (existsSync(target) && !force) continue; await mkdir(path.dirname(target), { recursive: true }); await writeFile(target, file.content); written.push(file.rel); } await writeWizardState(cwd, { ...state, logPath, version: 1 }); return written;}
export default async function (pi: ExtensionAPI) { let state = await loadState(); let monitorInterval: ReturnType<typeof setInterval> | undefined;
function sendReport(title: string, body: string, level: ReportLevel = "info") { pi.sendMessage({ customType: MESSAGE_TYPE, content: `# ${title}\n\n${body}`, display: true, details: { level, timestamp: Date.now() }, }); if (process.env.PI_TINKER_AGENT_CLI === "1") { process.stdout.write(`# ${title}\n\n${body}\n`); } }
function registerTinkerProvider() { pi.registerProvider("tinker", { name: "Tinker", baseUrl: TINKER_ANTHROPIC_BASE_URL, apiKey: "$TINKER_API_KEY", api: "anthropic-messages", models: listTinkerChatModels(state.checkpoints), }); }
registerTinkerProvider();
pi.registerMessageRenderer(MESSAGE_TYPE, (message, _options, theme) => { const details = message.details as { level?: ReportLevel } | undefined; const level = details?.level ?? "info"; const color = level === "error" ? "error" : level === "warning" ? "warning" : level === "success" ? "success" : "accent"; const box = new Box(1, 1, (s: string) => theme.bg("customMessageBg", s)); box.addChild(new Text(theme.fg(color, String(message.content)), 0, 0)); return box; });
pi.on("session_shutdown", () => { if (monitorInterval) clearInterval(monitorInterval); });
pi.registerCommand("tinker", { description: "Tinker + Inkling helper: chat, sweep effort, fine-tune, evaluate, monitor, and use checkpoints", handler: async (args, ctx) => { const [subcommandRaw, ...rest] = shellSplit(args); const subcommand = subcommandRaw ?? "help";
if (subcommand === "help" || subcommand === "--help" || subcommand === "-h") { sendReport("Tinker helper", [ "## Start here", "- `/tinker demo` — create a free local example; no API calls.", "- `/tinker improve data.csv --goal \"what should improve\" --budget demo` — prepare your own data without API calls.", "- `/tinker doctor` — check what is installed or missing.", "- `/tinker next` — show the next safe step.", "- `/tinker inkling` — explain Inkling and how to select it with `/model`.", "", "## When you are ready to use the API", "- `/tinker improve data.csv --goal \"...\" --budget smoke --yes` — baseline eval plus two training steps.", "- `/tinker improve data.csv --goal \"...\" --budget small --yes` — short training plus checkpoint comparison.", "- `/tinker monitor <log_dir>` — watch metrics.", "- `/tinker checkpoints <log_dir>` — find and register checkpoints.", "", "## More commands", "- Data: `prepare`, `new`, `examples`, `validate`.", "- Experiments: `eval`, `sft`, `smoke`, `status`.", "- Checkpoints: `use`, `deploy`.", "- Project state: `start`, `next`, `reset`, `setup`, `init`.", "", "Full reference: `docs/commands.md`. API-using commands require confirmation; `--yes` is explicit approval.", ].join("\n")); return; }
if (subcommand === "inkling") { const [actionRaw, ...inklingRest] = rest; const action = actionRaw ?? "info"; const { positional, options } = parseOptions(inklingRest);
if (action === "info" || action === "help" || action === "--help") { sendReport("Inkling on Tinker", [ "Inkling is ready to use in Pi. Set `TINKER_API_KEY`, run `/model`, and choose an Inkling model under the Tinker provider.", "", "## What works", "- Chat, coding tools, images, and streamed reasoning.", "- Shift+Tab changes reasoning effort: low, medium, high, or xhigh.", "- Inkling and Inkling-Small each have 64K and 256K context options.", "", "## To fine-tune Inkling", "Install the Python 3.11+ dependencies:", "```bash", "uv pip install -U 'tinker-cookbook[inkling]'", "```", "Then start without API usage:", "```text", "/tinker improve data.csv --goal \"what should improve\" --budget demo", "```", "", "Inkling defaults to high effort (0.9). Keep the same effort when comparing the base model and a trained checkpoint.", "Model ids: `thinkingmachines/Inkling`, `thinkingmachines/Inkling:peft:262144`, `thinkingmachines/Inkling-Small`, and `thinkingmachines/Inkling-Small:peft:262144`." ].join("\n")); return; }
if (action === "sweep" || action === "sample") { const prompt = String(options.prompt ?? positional.join(" ")).trim(); if (!prompt) { sendReport("Inkling effort sweep", "Usage: `/tinker inkling sweep --prompt \"your representative task\" --efforts low,medium,high,xhigh --yes`", "warning"); return; } let efforts: number[]; try { efforts = parseInklingEfforts(options.efforts); } catch (error) { sendReport("Inkling effort sweep", error instanceof Error ? error.message : String(error), "warning"); return; } if (!process.env.TINKER_API_KEY) { sendReport("Inkling effort sweep", "`TINKER_API_KEY` is not set. Run `/tinker setup` after exporting the key.", "warning"); return; } const confirmed = options.yes === true || (ctx.hasUI ? await ctx.ui.confirm("Run Inkling effort sweep?", `Sampling ${efforts.length} responses uses the Tinker API.`) : false); if (!confirmed) { sendReport("Inkling effort sweep", "Stopped before API usage. Re-run with `--yes` when ready.", "warning"); return; } const pythonArgs = [ "-m", "tinker_cookbook.scripts.inkling.sample_reasoning", `prompt=${prompt}`, `efforts=[${efforts.join(",")}]`, ]; if (options["max-tokens"]) pythonArgs.push(`max_tokens=${Number(options["max-tokens"])}`); if (options.temperature) pythonArgs.push(`temperature=${Number(options.temperature)}`); try { ctx.ui.setStatus("tinker", "Inkling effort sweep"); const { stdout, stderr } = await execFile("python3", pythonArgs, { cwd: ctx.cwd, timeout: Number(options.timeout ?? 1_800_000), maxBuffer: 24 * 1024 * 1024, }); sendReport("Inkling effort sweep completed", `\`efforts=${efforts.join(", ")}\`\n\n\`\`\`text\n${`${stdout}\n${stderr}`.trim()}\n\`\`\``, "success"); } catch (error: any) { sendReport("Inkling effort sweep failed", [ error?.message ?? String(error), error?.stdout ? `\n## stdout\n\`\`\`text\n${String(error.stdout).split(/\n/).slice(-100).join("\n")}\n\`\`\`` : "", error?.stderr ? `\n## stderr\n\`\`\`text\n${String(error.stderr).split(/\n/).slice(-100).join("\n")}\n\`\`\`` : "", "\nInstall/upgrade with `uv pip install -U 'tinker-cookbook[inkling]'`.", ].filter(Boolean).join("\n"), "error"); } finally { ctx.ui.setStatus("tinker", undefined); } return; }
sendReport("Inkling", `Unknown action: ${action}. Use \`/tinker inkling\` or \`/tinker inkling sweep ...\`.`, "warning"); return; }
if (subcommand === "new" || subcommand === "finetune" || subcommand === "demo") { const { positional, options } = parseOptions(rest); const force = options.force === true; let dataFileInput = positional[0] ? String(positional[0]) : ""; let metric = String(options.goal ?? options.metric ?? ""); let model = String(options.model ?? ""); const exampleSlug = subcommand === "demo" ? "customer-support" : options.example === true ? "customer-support" : options.example ? String(options.example) : "";
if (exampleSlug) { try { const result = await writeExampleTemplate(ctx.cwd, exampleSlug, force); dataFileInput = path.join("examples", result.template.slug, "train.jsonl"); metric = metric || result.template.goal; } catch (error) { sendReport("Tinker new", error instanceof Error ? error.message : String(error), "error"); return; } }
if (ctx.hasUI && !dataFileInput) { const choice = await ctx.ui.select("How do you want to start?", [ "use built-in customer-support example", "use built-in structured-extraction example", "use built-in concise-writing example", "convert my CSV/JSON/docs", "I already have chat JSONL", ]); if (choice?.startsWith("use built-in")) { const slug = choice.includes("structured") ? "structured-extraction" : choice.includes("concise") ? "concise-writing" : "customer-support"; const result = await writeExampleTemplate(ctx.cwd, slug, force); dataFileInput = path.join("examples", result.template.slug, "train.jsonl"); metric = metric || result.template.goal; } else { dataFileInput = (await ctx.ui.input("Path to CSV, JSON, JSONL, docs dir, or train.jsonl", "data/train.csv"))?.trim() ?? ""; } }
if (!dataFileInput) { sendReport("Tinker new", [ "Usage:", "```text", "/tinker new --example customer-support", "/tinker new data/train.csv --goal \"better support answers\"", "/tinker new data/train.jsonl --goal \"what should improve\"", "```", ].join("\n"), "warning"); return; }
let dataFile = path.resolve(ctx.cwd, dataFileInput); if (!existsSync(dataFile)) { sendReport("Tinker new", `Input not found: ${dataFile}`, "error"); return; } if (!/\.jsonl$/i.test(dataFile) || options.prepare === true) { const out = path.resolve(ctx.cwd, String(options.out ?? "data/train.jsonl")); try { const prepared = await prepareDataset(dataFile, out); dataFile = prepared.outFile; dataFileInput = rel(ctx.cwd, prepared.outFile); sendReport("Dataset prepared", [ `Converted \`${rel(ctx.cwd, prepared.inputPath)}\` (${prepared.format}) to \`${rel(ctx.cwd, prepared.outFile)}\` with ${prepared.rows} rows.`, prepared.warnings.length ? `\n## Warnings\n${prepared.warnings.map((w) => `- ${w}`).join("\n")}` : "", `\n## Samples\n${prepared.samples.map((x, i) => `### ${i + 1}\n\`\`\`json\n${x}\n\`\`\``).join("\n\n")}`, ].filter(Boolean).join("\n"), "success"); } catch (error) { sendReport("Dataset prepare failed", error instanceof Error ? error.message : String(error), "error"); return; } }
if (ctx.hasUI && !model) { const choice = await ctx.ui.select("Pick a starter model", [ `${INKLING_MODEL} — Thinking Machines' multimodal open-weights model`, `${QWEN_38_27B_MODEL} — dense 27B student`, `${QWEN_38_27B_256K_MODEL} — same model, 256K chat`, "Qwen/Qwen3.5-9B-Base — cheaper small base model", "Qwen/Qwen3.5-35B-A3B-Base — stronger Qwen MoE base", "custom", ]); model = choice === "custom" ? ((await ctx.ui.input("Tinker model id", DEFAULT_MODEL))?.trim() || DEFAULT_MODEL) : (choice?.split(" — ")[0] || DEFAULT_MODEL); } model = model || DEFAULT_MODEL;
if (ctx.hasUI && !metric) { metric = (await ctx.ui.input("What should improve?", "e.g. support answer quality, extraction accuracy, concise writing"))?.trim() ?? ""; } metric = metric || "Improve task quality on held-out examples.";
const state: WizardState = { version: 1, createdAt: Date.now(), updatedAt: Date.now(), dataFile, dataFileInput, model, metric, logPath: String(options.log ?? `logs/sft-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19)}`), }; await writeWizardState(ctx.cwd, state); const written = await createWizardFiles(ctx.cwd, state, force); const saved = await readWizardState(ctx.cwd) ?? state; sendReport("Tinker golden path ready", [ written.length ? `Created ${written.map((x) => `\`${x}\``).join(", ")}.` : "Project files already existed; left them untouched.", "", recommendPlan(metric, readJsonl(dataFile).length), "", renderWizard(ctx.cwd, saved), ].join("\n"), "success"); return; }
if (subcommand === "prepare") { const { positional, options } = parseOptions(rest); const input = positional[0]; if (!input) { sendReport("Tinker prepare", "Usage: `/tinker prepare data.csv --out data/train.jsonl`", "warning"); return; } const inputPath = path.resolve(ctx.cwd, input); const outFile = path.resolve(ctx.cwd, String(options.out ?? "data/train.jsonl")); if (!existsSync(inputPath)) { sendReport("Tinker prepare", `Input not found: ${inputPath}`, "error"); return; } try { const prepared = await prepareDataset(inputPath, outFile); sendReport("Dataset prepared", [ `Converted \`${rel(ctx.cwd, inputPath)}\` (${prepared.format}) to \`${rel(ctx.cwd, outFile)}\`.`, `- Rows: ${prepared.rows}`, prepared.warnings.length ? `\n## Warnings\n${prepared.warnings.map((w) => `- ${w}`).join("\n")}` : "", "\n## Next", "```text", `/tinker new ${rel(ctx.cwd, outFile)} --goal "what should improve"`, `/tinker validate ${rel(ctx.cwd, outFile)} --quick`, "```", ].filter(Boolean).join("\n"), "success"); } catch (error) { sendReport("Dataset prepare failed", error instanceof Error ? error.message : String(error), "error"); } return; }
if (subcommand === "recommend") { const { positional, options } = parseOptions(rest); const goal = String(options.goal ?? positional.join(" ") ?? ""); const data = options.data ? path.resolve(ctx.cwd, String(options.data)) : undefined; let rows: number | undefined; if (data && existsSync(data) && /\.jsonl$/i.test(data)) { try { rows = readJsonl(data).length; } catch {} } sendReport("Tinker recommendation", recommendPlan(goal, rows), "info"); return; }
if (subcommand === "examples") { const { positional, options } = parseOptions(rest); const action = positional[0] ?? "list"; if (action === "list") { sendReport("Tinker examples", [ "Available starter examples:", ...EXAMPLE_TEMPLATES.map((t) => `- \`${t.slug}\` — ${t.title}`), "", "Copy one into this project:", "```text", "/tinker examples copy customer-support", "/tinker new --example customer-support", "```", ].join("\n")); return; } if (action === "copy") { const slug = positional[1] ?? "customer-support"; try { const result = await writeExampleTemplate(ctx.cwd, slug, options.force === true); sendReport("Tinker example copied", [ `Copied \`${result.template.title}\`.`, result.files.length ? `Wrote ${result.files.map((f) => `\`${f}\``).join(", ")}.` : "Files already existed; left them untouched.", "", "Start from it:", "```text", `/tinker new examples/${result.template.slug}/train.jsonl --goal "${result.template.goal}"`, "```", ].join("\n"), "success"); } catch (error) { sendReport("Tinker examples", error instanceof Error ? error.message : String(error), "error"); } return; } sendReport("Tinker examples", "Usage: `/tinker examples list` or `/tinker examples copy customer-support`", "warning"); return; }
if (subcommand === "doctor") { const { positional } = parseOptions(rest); sendReport("Tinker doctor", await buildDoctorReport(ctx.cwd, positional[0]), "info"); return; }
if (subcommand === "improve") { const { positional, options } = parseOptions(rest); let budget: ImproveBudget; try { budget = parseImproveBudget(options.budget ?? "demo"); } catch (error) { sendReport("Tinker improve", error instanceof Error ? error.message : String(error), "warning"); return; }
const force = options.force === true; const yes = options.yes === true; const model = String(options.model ?? DEFAULT_MODEL); const metric = String(options.goal ?? options.metric ?? "Improve task quality on held-out examples."); const steps = budgetMaxSteps(budget, options.steps ?? options.max_steps); const existingWizard = await readWizardState(ctx.cwd); let dataFileInput = positional[0] ? String(positional[0]) : existingWizard?.dataFileInput ?? existingWizard?.dataFile ?? ""; let dataFile = dataFileInput ? path.resolve(ctx.cwd, dataFileInput) : ""; const sections: string[] = [managedRunPlan(budget)];
try { if (!dataFileInput) { sendReport("Tinker improve", [ "Usage:", "```text", "/tinker improve data.csv --goal \"better support answers\" --budget demo", "/tinker improve data/train.jsonl --goal \"better extraction\" --budget small --yes", "```", "", "Use `--budget demo` to do everything up to real API usage.", ].join("\n"), "warning"); return; } if (!existsSync(dataFile)) { sendReport("Tinker improve", `Input not found: ${dataFile}`, "error"); return; }
if (!/\.jsonl$/i.test(dataFile) || options.prepare === true) { const prepared = await prepareDataset(dataFile, path.resolve(ctx.cwd, String(options.out ?? "data/train.jsonl"))); dataFile = prepared.outFile; dataFileInput = rel(ctx.cwd, prepared.outFile); sections.push(`## Data prepared\nConverted \`${rel(ctx.cwd, prepared.inputPath)}\` (${prepared.format}) to \`${rel(ctx.cwd, prepared.outFile)}\` with ${prepared.rows} rows.`); if (prepared.warnings.length) sections.push(`## Data preparation warnings\n${prepared.warnings.map((w) => `- ${w}`).join("\n")}`); } else { sections.push(`## Data selected\nUsing \`${rel(ctx.cwd, dataFile)}\`.`); }
const stateForRun: WizardState = existingWizard ?? { version: 1, createdAt: Date.now(), updatedAt: Date.now(), dataFile, dataFileInput, model, metric, logPath: String(options.log ?? `logs/sft-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19)}`), }; stateForRun.dataFile = dataFile; stateForRun.dataFileInput = dataFileInput; stateForRun.model = String(options.model ?? stateForRun.model ?? model); stateForRun.metric = String(options.goal ?? options.metric ?? stateForRun.metric ?? metric); stateForRun.logPath = String(options.log ?? stateForRun.logPath ?? `logs/sft-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19)}`); await writeWizardState(ctx.cwd, stateForRun); const written = await createWizardFiles(ctx.cwd, stateForRun, force); sections.push(written.length ? `## Project files\nCreated ${written.map((x) => `\`${x}\``).join(", ")}.` : "## Project files\nProject files already exist; left them untouched.");
sections.push(await buildDoctorReport(ctx.cwd, dataFileInput)); sections.push(`## Lightweight validation\n${validateDataset(dataFile)}`);
if (budget === "demo") { sections.push("## Stopped before API usage\nDemo budget completed setup, project files, doctor, and lightweight validation without touching the Tinker API."); sections.push("## Next commands\n```text\n/tinker validate " + dataFileInput + " --model " + stateForRun.model + "\n/tinker eval baseline --model " + stateForRun.model + " --yes\n/tinker smoke train_sft.py --yes\n```" ); sendReport("Tinker improve plan ready", sections.join("\n\n---\n\n"), "success"); return; }
if (!process.env.TINKER_API_KEY) { sendReport("Tinker improve", "`TINKER_API_KEY` is not set. Run `/tinker improve ... --budget demo` for no-API setup, or set the key before `smoke`, `small`, or `real` budgets.", "warning"); return; }
const confirmed = yes || (ctx.hasUI ? await ctx.ui.confirm("Run managed Tinker improvement?", `Budget ${budget} may incur Tinker API usage. Continue?`) : false); if (!confirmed) { sendReport("Tinker improve", "Stopped before API usage. Re-run with `--yes` when you are ready.", "warning"); return; }
const script = path.resolve(ctx.cwd, "eval.py"); const evalData = path.resolve(ctx.cwd, "data", "eval.jsonl"); const evalEffort = String(options.effort ?? 0.9); if (!existsSync(script) || !existsSync(evalData)) { sendReport("Tinker improve", "Missing `eval.py` or `data/eval.jsonl`. Run `/tinker eval init`, edit eval rows, then re-run improve.", "warning"); return; }
ctx.ui.setStatus("tinker", "managed improve: baseline eval"); const baselineOut = path.resolve(ctx.cwd, String(options["baseline-out"] ?? "eval_results/baseline.json")); let baselineNeedsRun = !existsSync(baselineOut) || force; if (!baselineNeedsRun && isInklingModel(stateForRun.model)) { try { baselineNeedsRun = readEvalSummary(baselineOut).effort !== Number(evalEffort); if (baselineNeedsRun) sections.push(`## Baseline invalidated\nExisting baseline used a different or unknown Inkling effort; re-running at effort ${evalEffort}.`); } catch { baselineNeedsRun = true; } } if (baselineNeedsRun) { const { stdout, stderr } = await execFile("python3", [script, "--base-model", stateForRun.model ?? model, "--effort", evalEffort, "--data", evalData, "--out", baselineOut], { cwd: ctx.cwd, timeout: Number(options.timeout ?? 1_800_000), maxBuffer: 12 * 1024 * 1024 }); sections.push(`## Baseline eval\n${formatEvalSummary(baselineOut)}\n\n<details><summary>Output tail</summary>\n\n\`\`\`text\n${`${stdout}\n${stderr}`.trim().split(/\n/).slice(-40).join("\n")}\n\`\`\`\n</details>`); await patchWizardState(ctx.cwd, { baselineResult: baselineOut, model: stateForRun.model }); } else { sections.push(`## Baseline eval\nUsing existing \`${rel(ctx.cwd, baselineOut)}\`.\n\n${formatEvalSummary(baselineOut)}`); }
ctx.ui.setStatus("tinker", "managed improve: smoke train"); const trainScript = path.resolve(ctx.cwd, "train_sft.py"); const baseLog = String(stateForRun.logPath ?? "logs/sft-managed"); const smokeLog = `${baseLog}-smoke`; const smoke = await execFile("python3", [trainScript, "max_steps=2", `log_path=${smokeLog}`], { cwd: ctx.cwd, timeout: Number(options.timeout ?? 1_800_000), maxBuffer: 12 * 1024 * 1024 }); sections.push(`## Smoke train\n2-step smoke training completed.\n\nLatest metrics:\n${latestMetrics(path.join(ctx.cwd, smokeLog, "metrics.jsonl")) ?? "No metrics found yet."}\n\n<details><summary>Output tail</summary>\n\n\`\`\`text\n${`${smoke.stdout}\n${smoke.stderr}`.trim().split(/\n/).slice(-40).join("\n")}\n\`\`\`\n</details>`); await patchWizardState(ctx.cwd, { smokeAt: Date.now(), smokeLogDir: smokeLog });
if (budget === "smoke") { sections.push("## Stopped after smoke budget\nSmoke completed. Inspect examples/metrics, then re-run with `--budget small --yes` if this looks sane."); sendReport("Tinker improve smoke completed", sections.join("\n\n---\n\n"), "success"); return; }
ctx.ui.setStatus("tinker", `managed improve: train ${steps} steps`); const trainLog = `${baseLog}-${budget}-${steps}`; const train = await execFile("python3", [trainScript, `max_steps=${steps}`, `log_path=${trainLog}`], { cwd: ctx.cwd, timeout: Number(options.trainTimeout ?? options.timeout ?? 3_600_000), maxBuffer: 12 * 1024 * 1024 }); const checkpoint = firstSamplerCheckpoint(path.resolve(ctx.cwd, trainLog)); sections.push(`## Training\nRan ${steps} steps in \`${trainLog}\`.\n\nLatest metrics:\n${latestMetrics(path.join(ctx.cwd, trainLog, "metrics.jsonl")) ?? "No metrics found yet."}\n\n${checkpoint ? `Sampler checkpoint: \`${checkpoint}\`` : "No sampler checkpoint found yet."}\n\n<details><summary>Output tail</summary>\n\n\`\`\`text\n${`${train.stdout}\n${train.stderr}`.trim().split(/\n/).slice(-60).join("\n")}\n\`\`\`\n</details>`); await patchWizardState(ctx.cwd, { checkpointPath: checkpoint, logPath: trainLog });
if (checkpoint) { ctx.ui.setStatus("tinker", "managed improve: checkpoint eval"); const candidateOut = path.resolve(ctx.cwd, String(options["candidate-out"] ?? `eval_results/${sanitizeName(path.basename(trainLog))}.json`)); const evalRun = await execFile("python3", [script, "--model-path", checkpoint, "--renderer-model", stateForRun.model ?? model, "--effort", evalEffort, "--data", evalData, "--out", candidateOut], { cwd: ctx.cwd, timeout: Number(options.timeout ?? 1_800_000), maxBuffer: 12 * 1024 * 1024 }); sections.push(`## Checkpoint eval\n${formatEvalSummary(candidateOut)}\n\n<details><summary>Output tail</summary>\n\n\`\`\`text\n${`${evalRun.stdout}\n${evalRun.stderr}`.trim().split(/\n/).slice(-40).join("\n")}\n\`\`\`\n</details>`); sections.push(compareEvalSummaries(baselineOut, candidateOut)); sections.push(suggestDataImprovementsFromEval(candidateOut)); const alias = sanitizeName(String(options.alias ?? `tinker-${path.basename(trainLog)}`)); if (options.register !== false) { state.checkpoints = state.checkpoints.filter((m) => m.id !== checkpoint && m.name !== alias); state.checkpoints.push(checkpointRegistration({ id: checkpoint, name: alias, baseModel: stateForRun.model })); await saveState(state); registerTinkerProvider(); await patchWizardState(ctx.cwd, { checkpointPath: checkpoint, candidateResult: candidateOut, registeredModel: alias }); sections.push(`## Registered for chat\nRegistered \`${alias}\`. Use \`/model\` to select it, or generate app snippets with:\n\n\`\`\`text\n/tinker deploy ${alias}\n\`\`\``); } } else { sections.push("## No checkpoint yet\nTraining completed but no sampler checkpoint was found. Check `checkpoints.jsonl` or increase steps/save frequency."); }
sendReport("Tinker improve completed", sections.join("\n\n---\n\n"), "success"); } catch (error: any) { sendReport("Tinker improve failed", [ error?.message ?? String(error), error?.stdout ? `\n## stdout\n\`\`\`text\n${String(error.stdout).split(/\n/).slice(-80).join("\n")}\n\`\`\`` : "", error?.stderr ? `\n## stderr\n\`\`\`text\n${String(error.stderr).split(/\n/).slice(-80).join("\n")}\n\`\`\`` : "", "\nRun `/tinker doctor` and `/skill:tinker-debug` with this output.", ].filter(Boolean).join("\n"), "error"); } finally { ctx.ui.setStatus("tinker", undefined); } return; }
if (subcommand === "deploy") { const { positional, options } = parseOptions(rest); const wizard = await readWizardState(ctx.cwd); const resolved = resolveCheckpointRef(positional[0], state, wizard); if (!resolved.checkpoint) { sendReport("Tinker deploy", [ "No checkpoint found to deploy.", "", "Usage:", "```text", "/tinker deploy tinker://.../sampler_weights/... my-model", "/tinker deploy latest", "/tinker deploy <registered-alias>", "```", ].join("\n"), "warning"); return; } const alias = sanitizeName(String(options.alias ?? positional[1] ?? resolved.alias ?? "my-finetune")); const outDir = String(options.out ?? path.join("deploy", alias)); const written = await writeDeployFiles(ctx.cwd, { checkpoint: resolved.checkpoint, alias, outDir, force: options.force === true }); sendReport("Tinker deploy files ready", [ written.length ? `Wrote ${written.map((x) => `\`${x}\``).join(", ")}.` : "Deploy files already existed; left them untouched. Re-run with `--force` to overwrite.", "", "## Use it", "```bash", `cd ${outDir}`, "cp .env.example .env # then fill TINKER_API_KEY", "python3 -m pip install openai", "python3 python_client.py", "```", "", "## App wrapper", "```bash", "python3 -m pip install fastapi uvicorn openai", "uvicorn fastapi_app:app --reload", "```", ].join("\n"), "success"); return; }
if (subcommand === "start") { const { positional, options } = parseOptions(rest); let dataFileInput = positional[0] ? String(positional[0]) : ""; let model = String(options.model ?? ""); let metric = String(options.metric ?? ""); const force = options.force === true;
if (ctx.hasUI && !dataFileInput) { dataFileInput = (await ctx.ui.input("Where is your training JSONL?", "data/train.jsonl"))?.trim() ?? ""; } if (!dataFileInput) { sendReport("Fine-tune wizard", "Usage: `/tinker start data/train.jsonl --model thinkingmachines/Inkling --metric 'what should improve'`", "warning"); return; } const dataFile = path.resolve(ctx.cwd, dataFileInput); if (!existsSync(dataFile)) { sendReport("Fine-tune wizard", `Data file not found: ${dataFile}`, "error"); return; }
if (ctx.hasUI && !model) { const choice = await ctx.ui.select("Pick a starter model", [ `${INKLING_MODEL} — Thinking Machines' multimodal open-weights model`, `${QWEN_38_27B_MODEL} — dense 27B student`, `${QWEN_38_27B_256K_MODEL} — same model, 256K chat`, "Qwen/Qwen3.5-9B-Base — cheaper small base model", "Qwen/Qwen3.5-35B-A3B-Base — stronger Qwen MoE base", "custom", ]); model = choice === "custom" ? ((await ctx.ui.input("Tinker model id", DEFAULT_MODEL))?.trim() || DEFAULT_MODEL) : (choice?.split(" — ")[0] || DEFAULT_MODEL); } model = model || DEFAULT_MODEL;
if (ctx.hasUI && !metric) { metric = (await ctx.ui.input("What should the model get better at?", "e.g. support answers, concise rewrites, JSON extraction accuracy"))?.trim() ?? ""; } metric = metric || "Define success before scaling beyond a smoke test.";
const state: WizardState = { version: 1, createdAt: Date.now(), updatedAt: Date.now(), dataFile, dataFileInput, model, metric, logPath: String(options.log ?? `logs/sft-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19)}`), }; await writeWizardState(ctx.cwd, state); const written = await createWizardFiles(ctx.cwd, state, force); const saved = await readWizardState(ctx.cwd) ?? state; sendReport("Fine-tune wizard started", [ written.length ? `Created ${written.map((x) => `\`${x}\``).join(", ")}.` : "Project files already existed; left them untouched.", "", renderWizard(ctx.cwd, saved), ].join("\n"), "success"); return; }
if (subcommand === "next") { const { options } = parseOptions(rest); const state = await readWizardState(ctx.cwd); if (!state) { sendReport("Fine-tune wizard", "No wizard state found. Start with `/tinker start data/train.jsonl`.", "warning"); return; } const steps = wizardSteps(ctx.cwd, state); const next = steps.find((step) => !step.done); if ((options["create-files"] === true || next?.key === "files") && state.dataFile) { const written = await createWizardFiles(ctx.cwd, state, options.force === true); const saved = await readWizardState(ctx.cwd) ?? state; sendReport("Fine-tune wizard files", [ written.length ? `Created ${written.map((x) => `\`${x}\``).join(", ")}.` : "All project files already exist.", "", renderWizard(ctx.cwd, saved), ].join("\n"), "success"); return; } sendReport("Fine-tune wizard", renderWizard(ctx.cwd, state), next?.apiUsage ? "warning" : "info"); return; }
if (subcommand === "reset") { await rm(wizardDir(ctx.cwd), { recursive: true, force: true }); ctx.ui.setWidget("tinker-monitor", undefined); sendReport("Fine-tune wizard", "Reset `.tinker-pi/`. Run `/tinker start data/train.jsonl` to begin again.", "success"); return; }
if (subcommand === "setup") { const checks: string[] = []; checks.push(process.env.TINKER_API_KEY ? "✅ `TINKER_API_KEY` is set." : "❌ `TINKER_API_KEY` is not set. Get one from https://tinker-console.thinkingmachines.ai and export it."); checks.push((await commandExists("python3")) ? "✅ `python3` found." : "❌ `python3` not found."); try { const { stdout } = await execFile("python3", ["-c", "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"], { timeout: 20_000 }); const [major, minor] = stdout.trim().split(".").map(Number); checks.push(major > 3 || (major === 3 && minor >= 11) ? `✅ Python ${stdout.trim()} supports Inkling.` : `❌ Inkling requires Python 3.11+ (found ${stdout.trim()}).`); } catch {} checks.push((await commandExists("uv")) ? "✅ `uv` found." : "⚠️ `uv` not found. You can still use pip, but uv is recommended."); checks.push((await commandExists("tinker")) ? "✅ `tinker` CLI found." : "⚠️ `tinker` CLI not found on PATH."); try { const { stdout, stderr } = await execFile("python3", ["-c", "import importlib.metadata as m; print(m.version('tinker'))"], { timeout: 20_000 }); const version = stdout.trim() || stderr.trim(); const [major, minor] = version.split(".").map(Number); checks.push(major > 0 || minor >= 23 ? `✅ Tinker SDK ${version} supports Inkling.` : `❌ Inkling requires Tinker SDK 0.23+ (found ${version}).`); } catch { checks.push("⚠️ Python cannot import `tinker`. Install with: `uv pip install -U 'tinker-cookbook[inkling]'`."); } try { const { stdout } = await execFile("python3", ["-c", "import tinker_cookbook; print('ok')"], { timeout: 20_000 }); checks.push(`✅ Python can import \`tinker_cookbook\` (${stdout.trim()}).`); } catch { checks.push("⚠️ Python cannot import `tinker_cookbook`. Install with: `uv pip install -U 'tinker-cookbook[inkling]'`."); } try { await execFile("python3", ["-c", "import tml_renderers; print('ok')"], { timeout: 20_000 }); checks.push("✅ Python can import `tml_renderers` for Inkling."); } catch { checks.push("⚠️ Python cannot import `tml_renderers`. Install with: `uv pip install -U 'tinker-cookbook[inkling]'` (Python 3.11+ required)."); } sendReport("Tinker setup check", checks.join("\n")); return; }
if (subcommand === "validate") { const { positional, options } = parseOptions(rest); const file = positional[0]; if (!file) { sendReport("Tinker validate", "Usage: `/tinker validate data/train.jsonl --model thinkingmachines/Inkling`", "warning"); return; } const dataFile = path.resolve(ctx.cwd, file); const model = String(options.model ?? DEFAULT_MODEL); const maxExamples = Number(options.examples ?? 200); const maxLength = Number(options["max-length"] ?? 32768); const lightweight = validateDataset(dataFile); if (options.quick === true) { sendReport("Tinker dataset validation", lightweight, "info"); return; } try { const python = await validateDatasetWithPython(dataFile, model, maxExamples, maxLength); await patchWizardState(ctx.cwd, { validationAt: Date.now(), dataFile, dataFileInput: file, model }); sendReport("Tinker dataset validation", `${python}\n\n---\n\n${lightweight}`, "info"); } catch (error) { sendReport("Tinker dataset validation", `${lightweight}\n\n## Python-backed validation failed\n${error instanceof Error ? error.message : String(error)}`, "warning"); } return; }
if (subcommand === "eval") { const [actionRaw, ...evalRest] = rest; const action = actionRaw ?? "help"; const { positional, options } = parseOptions(evalRest);
if (action === "help" || action === "--help" || action === "-h") { sendReport("Tinker eval", [ "Commands:", "- `/tinker eval init` — create `eval.py` and `data/eval.jsonl`.", "- `/tinker eval baseline --model thinkingmachines/Inkling --effort 0.9` — evaluate base Inkling.", "- `/tinker eval checkpoint tinker://... --model thinkingmachines/Inkling --effort 0.9` — evaluate a sampler checkpoint at the same effort.", "- `/tinker eval compare eval_results/baseline.json eval_results/checkpoint.json` — compare results.", ].join("\n")); return; }
if (action === "init") { const force = options.force === true; const files = [ { rel: "eval.py", content: makeExactEvalScript() }, { rel: path.join("data", "eval.jsonl"), content: makeExampleEvalJsonl() }, ]; const written: string[] = []; for (const file of files) { const target = path.join(ctx.cwd, file.rel); if (existsSync(target) && !force) { sendReport("Tinker eval init", `${target} already exists. Re-run with --force to overwrite.`, "warning"); return; } await mkdir(path.dirname(target), { recursive: true }); await writeFile(target, file.content); written.push(file.rel); } sendReport("Tinker eval initialized", [ `Wrote ${written.map((x) => `\`${x}\``).join(", ")}.`, "", "Edit `data/eval.jsonl`, then run:", "```text", "/tinker eval baseline --model thinkingmachines/Inkling --effort 0.9 --yes", "/tinker eval checkpoint tinker://.../sampler_weights/... --model thinkingmachines/Inkling --effort 0.9 --yes", "/tinker eval compare eval_results/baseline.json eval_results/checkpoint.json", "```", ].join("\n"), "success"); return; }
if (action === "baseline" || action === "checkpoint") { const script = path.resolve(ctx.cwd, String(options.script ?? "eval.py")); if (!existsSync(script)) { sendReport("Tinker eval", `Missing \`${script}\`. Run \`/tinker eval init\` first.`, "warning"); return; } const model = String(options.model ?? options["base-model"] ?? DEFAULT_MODEL); const data = path.resolve(ctx.cwd, String(options.data ?? "data/eval.jsonl")); if (!existsSync(data)) { sendReport("Tinker eval", `Eval data not found: ${data}`, "error"); return; } const checkpoint = action === "checkpoint" ? positional[0] : undefined; if (action === "checkpoint" && !checkpoint) { sendReport("Tinker eval checkpoint", "Usage: `/tinker eval checkpoint tinker://.../sampler_weights/... --model thinkingmachines/Inkling --effort 0.9`", "warning"); return; } const out = path.resolve(ctx.cwd, String(options.out ?? (action === "baseline" ? "eval_results/baseline.json" : `eval_results/${String(checkpoint).split("/").slice(-1)[0] || "checkpoint"}.json`))); const ok = options.yes === true || !ctx.hasUI ? true : await ctx.ui.confirm("Run Tinker eval?", "This will sample from Tinker and may incur API usage. Continue?"); if (!ok) { sendReport("Tinker eval", "Cancelled.", "warning"); return; } const argsForPython = action === "baseline" ? [script, "--base-model", model, "--data", data, "--out", out] : [script, "--model-path", String(checkpoint), "--renderer-model", model, "--data", data, "--out", out]; if (options.limit) argsForPython.push("--limit", String(options.limit)); if (options["max-tokens"]) argsForPython.push("--max-tokens", String(options["max-tokens"])); if (options.temperature) argsForPython.push("--temperature", String(options.temperature)); if (options.effort) argsForPython.push("--effort", String(options.effort)); try { ctx.ui.setStatus("tinker", `Tinker eval: ${action}`); const { stdout, stderr } = await execFile("python3", argsForPython, { cwd: ctx.cwd, timeout: Number(options.timeout ?? 1_800_000), maxBuffer: 12 * 1024 * 1024, }); await patchWizardState(ctx.cwd, action === "baseline" ? { baselineResult: out, model } : { candidateResult: out, checkpointPath: String(checkpoint), model }); sendReport("Tinker eval completed", [ formatEvalSummary(out), `\n## Output tail\n\`\`\`text\n${`${stdout}\n${stderr}`.trim().split(/\n/).slice(-60).join("\n")}\n\`\`\``, ].join("\n"), "success"); } catch (error: any) { sendReport("Tinker eval failed", [ error?.message ?? String(error), error?.stdout ? `\n## stdout\n\`\`\`text\n${String(error.stdout).split(/\n/).slice(-80).join("\n")}\n\`\`\`` : "", error?.stderr ? `\n## stderr\n\`\`\`text\n${String(error.stderr).split(/\n/).slice(-80).join("\n")}\n\`\`\`` : "", ].filter(Boolean).join("\n"), "error"); } finally { ctx.ui.setStatus("tinker", undefined); } return; }
if (action === "compare") { const baseline = positional[0] ? path.resolve(ctx.cwd, positional[0]) : path.resolve(ctx.cwd, "eval_results/baseline.json"); const candidate = positional[1] ? path.resolve(ctx.cwd, positional[1]) : ""; if (!candidate) { sendReport("Tinker eval compare", "Usage: `/tinker eval compare eval_results/baseline.json eval_results/checkpoint.json`", "warning"); return; } try { sendReport("Tinker eval comparison", compareEvalSummaries(baseline, candidate), "info"); } catch (error) { sendReport("Tinker eval comparison", `Could not compare eval results: ${error instanceof Error ? error.message : String(error)}`, "error"); } return; }
sendReport("Tinker eval", `Unknown eval action: ${action}. Run \`/tinker eval help\`.`, "warning"); return; }
if (subcommand === "init") { const { positional, options } = parseOptions(rest); let dataFileArg = positional[0]; let model = String(options.model ?? ""); let successMetric = String(options.metric ?? ""); if (ctx.hasUI && !dataFileArg) { dataFileArg = (await ctx.ui.input("Training JSONL path", "data/train.jsonl"))?.trim(); } if (ctx.hasUI && !model) { const modelChoice = await ctx.ui.select("Choose a starting model", [ `${INKLING_MODEL} — Thinking Machines' multimodal open-weights model`, `${QWEN_38_27B_MODEL} — dense 27B student`, `${QWEN_38_27B_256K_MODEL} — same model, 256K chat`, "Qwen/Qwen3.5-9B-Base — cheaper small base model", "Qwen/Qwen3.5-35B-A3B-Base — stronger Qwen MoE base", "custom", ]); if (modelChoice === "custom") model = (await ctx.ui.input("Tinker model id", DEFAULT_MODEL))?.trim() || DEFAULT_MODEL; else model = modelChoice?.split(" — ")[0] || DEFAULT_MODEL; } if (ctx.hasUI && !successMetric) { successMetric = (await ctx.ui.input("What should improve?", "e.g. held-out exact match, support response quality, benchmark score"))?.trim() || "Define before scaling beyond a smoke test."; } if (!dataFileArg) { sendReport("Tinker init", "Usage: `/tinker init data/train.jsonl --model thinkingmachines/Inkling --metric 'held-out accuracy'`", "warning"); return; } const dataFile = path.resolve(ctx.cwd, dataFileArg); if (!existsSync(dataFile)) { sendReport("Tinker init", `Data file not found: ${dataFile}`, "error"); return; } model = model || DEFAULT_MODEL; const force = options.force === true; const stamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19); const logPath = String(options.log ?? `logs/sft-${stamp}`); const maxSteps = String(options.steps ?? "20"); const batchSize = String(options["batch-size"] ?? "8"); const learningRate = String(options.lr ?? options["learning-rate"] ?? "2e-4"); const testSize = String(options["test-size"] ?? "0"); const maxLength = String(options["max-length"] ?? "32768"); const files = [ { rel: "README.md", content: makeProjectReadme({ model, dataFile, logPath, successMetric }) }, { rel: "train_sft.py", content: makeSftScript({ dataFile, model, logPath, maxSteps, batchSize, learningRate, testSize, maxLength }) }, { rel: "eval_checkpoint.py", content: makeEvalScript() }, { rel: "tinker.yaml", content: `task: sft\ndata: ${dataFile}\nmodel: ${model}\nlog_path: ${logPath}\nmax_steps: ${maxSteps}\nbatch_size: ${batchSize}\nlearning_rate: ${learningRate}\nsuccess_metric: ${successMetric}\n` }, { rel: path.join("notes", "plan.md"), content: `# Tinker SFT plan\n\n## Goal\n\nFine-tune ${model} on ${dataFile}.\n\n## Success metric\n\n${successMetric || "TODO"}\n\n## First checks\n\n- [ ] /tinker validate ${dataFileArg} --model ${model}\n- [ ] python train_sft.py max_steps=2\n- [ ] inspect metrics and decoded examples\n- [ ] define baseline eval before scale-up\n` }, ]; const written: string[] = []; for (const file of files) { const target = path.join(ctx.cwd, file.rel); if (existsSync(target) && !force) { sendReport("Tinker init", `${target} already exists. Re-run with --force to overwrite.`, "warning"); return; } await mkdir(path.dirname(target), { recursive: true }); await writeFile(target, file.content); written.push(file.rel); } sendReport("Tinker project initialized", [ `Wrote ${written.map((x) => `\`${x}\``).join(", ")}.`, "", "Recommended next steps:", "```text", `/tinker validate ${dataFileArg} --model ${model}`, "/tinker smoke train_sft.py --yes", `/tinker monitor ${logPath}`, "```", ].join("\n"), "success"); return; }
if (subcommand === "sft") { const { positional, options } = parseOptions(rest); const dataFileArg = positional[0]; if (!dataFileArg) { sendReport("Tinker SFT scaffold", "Usage: `/tinker sft data/train.jsonl --model thinkingmachines/Inkling --steps 20`", "warning"); return; } const dataFile = path.resolve(ctx.cwd, dataFileArg); if (!existsSync(dataFile)) { sendReport("Tinker SFT scaffold", `Data file not found: ${dataFile}`, "error"); return; } const force = options.force === true; const model = String(options.model ?? DEFAULT_MODEL); const maxSteps = String(options.steps ?? options.max_steps ?? "20"); const batchSize = String(options["batch-size"] ?? "8"); const learningRate = String(options.lr ?? options["learning-rate"] ?? "2e-4"); const testSize = String(options["test-size"] ?? "0"); const maxLength = String(options["max-length"] ?? "32768"); const stamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19); const logPath = String(options.log ?? `logs/sft-${stamp}`);
const files = [ { rel: "train_sft.py", content: makeSftScript({ dataFile, model, logPath, maxSteps, batchSize, learningRate, testSize, maxLength }) }, { rel: "eval_checkpoint.py", content: makeEvalScript() }, { rel: "tinker.yaml", content: `task: sft\ndata: ${dataFile}\nmodel: ${model}\nlog_path: ${logPath}\nmax_steps: ${maxSteps}\nbatch_size: ${batchSize}\nlearning_rate: ${learningRate}\n` }, { rel: path.join("notes", "plan.md"), content: `# Tinker SFT plan\n\n## Goal\n\nFine-tune ${model} on ${dataFile}.\n\n## Baseline eval\n\nTODO: define success metric before scaling the run.\n\n## Smoke test\n\nRun a tiny job first:\n\n\`\`\`bash\npython train_sft.py max_steps=2\n\`\`\`\n\nThen inspect metrics and decoded samples before increasing steps.\n` }, ];
const written: string[] = []; for (const file of files) { const target = path.join(ctx.cwd, file.rel); if (existsSync(target) && !force) { sendReport("Tinker SFT scaffold", `${target} already exists. Re-run with --force to overwrite.`, "warning"); return; } await mkdir(path.dirname(target), { recursive: true }); await writeFile(target, file.content); written.push(file.rel); }
sendReport("Tinker SFT scaffold created", [ `Wrote ${written.map((x) => `\`${x}\``).join(", ")}.`, "", "Next commands:", "```bash", isInklingModel(model) ? "uv pip install -U 'tinker-cookbook[inkling]'" : "uv pip install -U tinker-cookbook", "python train_sft.py max_steps=2", "# inspect logs, then scale up:", `python train_sft.py max_steps=${maxSteps}`, "```", "", "Tip: use `/skill:tinker-research` for experiment planning/monitoring before spending real compute.", ].join("\n"), "success"); return; }
if (subcommand === "smoke") { const { positional, options } = parseOptions(rest); const script = path.resolve(ctx.cwd, positional[0] ?? "train_sft.py"); if (!existsSync(script)) { sendReport("Tinker smoke test", `Script not found: ${script}`, "error"); return; } const ok = options.yes === true || !ctx.hasUI ? true : await ctx.ui.confirm("Run Tinker smoke test?", "This may create a real Tinker training client and incur small API usage. Run `python train_sft.py max_steps=2`?"); if (!ok) { sendReport("Tinker smoke test", "Cancelled.", "warning"); return; } try { ctx.ui.setStatus("tinker", "Tinker smoke running"); const { stdout, stderr } = await execFile("python3", [script, "max_steps=2"], { cwd: ctx.cwd, timeout: Number(options.timeout ?? 1_800_000), maxBuffer: 10 * 1024 * 1024, }); const combined = `${stdout}\n${stderr}`.trim(); const logMatch = combined.match(/log_path='([^']+)'|log_path=([^,\s)]+)/); const logDir = logMatch ? path.resolve(ctx.cwd, (logMatch[1] ?? logMatch[2] ?? "").replace(/^['\"]|['\"]$/g, "")) : undefined; const metrics = logDir ? latestMetrics(path.join(logDir, "metrics.jsonl")) : undefined; await patchWizardState(ctx.cwd, { smokeAt: Date.now(), smokeLogDir: logDir, logPath: logDir ? rel(ctx.cwd, logDir) : undefined }); sendReport("Tinker smoke test passed", [ "The 2-step command completed.", metrics ? `\n## Latest metrics\n${metrics}` : "", `\n## Output tail\n\`\`\`text\n${combined.split(/\n/).slice(-80).join("\n")}\n\`\`\``, ].filter(Boolean).join("\n"), "success"); } catch (error: any) { sendReport("Tinker smoke test failed", [ error?.message ?? String(error), error?.stdout ? `\n## stdout\n\`\`\`text\n${String(error.stdout).split(/\n/).slice(-80).join("\n")}\n\`\`\`` : "", error?.stderr ? `\n## stderr\n\`\`\`text\n${String(error.stderr).split(/\n/).slice(-80).join("\n")}\n\`\`\`` : "", "\nUse `/skill:tinker-debug` with the error output if this is not obvious.", ].filter(Boolean).join("\n"), "error"); } finally { ctx.ui.setStatus("tinker", undefined); } return; }
if (subcommand === "monitor") { const logDirArg = rest[0]; if (!logDirArg) { sendReport("Tinker monitor", "Usage: `/tinker monitor logs/my-run` or `/tinker monitor --stop`", "warning"); return; } if (logDirArg === "--stop" || logDirArg === "stop") { if (monitorInterval) clearInterval(monitorInterval); monitorInterval = undefined; ctx.ui.setWidget("tinker-monitor", undefined); ctx.ui.setStatus("tinker", undefined); sendReport("Tinker monitor", "Stopped.", "success"); return; } const logDir = path.resolve(ctx.cwd, logDirArg); if (monitorInterval) clearInterval(monitorInterval); const update = () => { const lines = monitorSummary(logDir); ctx.ui.setWidget("tinker-monitor", lines); ctx.ui.setStatus("tinker", `Tinker: ${lines[1] ?? "monitoring"}`); }; update(); monitorInterval = setInterval(update, 5000); sendReport("Tinker monitor", `Monitoring \`${logDir}\`. Use \`/tinker monitor --stop\` to clear the widget.`, "success"); return; }
if (subcommand === "checkpoints") { const logDirArg = rest[0]; if (!logDirArg) { sendReport("Tinker checkpoints", "Usage: `/tinker checkpoints logs/my-run`", "warning"); return; } const logDir = path.resolve(ctx.cwd, logDirArg); let checkpoints: CheckpointRecord[] = []; try { checkpoints = readCheckpoints(logDir).filter((c) => c.sampler_path || c.state_path); } catch (error) { sendReport("Tinker checkpoints", `Could not read checkpoints.jsonl: ${error instanceof Error ? error.message : String(error)}`, "error"); return; } if (checkpoints.length === 0) { sendReport("Tinker checkpoints", `No checkpoints found in \`${path.join(logDir, "checkpoints.jsonl")}\`.`, "warning"); return; } const lines = checkpoints.map((c, i) => `${i + 1}. ${c.name ?? "checkpoint"}${c.final ? " (final)" : ""}${c.batch !== undefined ? ` batch=${c.batch}` : ""}\n sampler: ${c.sampler_path ?? "none"}\n state: ${c.state_path ?? "none"}`); const samplerCheckpoints = checkpoints.filter((c) => c.sampler_path); const latestSampler = samplerCheckpoints.slice(-1)[0]?.sampler_path; if (latestSampler) await patchWizardState(ctx.cwd, { checkpointPath: latestSampler, logPath: rel(ctx.cwd, logDir) }); if (ctx.hasUI && samplerCheckpoints.length > 0) { const choice = await ctx.ui.select("Register sampler checkpoint as Pi model?", ["just list", ...samplerCheckpoints.map((c) => `${c.name ?? "checkpoint"} — ${c.sampler_path}`)]); if (choice && choice !== "just list") { const record = samplerCheckpoints.find((c) => choice.endsWith(String(c.sampler_path))); if (record?.sampler_path) { const alias = `${path.basename(logDir)}-${record.name ?? "checkpoint"}`.replace(/[^a-zA-Z0-9._-]+/g, "-"); state.checkpoints = state.checkpoints.filter((m) => m.id !== record.sampler_path && m.name !== alias); const wizard = await readWizardState(ctx.cwd); state.checkpoints.push(checkpointRegistration({ id: record.sampler_path, name: alias, baseModel: wizard?.model })); await saveState(state); await patchWizardState(ctx.cwd, { checkpointPath: record.sampler_path, registeredModel: alias }); registerTinkerProvider(); sendReport("Tinker checkpoint registered", `Registered \`${alias}\` for \`${record.sampler_path}\`. Use \`/model\` to select it.`, "success"); return; } } } sendReport("Tinker checkpoints", lines.join("\n\n")); return; }
if (subcommand === "status") { const logDir = rest[0] ? path.resolve(ctx.cwd, rest[0]) : undefined; const sections: string[] = []; if (await commandExists("tinker")) { try { const { stdout, stderr } = await execFile("tinker", ["run", "list", "--limit", "10"], { timeout: 30_000 }); sections.push(`## Recent Tinker runs\n\n\`\`\`text\n${(stdout || stderr).trim()}\n\`\`\``); } catch (error: any) { sections.push(`## Recent Tinker runs\n\nCould not run \`tinker run list\`: ${error?.message ?? String(error)}`); } } else { sections.push("## Recent Tinker runs\n\n`tinker` CLI not found on PATH."); } if (logDir) { const metrics = latestMetrics(path.join(logDir, "metrics.jsonl")); sections.push(`## Latest metrics from ${logDir}\n\n${metrics ?? "No metrics.jsonl found."}`); } sendReport("Tinker status", sections.join("\n\n")); return; }
if (subcommand === "use") { const { positional, options } = parseOptions(rest); if (options.list === true) { const lines = state.checkpoints.length ? state.checkpoints.map((m) => `- ${m.name}: \`${m.id}\` (${m.contextWindow} ctx, max ${m.maxTokens})`) : ["No Tinker checkpoints registered yet."]; sendReport("Registered Tinker checkpoint models", lines.join("\n")); return; } if (options.remove) { const key = String(options.remove); const before = state.checkpoints.length; state.checkpoints = state.checkpoints.filter((m) => m.id !== key && m.name !== key); await saveState(state); registerTinkerProvider(); sendReport("Tinker checkpoint removed", before === state.checkpoints.length ? `No checkpoint matched ${key}.` : `Removed ${key}.`, "success"); return; } const checkpoint = positional[0]; if (!checkpoint || !checkpoint.startsWith("tinker://")) { sendReport("Register Tinker checkpoint", "Usage: `/tinker use tinker://.../sampler_weights/... [alias]`", "warning"); return; } const alias = String(options.alias ?? positional[1] ?? `tinker-${state.checkpoints.length + 1}`); const baseModel = String(options["base-model"] ?? options.model ?? "") || undefined; const inkling = isInklingModel(baseModel); const contextWindow = Number(options.context ?? (inkling ? 65_536 : 32_768)); const maxTokens = Number(options["max-tokens"] ?? (inkling ? 16_384 : 4_096)); state.checkpoints = state.checkpoints.filter((m) => m.id !== checkpoint && m.name !== alias); state.checkpoints.push(checkpointRegistration({ id: checkpoint, name: alias, baseModel, contextWindow, maxTokens })); await saveState(state); await patchWizardState(ctx.cwd, { checkpointPath: checkpoint, registeredModel: alias }); registerTinkerProvider(); sendReport("Tinker checkpoint registered", [ `Registered \`${alias}\` as provider/model \`tinker/${checkpoint}\`.`, "", "Use `/model` to select it. Pi uses Tinker's beta Anthropic-compatible endpoint for tool use, image input, streaming thinking, and Inkling effort controls; it is still intended for testing rather than production serving.", "", `Saved registrations in \`${STATE_PATH}\`.`, ].join("\n"), "success"); return; }
sendReport("Unknown Tinker command", `Unknown subcommand: ${subcommand}\n\nRun \`/tinker help\` for usage.`, "warning"); }, });}