Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940#!/usr/bin/env bun// machine: run, use interactively, and grade the machines in a workspace directory.// Machines live at <workspace>/machines/<name>/machine.ts; runs and sessions// are stored under <workspace>/.machine/.
import { readFile, writeFile } from "node:fs/promises";import path from "node:path";import { parseArgs } from "node:util";import { pathToFileURL } from "node:url";import { agreement, contamination } from "./agree.ts";import { compareTrials, summarize, trialStats, type Comparison, type TrialStats } from "./compare.ts";import { judge, type Grader } from "./evaluator.ts";import { jev } from "./graders/jev.ts";import type { Harness } from "./harness.ts";import { DEFAULT_TINKER_JUDGE, defaultHarness, lazyHarness } from "./harnesses/select.ts";import { INSTRUCTIONS_FILE, withEvaluation, withInstructions, withModel, type Machine } from "./machine.ts";import { decide, failureReport, optimizer, parseProposal } from "./optimizer.ts";import { Budget, FREE, isFreeModel, tinkerPrice } from "./pricing.ts";import { checkoutRevision, descendants, git, loadMachineFrom, machineChangedSince } from "./revision.ts";import { exportRuns } from "./export.ts";import { converse } from "./converse.ts";import { hasCaseFile, loadCases, type Case } from "./cases.ts";import { comparePaired, mapLimit, runMatrix } from "./trials.ts";import { CommitRefused, formatReport, machine as sdkMachine } from "./sdk.ts";import { formatSpecialization } from "./specialize.ts";import { findVersion, versionFile } from "./versions.ts";import { interactive } from "./interactive.ts";import { fileEvents } from "./events.ts";import { serveDashboard } from "./dashboard.ts";import { nameSegments } from "./store.ts";import type { Run } from "./types.ts";import { createWorkspace, type Workspace } from "./workspace.ts";
const USAGE = `Usage: machine dashboard Live view of runs, sessions, and training in this workspace (--port, default 4242; --judge-model and --budget for grading from the page, default $1 per request; --host 0.0.0.0 to reach it from other machines, with a printed token) machine <machine> Use the machine interactively in Pi (--continue resumes, --eval grades each exchange) machine run <machine> [input...] Run once and print the reply; without input, open it in Pi machine runs <machine> List recent runs, newest first machine eval <machine> <run-id> Grade a stored run against the machine's rubric machine suite <machine> Run and grade every input in machines/<machine>/cases.json machine optimize <machine> Rewrite the machine's instructions.md from its failures; keep a winning rewrite machine agree <machine> <criterion> Measure another grader against the machine's own on stored runs machine compare <machine> Compare the working tree's machine with a git revision of it machine tree <machine> Show a machine, its children, and what each inherits machine export <machine> Write graded runs as chat JSONL for fine-tuning machine train <machine> Fine-tune the live model on good runs; report the candidate against the live version on holdout.json and leave it pending (never changes the machine) machine specialize <machine> --focus <area> An author model writes a child of the machine for the focus (new criteria with graders, instructions) and rehearsal cases (cases.json, holdout.json), validated machine propose <machine> --model <id> Measure another model against the live version on holdout.json, as a candidate machine commit <machine> [candidate] Make a pending candidate (default: the newest) the live version, if it passed the promotion gate (--force to override) machine revert <machine> Make the previous live version live again machine versions <machine> List the machine's versions; the first call records the current one as v1 machine converse <machine> [case] Run one conversation with the machine's counterpart and print it as it happens (case: a number in --cases, default 1, or the text of a new case)
Options: --harness tinker|pi|openrouter|opencode|fx|openai|echo Agent loop (default: $MACHINE_HARNESS, else pi for openrouter/ models and tinker otherwise) --model <id> Override the machine's model --judge-model <id> Model for judge graders (default: $MACHINE_JUDGE_MODEL; on tinker, thinkingmachines/Inkling-Small; otherwise the run's model) --budget <usd>|none Stop spending at this many dollars (default: 1 on tinker and pi, none elsewhere) --effort <level> Reasoning effort for runs: none, low, medium, high, xhigh, or 0-0.99 (default: the model's) --judge-effort <level> Reasoning effort for judges (default: medium on tinker, else the model's) --price-as <model> Price a fine-tuned tinker:// checkpoint as this base model --concurrency <n> Runs at once in suite, compare, and optimize (default: 4) --base-url <url> Endpoint for --harness openai ($OPENAI_BASE_URL) --eval Grade each run after it finishes --workspace <dir> Workspace directory (default: current directory) --name <name> Workspace name (default: $MACHINE_WORKSPACE or local) --limit <n> Number of runs to list (default: 20; agree uses all stored runs) --json Print runs as JSON --rounds <n> In optimize, rewrite rounds (default: 1) --optimizer-model <id> In optimize, the optimizer's model (default: the machine's) --with jev|judge In agree, the grader to measure (default: jev) --question <text> In agree, ask Jev this instead of the criterion's description --ref <git-ref> In compare, the revision to compare against (default: HEAD) --holdout <file>|none In optimize, held-out cases (default: holdout.json if present) --cases <file> In suite, compare, and converse, a case file in the machine's directory (default: cases.json); in export, only runs made for these cases --trials <n> In suite and compare, run each case n times (default: 1) --tree In suite and export, include the machine's descendants --min-z <z> In optimize, standard errors a rewrite must win by (default: 2) --min-score <x> In export, the lowest scorecard total to include (default: 1) --best <k> In export, keep only each case's best k runs (rejection sampling) --since <time> In export, only runs started at or after this ISO time --per-turn In export, one example per assistant message, and every turn of a conversation --out <file> In export, the JSONL file to write (default: stdout) --from <machine> In export, write that machine's good runs as this machine's training data (its instructions, its transformReasoning), e.g. a parent's runs for a child --no-reasoning In export, leave the model's reasoning out (default: included, through the machine's transformReasoning) --samples <n> In train, new runs per case of cases.json before training (default: 0, stored runs) --best-of <k> In train, at most k good runs per case (default: 2) --epochs <n>, --lr <x> In train, SFT epochs (default: 2) and learning rate (default: the cookbook's) --batch-size <n> In train, examples per training step (default: 8) --any-model In train, also train on good runs by other models (distillation) --force In commit, commit a candidate that did not pass the gate; in specialize, replace an existing child directory --focus <area> In specialize, what the child specializes in (required) --author-model <id> In specialize, the model that writes the child (default: thinkingmachines/Inkling); --model is the child's model (default: the parent's) --slug <name>, --train-cases <n>, --holdout-cases <n> In specialize, the child's directory name (default: the author's choice) and how many cases to keep (default: 20 and 8) --reuse-cases In specialize, rewrite only the child (with --slug), keeping its cases -v, --verbose In suite and compare, show the reason for each imperfect score
In train, --trials is runs per held-out case for each version (default: 2), --min-score the lowest score totrain on, --min-z the margin the candidate must win by, and --budget caps sampling, grading, training, and thecomparison together. Training runs scripts/train_sft.py with $MACHINE_PYTHON (a Python with tinker-cookbook).`;
async function main(argv: string[]): Promise<number> { const { values, positionals } = parseArgs({ args: argv, allowPositionals: true, options: { harness: { type: "string" }, model: { type: "string" }, "judge-model": { type: "string" }, "base-url": { type: "string" }, eval: { type: "boolean", default: false }, workspace: { type: "string", default: process.cwd() }, name: { type: "string" }, limit: { type: "string" }, trials: { type: "string" }, ref: { type: "string", default: "HEAD" }, with: { type: "string", default: "jev" }, rounds: { type: "string", default: "1" }, cases: { type: "string" }, holdout: { type: "string" }, "optimizer-model": { type: "string" }, question: { type: "string" }, budget: { type: "string" }, "price-as": { type: "string" }, effort: { type: "string" }, "judge-effort": { type: "string" }, concurrency: { type: "string", default: "4" }, tree: { type: "boolean", default: false }, "min-z": { type: "string", default: "2" }, "min-score": { type: "string", default: "1" }, best: { type: "string" }, since: { type: "string" }, "per-turn": { type: "boolean", default: false }, out: { type: "string" }, "no-reasoning": { type: "boolean", default: false }, from: { type: "string" }, json: { type: "boolean", default: false }, continue: { type: "boolean", short: "c", default: false }, port: { type: "string" }, host: { type: "string" }, samples: { type: "string" }, "best-of": { type: "string" }, epochs: { type: "string" }, lr: { type: "string" }, "any-model": { type: "boolean", default: false }, force: { type: "boolean", default: false }, focus: { type: "string" }, "author-model": { type: "string" }, slug: { type: "string" }, "train-cases": { type: "string" }, "holdout-cases": { type: "string" }, "reuse-cases": { type: "boolean", default: false }, "batch-size": { type: "string" }, verbose: { type: "boolean", short: "v", default: false }, help: { type: "boolean", short: "h", default: false }, }, }); if (positionals[0] === "dashboard") { const workspaceDir = path.resolve(values.workspace); const port = Number(values.port ?? 4242); if (!Number.isInteger(port) || port < 0 || port > 65535) throw new Error("--port must be a port number."); const gradeBudgetUsd = values.budget === undefined || values.budget === "none" ? undefined : Number(values.budget); if (gradeBudgetUsd !== undefined && !(gradeBudgetUsd >= 0)) throw new Error("--budget must be a dollar amount."); const host = values.host ?? "127.0.0.1"; const server = await serveDashboard({ workspaceDir, port, host, judgeModel: values["judge-model"], gradeBudgetUsd }); if (server.token) { // Reachable from other machines: print an address they can use, with the token. const { networkInterfaces } = await import("node:os"); const lan = Object.values(networkInterfaces()).flat().filter((a) => a && a.family === "IPv4" && !a.internal).map((a) => a!.address); const hosts = host === "0.0.0.0" ? lan : [host]; console.log(`machine dashboard on ${host}:${server.port}. Open once with the token; your browser keeps it:`); for (const h of hosts) console.log(` http://${h}:${server.port}/?token=${server.token}`); console.log("Anyone with this link can read runs and spend on grading. Ctrl+C to stop."); } else { console.log(`machine dashboard: ${server.url} (Ctrl+C to stop)`); } await new Promise(() => {}); } // `machine <name>` opens the machine; anything else is `machine <command> <name> ...`. const interactiveMode = positionals.length === 1 && !COMMANDS.has(positionals[0]); const [command, machineName, ...rest] = interactiveMode ? ["interactive", positionals[0]] : positionals; if (values.help || !command || !machineName) { console.log(USAGE); return values.help ? 0 : 1; }
const workspaceDir = path.resolve(values.workspace); if (VERSION_COMMANDS.has(command)) return versionCommand(command, machineName, rest, values, workspaceDir); const machine = await loadMachineFrom(workspaceDir, machineName); // Runs of the definition on disk record its version when it is in the machine's history. const version = values.model ? undefined : await recordedVersion(workspaceDir, machine); const kind = values.harness ?? process.env.MACHINE_HARNESS ?? defaultHarness(values.model ?? machine.model); // A conversation on Pi is Pi's own terminal UI, recorded by the machine like `machine <name>`. if (command === "interactive" || (command === "run" && rest.length === 0)) { const workspace = createWorkspace({ name: values.name ?? process.env.MACHINE_WORKSPACE ?? "local", root: path.join(workspaceDir, ".machine"), harness: lazyHarness(values.harness ?? process.env.MACHINE_HARNESS ?? "pi", { priceAs: values["price-as"] }), judgeModel: values["judge-model"] ?? process.env.MACHINE_JUDGE_MODEL ?? DEFAULT_TINKER_JUDGE, judgeEffort: effortValue(values["judge-effort"]), events: fileEvents(path.join(workspaceDir, ".machine", "events.jsonl")), }); return interactive(machine, workspace, { model: values.model, version, continue: values.continue, evaluate: values.eval, stateDir: path.join(workspaceDir, ".machine", "pi"), price: (model) => (isFreeModel(model) ? FREE : tinkerPrice(model.replace(/^tinker\//, ""), values["price-as"])), }); } const harness = lazyHarness(kind, { baseURL: values["base-url"], priceAs: values["price-as"] }); const budget = makeBudget(values.budget, kind); const workspace = createWorkspace({ name: values.name ?? process.env.MACHINE_WORKSPACE ?? "local", root: path.join(workspaceDir, ".machine"), harness, judgeModel: values["judge-model"] ?? process.env.MACHINE_JUDGE_MODEL ?? (kind === "tinker" || kind === "pi" ? DEFAULT_TINKER_JUDGE : undefined), budget, events: fileEvents(path.join(workspaceDir, ".machine", "events.jsonl")), reasoningEffort: effortValue(values.effort), judgeEffort: effortValue(values["judge-effort"] ?? (kind === "tinker" ? "medium" : undefined)), }); const runOptions = { model: values.model, evaluate: values.eval, version }; try { return await dispatch(command, rest, machine, workspace, runOptions, values, workspaceDir, harness); } finally { await harness.close?.(); if (budget && (budget.tokens.input || budget.tokens.output)) console.error(`budget: ${budget.summary()}`); }}
async function dispatch( command: string, rest: string[], machine: Machine, workspace: Workspace, runOptions: { model?: string; evaluate?: boolean; version?: string }, values: { json: boolean; limit?: string; verbose: boolean; trials?: string; ref: string; with: string; question?: string; "judge-model"?: string; "judge-effort"?: string; rounds: string; "optimizer-model"?: string; cases?: string; holdout?: string; concurrency: string; tree: boolean; "min-z": string; "min-score": string; best?: string; since?: string; "per-turn": boolean; out?: string; "no-reasoning": boolean; from?: string; }, workspaceDir: string, harness: Harness,): Promise<number> {
switch (command) { case "run": { const input = rest.join(" ").trim(); if (!input) throw new Error("run needs an input."); printRun(await workspace.run(machine, input, runOptions), values.json); return 0; } case "runs": { const runs = await workspace.store.listRuns(machine.name, { limit: Number(values.limit ?? 20) }); if (values.json) console.log(JSON.stringify(runs, null, 2)); else for (const run of runs) console.log(`${run.id} ${scoreLabel(run)} ${run.conversation ? `[turn ${run.conversation.turn}${run.conversation.final ? ", last" : ""}] ` : ""}${preview(run.input, 60)}`); return 0; } case "suite": { const trials = trialCount(values.trials ?? "1"); const concurrency = positiveInt(values.concurrency, "--concurrency"); const machines = values.tree ? [machine, ...(await descendants(workspaceDir, machine.name))] : [machine]; const results: Array<{ machine: Machine; runsByCase: Run[][] }> = []; for (const m of machines) { let cases: Case[]; try { cases = await loadCases(workspaceDir, m, values.cases); } catch (error) { if (values.tree) { console.log(`${m.name}: skipped (${error instanceof Error ? error.message : error})`); continue; } throw error; } if (values.tree) console.log(`\n== ${m.name}`); const [runsByCase] = await runMatrix(workspace, [{ machine: m, model: runOptions.model, version: m === machine ? runOptions.version : undefined }], cases, trials, concurrency, (i, [runs]) => { console.log(`${String(i + 1).padStart(3)}. ${runs.length ? statLabel(trialStats(runs)) : "error"} ${preview(cases[i].input, 50)}`); if (values.verbose) printFailures(runs, ""); }); printSummary(cases.length, trials, runsByCase); results.push({ machine: m, runsByCase }); } if (values.tree && results.length > 1) printTreeSummary(machine, results); return 0; } case "compare": { if (rest.length > 0) throw new Error("compare takes one machine; choose the revision to compare against with --ref."); const ref = values.ref; const checkout = await checkoutRevision(workspaceDir, ref); try { const old = await loadMachineFrom(checkout.workspaceDir, machine.name); // Grade both versions with today's rubric and evaluator, so the comparison // measures the change in behavior and nothing else. const baseline = withEvaluation(old, machine.rubric, machine.evaluator); if (!(await machineChangedSince(workspaceDir, machine.name, checkout.commit))) { console.error(`note: machines/${machine.name} has no changes since ${ref} (${checkout.short}); both sides run the same machine, so differences are noise.`); } if (JSON.stringify(old.rubric) !== JSON.stringify(machine.rubric)) { console.error(`note: the rubric changed since ${ref}; both sides are graded with the current one.`); } const cases = await loadCases(workspaceDir, machine, values.cases); const trials = trialCount(values.trials ?? "1"); const head = await git(workspaceDir, ["rev-parse", "--short", "HEAD"]); const labelA = `${ref} (${checkout.short})`; const labelB = "working tree"; const revisionA = checkout.short; const revisionB = `${head.stdout.trim()}+working-tree`; const width = Math.max(labelA.length, labelB.length, 14); console.log(`${"".padStart(4)} ${labelA.padStart(width)} ${labelB.padStart(width)}`); // Each case's trials of both versions are scheduled together, so both see the same conditions (rate limits, provider load). const [runsA, runsB] = await runMatrix( workspace, [ { machine: baseline, model: runOptions.model, revision: revisionA }, { machine, model: runOptions.model, revision: revisionB, version: runOptions.version }, ], cases, trials, positiveInt(values.concurrency, "--concurrency"), (i, [a, b]) => { const cell = (runs: Run[]) => (runs.length ? statLabel(trialStats(runs)) : "error"); const mark = a.length && b.length ? (trialStats(b).mean > trialStats(a).mean ? "+" : trialStats(b).mean < trialStats(a).mean ? "-" : " ") : "?"; console.log(`${String(i + 1).padStart(3)}. ${cell(a).padStart(width)} ${cell(b).padStart(width)} ${mark} ${preview(cases[i].input, 45)}`); if (values.verbose) { printFailures(a, `${labelA} `); printFailures(b, `${labelB} `); } }, ); printComparison(labelA, labelB, comparePaired(cases, runsA, runsB), trials); } finally { await checkout.close(); } return 0; } case "optimize": { await runOptimize(machine, workspace, workspaceDir, runOptions, values); return 0; } case "tree": { printTree(machine, await descendants(workspaceDir, machine.name)); return 0; } case "export": { // --from <machine>: that machine's good runs, written as this machine's training data. const source = values.from ? await loadMachineFrom(workspaceDir, values.from) : undefined; if (source && values.tree) throw new Error("export takes --from or --tree, not both."); const machines = source ? [source] : values.tree ? [machine, ...(await descendants(workspaceDir, machine.name))] : [machine]; const minScore = Number(values["min-score"]); if (!(minScore >= 0 && minScore <= 1)) throw new Error("--min-score must be between 0 and 1."); const bestOf = values.best === undefined ? undefined : positiveInt(values.best, "--best"); if (values.since && Number.isNaN(Date.parse(values.since))) throw new Error("--since must be an ISO time, e.g. 2026-10-11T18:00:00Z."); const cases = values.cases ? (await loadCases(workspaceDir, machine, values.cases)).map((c) => c.input) : undefined; const { lines, counts } = await exportRuns(workspace.store, machines, { minScore, reasoning: !values["no-reasoning"], as: source ? machine : undefined, bestOf, since: values.since ? new Date(values.since).toISOString() : undefined, cases, perTurn: values["per-turn"], }); const text = lines.map((l) => JSON.stringify(l)).join("\n") + (lines.length ? "\n" : ""); if (values.out) await writeFile(values.out, text, { mode: 0o600 }); else process.stdout.write(text); for (const [name, c] of Object.entries(counts)) console.error(`${name}: ${c.kept} of ${c.graded} graded runs at or above ${minScore}`); console.error(`${lines.length} examples${values.out ? ` written to ${values.out}` : ""}. They hold prompts and outputs: inspect before training or sharing.`); return 0; } case "agree": { const criterionId = rest[0]; if (!criterionId || !(criterionId in machine.rubric)) { throw new Error(`agree needs a criterion of ${machine.name}: ${Object.keys(machine.rubric).join(", ")}.`); } await runAgreement(machine, criterionId, workspace, harness, values); return 0; } case "converse": { if (!machine.counterpart) throw new Error(`${machine.name} has no counterpart; give its machine.ts a counterpart to converse with.`); const arg = rest.join(" ").trim(); const cases = await loadCases(workspaceDir, machine, values.cases); const index = arg === "" ? 1 : /^\d+$/.test(arg) ? Number(arg) : undefined; if (index !== undefined && !(index >= 1 && index <= cases.length)) throw new Error(`Case ${index} is out of range: there are ${cases.length}.`); const caseInput = index === undefined ? arg : cases[index - 1].input; const other = machine.counterpart.machine.name; console.log(`${machine.name} with ${other}${index === undefined ? "" : `, case ${index}`}: ${preview(caseInput, 100)}\n`); const result = await converse(workspace, machine, caseInput, { ...runOptions, evaluateCounterpart: runOptions.evaluate, onTurn: (side, run) => printTurn(side === "agent" ? machine.name : other, run), }); console.log(`${result.ended === "done" ? "Done" : "Stopped at the turn limit"} after ${result.agentRuns.length} turns · conversation ${result.id}`); for (const run of [result.run, ...(runOptions.evaluate ? [result.counterpartRuns.at(-1)!] : [])]) { if (!run.scorecard) continue; console.log(`\n${run.machine} ${run.id}: score ${run.scorecard.total}`); for (const [id, c] of Object.entries(run.scorecard.criteria)) console.log(` ${id.padEnd(20)} ${c.score.toFixed(2)} ${c.reason}`); } return 0; } case "eval": { const id = rest[0]; if (!id) throw new Error("eval needs a run id."); const run = await workspace.store.getRun(machine.name, id); if (!run) throw new Error(`No run ${id} for ${machine.name}.`); printRun(await workspace.evaluate(machine, run), values.json); return 0; } default: console.log(USAGE); return 1; }}
/** * Rewrite a machine's instructions from its failures. Each round runs the * suite, gives the failures to the optimizer machine, runs the suite with the * proposal, and keeps the proposal only if it scores higher. A kept proposal * is written to instructions.md and left uncommitted for review. */async function runOptimize( machine: Machine, workspace: Workspace, workspaceDir: string, runOptions: { model?: string }, values: { trials?: string; rounds: string; "optimizer-model"?: string; verbose: boolean; holdout?: string; concurrency: string; "min-z": string },) { const concurrency = positiveInt(values.concurrency, "--concurrency"); const minZ = Number(values["min-z"]); if (!(minZ >= 0)) throw new Error("--min-z must be a non-negative number."); const runCases = async (m: Machine, inputs: Case[], revision: string) => (await runMatrix(workspace, [{ machine: m, model: runOptions.model, revision }], inputs, trials, concurrency))[0]; const file = path.join(workspaceDir, "machines", ...nameSegments(machine.name, "machine name"), INSTRUCTIONS_FILE); let onDisk: string; try { onDisk = (await readFile(file, "utf8")).trim(); } catch { throw new Error(`${machine.name}'s instructions are not an optimization target: move them to ${file} and load them with instructionsFrom(import.meta.url).`); } if (onDisk !== machine.ownInstructions) throw new Error(`${machine.name}'s instructions do not come from ${file}.`);
const cases = await loadCases(workspaceDir, machine); const trials = trialCount(values.trials ?? "1"); const rounds = trialCount(values.rounds); // Held-out cases judge each proposal but are never shown to the optimizer. const machineDir = path.dirname(file); const holdoutFile = values.holdout === "none" ? undefined : (values.holdout ?? ((await hasCaseFile(workspaceDir, machine, "holdout.json")) ? "holdout.json" : undefined)); if (holdoutFile && path.resolve(machineDir, holdoutFile) === path.join(machineDir, "cases.json")) { throw new Error("The held-out file must not be cases.json, which the optimizer learns from."); } const holdout = holdoutFile ? await loadCases(workspaceDir, machine, holdoutFile) : undefined; if (holdout) console.log(`held-out: ${holdout.length} cases from ${holdoutFile}, never shown to the optimizer`); else console.log("warning: no held-out set (machines/<name>/holdout.json); a rewrite may win by memorizing its failures"); const head = (await git(workspaceDir, ["rev-parse", "--short", "HEAD"])).stdout.trim() || "unversioned"; const optimizerMachine = withModel(optimizer, values["optimizer-model"] ?? runOptions.model ?? machine.model);
let current = machine; console.log(`baseline: ${cases.length} cases × ${trials} trials`); let currentRuns = await runCases(current, cases, `${head}+working-tree`); let currentHoldout: Run[][] | undefined; let kept = 0; for (let round = 1; round <= rounds; round++) { const before = summarize(currentRuns); console.log(`\nround ${round}: current instructions score ${before.mean.toFixed(3)} (pass rate ${pct(before.passRate)})`); if (before.passRate === 1) { console.log(" every run passed; nothing to learn from."); break; } const optRun = await workspace.run(optimizerMachine, failureReport(current, currentRuns), { evaluate: true }); const proposal = parseProposal(optRun.output); if (!proposal || optRun.scorecard?.total !== 1) { console.log(` the optimizer made no usable proposal (${optRun.scorecard?.criteria.proposal?.reason ?? "no reply"}); run ${optRun.id}`); continue; } const diagnosis = optRun.output.slice(0, optRun.output.indexOf("<instructions>")).trim(); if (diagnosis) console.log(` optimizer: ${diagnosis.replace(/\s+/g, " ")}`); console.log(` proposal:\n${proposal.replace(/^/gm, " | ")}`); const candidate = withInstructions(current, proposal); const candidateRuns = await runCases(candidate, cases, `optimizer:${optRun.id}`); const c = comparePaired(cases, currentRuns, candidateRuns); console.log(` proposal scores ${c.b.mean.toFixed(3)} vs ${c.a.mean.toFixed(3)} (${signed(c.diff)} ± ${c.stderr.toFixed(3)}); better on ${c.better} cases, worse on ${c.worse}`); if (values.verbose) { for (const x of c.cases) if (x.diff !== 0) console.log(` ${signed(x.diff)} ${preview(x.input, 60)}`); } let h: Comparison | undefined; let candidateHoldout: Run[][] | undefined; if (c.diff > 0 && holdout) { currentHoldout ??= await runCases(current, holdout, `${head}+working-tree`); candidateHoldout = await runCases(candidate, holdout, `optimizer:${optRun.id}`); h = comparePaired(holdout, currentHoldout, candidateHoldout); console.log(` held out: ${h.b.mean.toFixed(3)} vs ${h.a.mean.toFixed(3)} (${signed(h.diff)} ± ${h.stderr.toFixed(3)}); better on ${h.better} cases, worse on ${h.worse}`); if (values.verbose) { for (const x of h.cases) if (x.diff !== 0) console.log(` ${signed(x.diff)} ${preview(x.input, 60)} (held out)`); } } const decision = decide(c, h, { minZ }); if (decision.keep) { current = candidate; currentRuns = candidateRuns; currentHoldout = candidateHoldout; kept++; await writeFile(file, `${proposal}\n`); console.log(` kept: ${decision.reason}; wrote ${path.relative(workspaceDir, file)}`); } else { console.log(` discarded: ${decision.reason}`); } } console.log( kept ? `\n${kept} rewrite${kept === 1 ? "" : "s"} kept. The change is uncommitted: review with git diff, commit to accept, or git checkout -- ${path.relative(workspaceDir, file)} to reject.` : "\nNo rewrite was kept; instructions.md is unchanged.", );}
/** * Grade a machine's stored runs with a candidate grader and measure it against * the machine's own grader for one criterion. Stored scores from that grader * are reused, so a judge reference is not paid for twice. */async function runAgreement( machine: Machine, criterionId: string, workspace: Workspace, harness: Harness, values: { limit?: string; with: string; question?: string; "judge-model"?: string; "judge-effort"?: string; verbose: boolean },) { const criterion = machine.rubric[criterionId]; const reference = machine.evaluator[criterionId]; // A conversation is graded on its last turn; earlier turns are not runs to grade on their own. const runs = (await workspace.store.listRuns(machine.name, { limit: Number(values.limit ?? 10_000) })).filter( (r) => r.harness !== "echo" && (!r.conversation || r.conversation.final), ); if (runs.length === 0) throw new Error(`No stored runs for ${machine.name}; run its suite first.`); let candidate: Grader; if (values.with === "jev") candidate = jev().noul({ soft: true, question: values.question }); else if (values.with === "judge") candidate = judge({ model: values["judge-model"], reasoningEffort: effortValue(values["judge-effort"]) }); else throw new Error(`--with must be jev or judge, not "${values.with}".`); if (candidate.name === reference.name) throw new Error(`${criterionId} is already graded by ${reference.name}.`);
const grade = async (grader: Grader, run: Run) => { workspace.budget?.check(`grading run ${run.id} with ${grader.name}`); const score = await grader.grade({ run, criterionId, criterion, harness, judgeModel: values["judge-model"] }); workspace.budget?.charge(score.usage, `Grading run ${run.id} with ${grader.name}`); return score.score; }; let reused = 0; const pairs = await mapLimit(runs, 8, async (run) => { const stored = run.scorecard?.criteria[criterionId]; const ref = stored && stored.grader === reference.name ? (reused++, stored.score) : await grade(reference, run); return { run, ref: ref >= 0.5, p: await grade(candidate, run) }; });
const refLabels = pairs.map((x) => x.ref); const passRate = refLabels.filter(Boolean).length / pairs.length; console.log(`${machine.name} · ${criterionId} · ${candidate.name} against ${reference.name} · ${pairs.length} stored runs (${reused} reference grades reused)`); console.log(`reference passes ${pct(passRate)} of runs\n`); console.log("threshold agree kappa false-pass false-fail precision bits/label bad kept at 10% / 50% pass rate"); const thresholds = values.with === "jev" ? [0.5, 0.7, 0.9] : [0.5]; for (const t of thresholds) { const a = agreement(refLabels, pairs.map((x) => x.p >= t)); const at = (p: number) => pct(contamination(p, a.falsePassRate, a.falseFailRate)); console.log( `${t.toFixed(1).padStart(9)} ${pct(a.agreement).padStart(5)} ${a.kappa.toFixed(2).padStart(5)} ${`${pct(a.falsePassRate)} (${a.negatives - a.trueFails}/${a.negatives})`.padStart(10)} ${`${pct(a.falseFailRate)} (${a.positives - a.truePasses}/${a.positives})`.padStart(10)} ${pct(a.precision).padStart(9)} ${Math.max(0, a.bitsPerLabel).toFixed(2).padStart(10)} ${at(0.1)} / ${at(0.5)}`, ); } const unsure = pairs.filter((x) => x.p > 0.3 && x.p < 0.7).length; if (unsure) { console.log(`\n${unsure} of ${pairs.length} runs scored between 0.3 and 0.7: the candidate is unsure there, and its scores can cross 0.5 from one call to the next. Escalate those to a stronger grader.`); } if (refLabels.every(Boolean) || refLabels.every((x) => !x)) { console.log(`\nnote: the reference ${refLabels[0] ? "passed" : "failed"} every run, so only one error rate can be measured; add runs of the other kind.`); } const disagreements = new Map<string, number>(); for (const x of pairs) { if ((x.p >= 0.5) === x.ref) continue; const key = `${x.ref ? "false fail" : "false pass"}: ${preview(x.run.input, 70)}`; disagreements.set(key, (disagreements.get(key) ?? 0) + 1); } if (disagreements.size) { console.log("\nDisagreements at 0.5:"); for (const [k, v] of [...disagreements].sort((a, b) => b[1] - a[1]).slice(0, values.verbose ? 50 : 10)) console.log(` ${v}× ${k}`); }}
function printComparison(nameA: string, nameB: string, c: Comparison, trials: number) { console.log(`\n${c.cases.length} cases × ${trials} trial${trials === 1 ? "" : "s"} per machine`); console.log(` ${nameA.padEnd(14)} mean ${c.a.mean.toFixed(3)} · pass rate ${pct(c.a.passRate)}`); console.log(` ${nameB.padEnd(14)} mean ${c.b.mean.toFixed(3)} · pass rate ${pct(c.b.passRate)}`); console.log(` difference ${signed(c.diff)} ± ${c.stderr.toFixed(3)} (standard error over cases)`); console.log(` ${nameB} better on ${c.better} cases, worse on ${c.worse}, tied on ${c.tied}`); for (const id of Object.keys(c.a.criteria)) { console.log(` ${id.padEnd(14)} ${c.a.criteria[id].toFixed(3)} → ${c.b.criteria[id].toFixed(3)} ${signed(c.b.criteria[id] - c.a.criteria[id])}`); }}
function printFailures(runs: Run[], label: string) { for (const [t, run] of runs.entries()) { for (const [id, c] of Object.entries(run.scorecard!.criteria)) { if (c.score < 1) console.log(` ${label}trial ${t + 1} ${id}: ${c.reason}`); } }}
function statLabel(s: TrialStats): string { return s.trials === 1 ? s.mean.toFixed(2) : `${s.mean.toFixed(2)} (${Math.round(s.passRate * s.trials)}/${s.trials})`;}
function pct(x: number): string { return `${Math.round(x * 100)}%`;}
function trialCount(value: string): number { const n = Number(value); if (!Number.isInteger(n) || n < 1 || n > 100) throw new Error("--trials must be a whole number from 1 to 100."); return n;}
function signed(x: number): string { return `${x >= 0 ? "+" : ""}${x.toFixed(3)}`;}
const COMMANDS = new Set(["dashboard", "run", "runs", "eval", "suite", "optimize", "agree", "compare", "tree", "export", "converse", "train", "propose", "commit", "revert", "versions", "specialize"]);
function makeBudget(value: string | undefined, kind: string): Budget | undefined { if (value === "none") return undefined; if (value === undefined) return kind === "tinker" || kind === "pi" ? new Budget(1) : undefined; const usd = Number(value); if (!Number.isFinite(usd) || usd < 0) throw new Error("--budget must be a dollar amount or none."); return new Budget(usd);}
/** "none" … "xhigh" pass through; a number in [0, 0.99] is sent as a number. */function effortValue(value: string | undefined): string | number | undefined { if (value === undefined || value === "default") return undefined; if (["none", "minimal", "low", "medium", "high", "xhigh"].includes(value)) return value; const n = Number(value); if (Number.isFinite(n) && n >= 0 && n <= 0.99) return n; throw new Error(`Effort must be none, minimal, low, medium, high, xhigh, default, or a number from 0 to 0.99; got "${value}".`);}
function positiveInt(value: string, flag: string): number { const n = Number(value); if (!Number.isInteger(n) || n < 1 || n > 64) throw new Error(`${flag} must be a whole number from 1 to 64.`); return n;}
function printSummary(caseCount: number, trials: number, runsByCase: Run[][]) { if (runsByCase.flat().length === 0) { console.log("\nno successful runs"); return; } const summary = summarize(runsByCase); console.log(`\n${caseCount} cases × ${trials} trial${trials === 1 ? "" : "s"} · mean ${summary.mean.toFixed(3)} · pass rate ${pct(summary.passRate)}`); for (const [id, score] of Object.entries(summary.criteria)) console.log(` ${id.padEnd(16)} ${score.toFixed(3)}`);}
/** * Criteria shared down a tree, scored across every machine that inherits * them: how well the family does at what all of it must do, next to what * each child adds. */function printTreeSummary(root: Machine, results: Array<{ machine: Machine; runsByCase: Run[][] }>) { console.log(`\n== ${root.name} tree: inherited criteria across ${results.length} machines`); for (const id of Object.keys(root.rubric)) { const scores = results.flatMap((r) => r.runsByCase.flat().map((run) => run.scorecard?.criteria[id]?.score)).filter((x): x is number => x !== undefined); if (scores.length) console.log(` ${id.padEnd(16)} ${(scores.reduce((a, b) => a + b, 0) / scores.length).toFixed(3)} (${scores.length} runs)`); }}
function printTree(root: Machine, children: Machine[]) { const own = (m: Machine) => Object.keys(m.rubric).filter((id) => !m.parent || !(id in m.parent.rubric)); const depth = (m: Machine) => m.name.split("/").length - root.name.split("/").length; for (const m of [root, ...children]) { const pad = " ".repeat(depth(m)); console.log(`${pad}${m.name} · ${m.model}`); console.log(`${pad} ${m.intent}`); if (m.parent) console.log(`${pad} inherits: ${Object.keys(m.parent.rubric).join(", ")}`); console.log(`${pad} ${m.parent ? "adds" : "criteria"}: ${own(m).join(", ")}`); }}
function printRun(run: Run, json: boolean) { if (json) { console.log(JSON.stringify(run, null, 2)); return; } console.log(run.output); console.error(`\n${run.id} · ${run.harness} · ${run.model}`); if (run.scorecard) { console.error(`score ${run.scorecard.total}`); for (const [id, c] of Object.entries(run.scorecard.criteria)) { console.error(` ${id.padEnd(12)} ${c.score.toFixed(2)} (${c.grader}) ${c.reason}`); } }}
/** One turn of a conversation: who spoke, their tool calls with results, and what they said. */function printTurn(speaker: string, run: Run) { console.log(`── ${speaker}`); for (const m of run.trajectory) { if (m.role === "tool") console.log(` [${m.name} →] ${m.content.replace(/\n/g, "\n ")}`); } console.log(`${run.output.trim() || "(no message)"}\n`);}
function scoreLabel(run: Run): string { return run.scorecard ? `score ${run.scorecard.total.toFixed(2)}` : "unscored ";}
function preview(text: string, limit: number): string { const flat = text.replace(/\s+/g, " ").trim(); return flat.length > limit ? `${flat.slice(0, limit - 1)}…` : flat;}
const VERSION_COMMANDS = new Set(["train", "propose", "commit", "revert", "versions", "specialize"]);
/** The recorded version this definition matches, if the machine keeps a history. Read-only. */async function recordedVersion(workspaceDir: string, machine: Machine): Promise<string | undefined> { try { const file = versionFile(path.join(workspaceDir, "machines", ...nameSegments(machine.name, "machine name"), "versions.json")); return findVersion(await file.read(machine.name), machine)?.id; } catch { return undefined; }}
/** train, commit, revert, and versions: through the SDK. */async function versionCommand( command: string, name: string, rest: string[], values: { harness?: string; model?: string; "judge-model"?: string; "judge-effort"?: string; effort?: string; budget?: string; name?: string; concurrency: string; trials?: string; "min-score": string; "min-z": string; cases?: string; holdout?: string; samples?: string; "best-of"?: string; epochs?: string; lr?: string; "any-model": boolean; force: boolean; json: boolean; focus?: string; "author-model"?: string; slug?: string; "train-cases"?: string; "holdout-cases"?: string; "reuse-cases": boolean; "batch-size"?: string; }, workspaceDir: string,): Promise<number> { const m = await sdkMachine.load(name, { workspace: workspaceDir, workspaceName: values.name, harness: values.harness, judgeModel: values["judge-model"], judgeEffort: effortValue(values["judge-effort"]), reasoningEffort: effortValue(values.effort), concurrency: positiveInt(values.concurrency, "--concurrency"), log: (line) => console.error(line), }); try { switch (command) { case "versions": { const { live, versions, liveOrder } = await m.versions(); if (values.json) { console.log(JSON.stringify({ live: live.id, liveOrder, versions }, null, 2)); return 0; } for (const v of versions) { const mark = v.id === live.id ? "*" : " "; const source = v.source.kind === "train" ? `trained (candidate ${v.source.candidate})` : v.source.kind === "optimize" ? `optimized (candidate ${v.source.candidate})` : v.source.kind === "propose" ? `proposed (candidate ${v.source.candidate})` : v.source.kind; console.log(`${mark} ${v.id.padEnd(4)} ${v.createdAt.slice(0, 16).replace("T", " ")} ${v.model} ${source}${v.note ? ` · ${v.note}` : ""}`); } console.log(`\nlive: ${live.id}${liveOrder.length > 1 ? ` (became live in order ${liveOrder.join(" → ")})` : ""}`); const pending = await m.candidates({ status: "pending" }); if (pending.length) console.log(`pending candidates:\n${pending.map((c) => ` ${c.id} ${c.report.decision.keep ? "passed" : "held "} ${c.report.kind} held out ${c.report.holdout.live.mean.toFixed(3)} → ${c.report.holdout.candidate.mean.toFixed(3)}`).join("\n")}`); return 0; } case "train": { if (values.budget === undefined || values.budget === "none") throw new Error("train needs --budget <usd>: it samples, grades, trains, and compares on paid endpoints."); const budget = Number(values.budget); if (!(budget >= 0)) throw new Error("--budget must be a dollar amount."); const definition = m.definition; const candidate = await m.train({ budget, samples: values.samples === undefined ? 0 : Number(values.samples), cases: values.cases ? await loadCases(workspaceDir, definition, values.cases) : undefined, holdout: values.holdout ? await loadCases(workspaceDir, definition, values.holdout) : undefined, trials: values.trials === undefined ? undefined : trialCount(values.trials), minScore: Number(values["min-score"]), minZ: Number(values["min-z"]), bestOf: values["best-of"] === undefined ? undefined : positiveInt(values["best-of"], "--best-of"), epochs: values.epochs === undefined ? undefined : positiveInt(values.epochs, "--epochs"), learningRate: values.lr === undefined ? undefined : Number(values.lr), batchSize: values["batch-size"] === undefined ? undefined : positiveInt(values["batch-size"], "--batch-size"), anyModel: values["any-model"], }); console.log(formatReport(candidate.report)); console.log( candidate.report.decision.keep ? `\nCandidate ${candidate.id} is pending. Make it live: machine commit ${name}` : `\nCandidate ${candidate.id} is pending but did not pass the gate (machine commit ${name} --force overrides).`, ); return 0; } case "specialize": { if (!values.focus?.trim()) throw new Error('specialize needs --focus "<area>".'); if (values.budget === undefined || values.budget === "none") throw new Error("specialize needs --budget <usd>: the author model is paid."); const result = await m.specialize({ focus: values.focus.trim(), budget: Number(values.budget), model: values.model, authorModel: values["author-model"], slug: values.slug, trainCases: values["train-cases"] === undefined ? undefined : positiveInt(values["train-cases"], "--train-cases"), holdoutCases: values["holdout-cases"] === undefined ? undefined : positiveInt(values["holdout-cases"], "--holdout-cases"), ...(values.effort === undefined ? {} : { authorEffort: effortValue(values.effort) }), overwrite: values.force, reuseCases: values["reuse-cases"], }); console.log(formatSpecialization(result)); if (values.json) console.log(JSON.stringify(result, null, 2)); return 0; } case "propose": { if (!values.model) throw new Error("propose needs --model <id>."); if (values.budget === undefined || values.budget === "none") throw new Error("propose needs --budget <usd>."); const definition = m.definition; const candidate = await m.propose( { model: values.model }, { budget: Number(values.budget), holdout: values.holdout ? await loadCases(workspaceDir, definition, values.holdout) : undefined, trials: values.trials === undefined ? undefined : trialCount(values.trials), minZ: Number(values["min-z"]), }, ); console.log(formatReport(candidate.report)); console.log(candidate.report.decision.keep ? `\nCandidate ${candidate.id} is pending. Make it live: machine commit ${name}` : `\nCandidate ${candidate.id} is pending but did not pass the gate.`); return 0; } case "commit": { const candidate = await m.candidate(rest[0] ?? "latest"); const from = await m.version(); try { const entry = await candidate.commit({ force: values.force }); const h = candidate.report.holdout; console.log(`${name}: ${from.id} → ${entry.id} (${entry.model})`); console.log(` held out: mean ${h.live.mean.toFixed(3)} → ${h.candidate.mean.toFixed(3)}, pass rate ${Math.round(h.live.passRate * 100)}% → ${Math.round(h.candidate.passRate * 100)}%`); console.log(` ${candidate.report.decision.keep ? "passed" : "FORCED past"} the gate: ${candidate.report.decision.reason}`); console.log(` undo with: machine revert ${name}`); return 0; } catch (error) { if (error instanceof CommitRefused) { const h = error.report.holdout; console.error(`${name}: not committed; ${from.id} stays live (${from.model})`); console.error(` candidate ${candidate.id} (${error.report.candidate.model})`); console.error(` held out: mean ${h.live.mean.toFixed(3)} → ${h.candidate.mean.toFixed(3)} (${h.diff >= 0 ? "+" : ""}${h.diff.toFixed(3)} ± ${h.stderr.toFixed(3)}), pass rate ${Math.round(h.live.passRate * 100)}% → ${Math.round(h.candidate.passRate * 100)}%`); console.error(` ${error.message.replace(/^Not committed: /, "").replace(/ Force it with .*$/, "")}`); console.error(` machine commit ${name} --force commits it anyway.`); return 1; } throw error; } } case "revert": { const from = await m.version(); const to = await m.revert(); console.log(`${name}: ${from.id} → ${to.id} (${to.model})`); return 0; } default: return 1; } } finally { await m.close(); }}
main(process.argv.slice(2)).then( (code) => process.exit(code), (error) => { console.error(`machine: ${error instanceof Error ? error.message : error}`); process.exit(1); },);