Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891// The machine SDK: load a machine, run it (streaming what it does), keep a// multi-turn session, grade what is pending, and improve it in place.//// const dev = await machine.load("dev");// for await (const message of dev.run("Fix the failing test")) console.log(message.type);// const candidate = await dev.train({ budget: 5 }); // never changes the live machine// console.log(formatReport(candidate.report));// await candidate.commit(); // only if it beat the live version on held-out cases// await dev.revert(); // back to the previous version//// A machine version is (instructions, model, rubric); see versions.ts. Every// run records the version that made it. train() and optimize() return a// Candidate: a proposed new version with a report measured against the live// one on held-out cases. Committing it is the only way it becomes live, and// commit() refuses a candidate that did not pass the promotion gate unless// forced.
import path from "node:path";import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";import { hasCaseFile, loadCases, type Case } from "./cases.ts";import type { Comparison, MachineSummary } from "./compare.ts";import { fileEvents, type EventSink, type HarnessEvent } from "./events.ts";import type { Harness } from "./harness.ts";import { DEFAULT_TINKER_JUDGE, defaultHarness, lazyHarness } from "./harnesses/select.ts";import { defineMachine, INSTRUCTIONS_FILE, MODEL_FILE, usesModelFile, withInstructions, withModel, type Machine, type MachineSpec } from "./machine.ts";import { decide, failureReport, optimizer, parseProposal } from "./optimizer.ts";import { baseModelOf, Budget, BudgetExceeded, priceCheckpointAs, TINKER_TRAIN_PRICES } from "./pricing.ts";import { loadMachineFrom } from "./revision.ts";import { rubricFingerprint, type Rubric } from "./rubric.ts";import { directorySandbox, localSandboxes, type Sandbox, type SandboxProvider } from "./sandbox.ts";import { diskStore, nameSegments, newRunId, type Store } from "./store.ts";import { DEFAULT_AUTHOR_MODEL, specialize, type SpecializeOptions, type SpecializeResult } from "./specialize.ts";import { effortScalar, estimateTokens, selectRuns, tinkerSft, wholeRunsFor, trainingCost, trainingLines, writeLines, type Trainer } from "./training.ts";import { comparePaired, mapLimit, runMatrix } from "./trials.ts";import type { Message, Run } from "./types.ts";import { addVersion, findVersion, liveVersion, versionFile, type VersionEntry, type VersionFile, type VersionHistory, type VersionSource } from "./versions.ts";import { createWorkspace, type Workspace } from "./workspace.ts";
export type MachineOptions = { /** Directory holding machines/ and .machine/. Default: the current directory. */ workspace?: string; /** Workspace name recorded on runs. Default: $MACHINE_WORKSPACE or "local". */ workspaceName?: string; /** Agent loop: a harness, or a name ("tinker", "pi", …). Default: Pi for openrouter/ models, Tinker otherwise. */ harness?: Harness | string; /** Model for judge graders. Default: $MACHINE_JUDGE_MODEL, else Inkling-Small on Tinker and Pi. */ judgeModel?: string; judgeEffort?: string | number; reasoningEffort?: string | number; /** Spending limit (USD) for run(), session(), and grade() through this handle. train() and optimize() take their own. */ budget?: number; /** Runs at once in grade(), train(), and optimize(). Default 4. */ concurrency?: number; store?: Store; sandboxes?: SandboxProvider; events?: EventSink; /** Where versions are kept. Default: machines/<name>/versions.json for a loaded machine, .machine/versions/ for machine.new. */ versions?: VersionFile; /** Progress lines from train() and optimize(). Default: none. */ log?: (line: string) => void;};
/** What run() streams: the run starting, its steps and tool calls as the harness reports them, then the stored run. */export type RunMessage = | { type: "start"; machine: string; version: string; model: string; input: string } | HarnessEvent | { type: "result"; run: Run };
export type RunCallOptions = { /** Files the working directory starts with. */ files?: Record<string, string>; /** Grade the run before it is returned. Default false. */ grade?: boolean; history?: Message[]; signal?: AbortSignal;};
/** * A run in progress: iterate it for messages as they happen (the last is the * result), or await it for the finished run. */export class RunHandle implements AsyncIterable<RunMessage>, PromiseLike<Run> { readonly done: Promise<Run>; #queue: RunMessage[] = []; #finished = false; #error: unknown; #wake: (() => void) | undefined;
constructor(start: (emit: (message: RunMessage) => void) => Promise<Run>) { const emit = (message: RunMessage) => { this.#queue.push(message); this.#wake?.(); }; this.done = start(emit).then( (run) => { emit({ type: "result", run }); this.#finished = true; this.#wake?.(); return run; }, (error) => { this.#error = error; this.#finished = true; this.#wake?.(); throw error; }, ); // Iterating without awaiting must not leave an unhandled rejection; awaiting still sees it. this.done.catch(() => {}); }
then<A = Run, B = never>(onFulfilled?: ((run: Run) => A | PromiseLike<A>) | null, onRejected?: ((reason: unknown) => B | PromiseLike<B>) | null): Promise<A | B> { return this.done.then(onFulfilled, onRejected); }
async *[Symbol.asyncIterator](): AsyncIterator<RunMessage> { for (;;) { if (this.#queue.length) yield this.#queue.shift()!; else if (this.#finished) { if (this.#error !== undefined) throw this.#error; return; } else await new Promise<void>((resolve) => (this.#wake = resolve)); } }}
export type SessionOptions = { /** Grade each turn as it finishes. */ autoGrade?: boolean; /** Work in this existing directory (tools change it). Default: a fresh directory for the session. */ cwd?: string; /** Files the session's fresh directory starts with. */ files?: Record<string, string>;};
/** A multi-turn conversation with one working directory that persists between turns. */export type MachineSession = { readonly id: string; /** This session's runs so far, oldest first (graded ones carry their scorecard). */ readonly runs: Run[]; send(input: string, options?: { signal?: AbortSignal }): RunHandle; close(): Promise<void>;};
export type TrainOptions = { /** Hard spending limit (USD) for this call: sampling, grading, training, and the held-out comparison. */ budget: number; /** Cases to sample before training. Default: the machine's cases.json (when samples > 0). */ cases?: Case[]; /** New runs per training case before training. Default 0: train on runs already stored. */ samples?: number; /** Held-out cases for the comparison; never trained on. Default: holdout.json. */ holdout?: Case[]; /** Runs per held-out case for each version. Default 2. Stored graded runs of the live version are reused. */ trials?: number; /** Lowest score to train on. Default 1. */ minScore?: number; /** Most runs per case to train on, best first. Default 2. */ bestOf?: number; /** Train on good runs by any model (distillation). Default: only the live model and its base (rejection sampling). */ anyModel?: boolean; epochs?: number; learningRate?: number; batchSize?: number; /** One example per run instead of per step (see wholeRunExample). Default: true for Inkling, whose renderer was checked for it. */ wholeRuns?: boolean; /** Standard errors the candidate must win by on held-out cases. Default 2. */ minZ?: number; /** How to train. Default: tinkerSft() (scripts/train_sft.py on Tinker). */ trainer?: Trainer; signal?: AbortSignal;};
export type OptimizeOptions = { budget: number; cases?: Case[]; holdout?: Case[]; trials?: number; minZ?: number; /** Model for the optimizer. Default: the live model's base. */ optimizerModel?: string;};
export type VersionSummary = { version: string; model: string };
export type ComparisonSummary = { cases: number; trials: number; live: MachineSummary; candidate: MachineSummary; diff: number; stderr: number; better: number; worse: number; tied: number; perCase: Array<{ input: string; live: number; candidate: number }>;};
export type CandidateReport = { kind: "train" | "optimize" | "propose"; machine: string; live: VersionSummary; candidate: { model: string; instructionsChanged: boolean }; data?: { runs: number; examples: number; inputs: number; tokens: number; byModel: Record<string, number>; wholeRuns?: boolean }; training?: { baseModel: string; checkpoint: string; steps: number; epochs: number; learningRate?: number; firstLoss?: number; lastLoss?: number; logDir: string; costUsd: number }; /** Optimize only: the cases the optimizer learned from. */ train?: ComparisonSummary; holdout: ComparisonSummary; decision: { keep: boolean; reason: string }; spentUsd: number;};
type CandidateRecord = { id: string; machine: string; createdAt: string; /** The live version the candidate was measured against. */ from: string; change: { model?: string; baseModel?: string; instructions?: string }; report: CandidateReport; status: "pending" | "committed"; /** The version it became, once committed. */ version?: string;};
/** commit() refused: the candidate did not pass the promotion gate (or the live version moved on). */export class CommitRefused extends Error { readonly report: CandidateReport; constructor(message: string, report: CandidateReport) { super(message); this.name = "CommitRefused"; this.report = report; }}
export type Candidate = { readonly id: string; readonly machine: string; /** The live version it was measured against. */ readonly from: string; readonly change: CandidateRecord["change"]; readonly report: CandidateReport; readonly status: CandidateRecord["status"]; /** * Make the candidate the live version, if it passed the promotion gate * (else throws CommitRefused with the report). `force` commits anyway. */ commit(options?: { force?: boolean }): Promise<VersionEntry>;};
/** Whether a run is due for grading: never graded, or graded under another rubric. */export function needsGrade(run: Run, rubric: Rubric): boolean { if (!run.scorecard) return true; if (run.scorecard.rubric) return run.scorecard.rubric !== rubricFingerprint(rubric); // Graded before scorecards recorded their rubric: due only if the criteria differ. const ids = Object.keys(run.scorecard.criteria).sort().join(","); return ids !== Object.keys(rubric).sort().join(",");}
/** * The promotion gate for a candidate measured against the live version on * held-out cases: it must score higher by at least `minZ` standard errors of * the per-case difference (the margin `machine optimize` uses). */export function promotionGate(holdout: Comparison, minZ = 2): { keep: boolean; reason: string } { const fmt = `${holdout.diff >= 0 ? "+" : ""}${holdout.diff.toFixed(3)} ± ${holdout.stderr.toFixed(3)}`; if (holdout.diff <= 1e-9) return { keep: false, reason: `it did not score higher than the live version on held-out cases (${fmt})` }; if (holdout.diff < minZ * holdout.stderr) return { keep: false, reason: `its gain on held-out cases (${fmt}) is under ${minZ} standard errors, within trial noise` }; return { keep: true, reason: `it scores higher than the live version on held-out cases by ${fmt}, at least ${minZ} standard errors` };}
export class MachineHandle { readonly name: string; readonly workspaceDir: string; readonly store: Store; #definition: Machine; #persist: "files" | "history"; #versions: VersionFile; #history: VersionHistory; #ensured: Promise<VersionEntry> | undefined; #options: MachineOptions; #harnesses = new Map<string, Harness>(); #events: EventSink; #budget: Budget | undefined;
constructor(definition: Machine, history: VersionHistory, versions: VersionFile, persist: "files" | "history", options: MachineOptions) { this.name = definition.name; this.workspaceDir = path.resolve(options.workspace ?? process.cwd()); this.store = options.store ?? diskStore(path.join(this.workspaceDir, ".machine")); this.#definition = definition; this.#history = history; this.#versions = versions; this.#persist = persist; this.#options = options; this.#events = options.events ?? fileEvents(path.join(this.workspaceDir, ".machine", "events.jsonl")); this.#budget = options.budget === undefined ? undefined : new Budget(options.budget); }
/** The machine as it runs now: the live version's instructions and model over its definition. */ get definition(): Machine { return this.#definition; }
get model(): string { return this.#definition.model; }
/** The live version, recording the current definition as a new version if it matches none. */ async version(): Promise<VersionEntry> { this.#ensured ??= this.#ensureVersion(); return this.#ensured; }
/** Every version, oldest first, and which is live. */ async versions(): Promise<{ live: VersionEntry; versions: VersionEntry[]; liveOrder: string[] }> { const live = await this.version(); return { live, versions: this.#history.versions, liveOrder: this.#history.live }; }
/** Run once on one input. Iterate the handle for progress, or await it for the run. */ run(input: string, options: RunCallOptions = {}): RunHandle { return new RunHandle(async (emit) => { const version = await this.version(); emit({ type: "start", machine: this.name, version: version.id, model: this.model, input }); return this.#workspace(this.#budget).run(this.#definition, input, { version: version.id, files: options.files, history: options.history, evaluate: options.grade, signal: options.signal, onEvent: emit, }); }); }
/** A multi-turn conversation. Each turn sees the earlier ones, and the working directory persists between turns. */ session(options: SessionOptions = {}): MachineSession { const id = `sdk-${newRunId()}`; const runs: Run[] = []; let history: Message[] = []; let sandbox: Promise<Sandbox> | undefined; let queue: Promise<unknown> = Promise.resolve(); const open = () => (sandbox ??= (async () => { const box = options.cwd ? await directorySandbox(options.cwd) : await (this.#options.sandboxes ?? localSandboxes(path.join(this.workspaceDir, ".machine", "sandboxes"))).create(id); for (const [file, text] of Object.entries(options.cwd ? {} : (options.files ?? {}))) await box.writeFile(file, text); return box; })()); return { id, runs, send: (input, sendOptions = {}) => new RunHandle(async (emit) => { const previous = queue; let release!: () => void; queue = new Promise<void>((resolve) => (release = resolve)); await previous.catch(() => {}); try { const version = await this.version(); emit({ type: "start", machine: this.name, version: version.id, model: this.model, input }); const run = await this.#workspace(this.#budget).run(this.#definition, input, { version: version.id, history, sandbox: await open(), evaluate: options.autoGrade, signal: sendOptions.signal, onEvent: emit, }); const tagged: Run = { ...run, session: id }; await this.store.putRun(tagged); history = [...history, ...run.trajectory]; await this.store.putSession({ machine: this.name, name: id, messages: history, updatedAt: run.finishedAt }); runs.push(tagged); return tagged; } finally { release(); } }), close: async () => { await queue.catch(() => {}); if (sandbox && !options.cwd) await (await sandbox).dispose(); }, }; }
/** Runs due for grading: never graded, or graded under an older rubric. Newest first. */ async pending(options: { limit?: number; all?: boolean } = {}): Promise<Run[]> { const runs = await this.store.listRuns(this.name, { limit: options.limit ?? 100_000 }); return runs.filter((r) => r.harness !== "echo" && (!r.conversation || r.conversation.final) && (options.all || needsGrade(r, this.#definition.rubric))); }
/** * Grade every pending run against the current rubric. Returns the runs * graded. `all` regrades every run, e.g. after changing the judge model, so * all scores come from one grader. */ async grade(options: { limit?: number; budget?: number; all?: boolean } = {}): Promise<Run[]> { return this.#gradeWith(this.#workspace(options.budget === undefined ? this.#budget : new Budget(options.budget)), options.limit, options.all); }
async #gradeWith(workspace: Workspace, limit?: number, all = false): Promise<Run[]> { const pending = await this.pending({ limit, all }); const graded: Run[] = []; let stop: unknown; await mapLimit(pending, this.#options.concurrency ?? 4, async (run) => { if (stop) return; try { graded.push(await workspace.evaluate(this.#definition, run)); } catch (error) { if (error instanceof BudgetExceeded) stop = error; else this.#log(` could not grade ${run.id}: ${error instanceof Error ? error.message : error}`); } }); if (stop) throw stop; return graded; }
/** * Change the machine: new instructions and/or model become a new live * version. A loaded machine's files change too (instructions.md and * model.json), so the CLI and git see it. */ async update(change: { instructions?: string; model?: string; baseModel?: string; note?: string }, source: VersionSource = { kind: "update", note: change.note }): Promise<VersionEntry> { await this.version(); let next = this.#definition; if (change.instructions !== undefined) next = withInstructions(next, change.instructions.trim()); if (change.model !== undefined) next = withModel(next, change.model); if (change.model && change.baseModel) priceCheckpointAs(change.model, change.baseModel); const baseModel = change.baseModel ?? baseModelOf(next.model); await this.#apply(next, baseModel); const { history, entry } = addVersion(this.#history, next, source, { baseModel, note: change.note }); await this.#versions.write(history); this.#history = history; this.#ensured = Promise.resolve(entry); return entry; }
/** Make the previous live version live again. Returns it. */ async revert(): Promise<VersionEntry> { const current = await this.version(); if (this.#history.live.length < 2) throw new Error(`${this.name} has no earlier version to revert to (live: ${current.id}).`); const previousId = this.#history.live.at(-2)!; const previous = this.#history.versions.find((v) => v.id === previousId)!; if (previous.baseModel) priceCheckpointAs(previous.model, previous.baseModel); const next = withModel(withInstructions(this.#definition, previous.instructions), previous.model); await this.#apply(next, previous.baseModel); const history = { ...this.#history, live: this.#history.live.slice(0, -1) }; await this.#versions.write(history); this.#history = history; this.#ensured = Promise.resolve(previous); if (previous.rubric !== rubricFingerprint(this.#definition.rubric)) { this.#log(`note: ${previous.id} was made under another rubric; the current rubric stays (rubric changes are their own step).`); } return previous; }
/** * Fine-tune the live model on good runs and measure the result against the * live version on held-out cases. Never changes the live machine: returns a * Candidate to commit. */ async train(options: TrainOptions): Promise<Candidate> { const budget = new Budget(options.budget); const workspace = this.#workspace(budget); const live = await this.version(); const def = this.#definition; const baseModel = baseModelOf(def.model); if (!baseModel || !(baseModel in TINKER_TRAIN_PRICES)) { throw new Error(`${this.name} runs on ${def.model}, which cannot be fine-tuned on Tinker. Move it to a Tinker model first: update({ model: "Qwen/Qwen3-8B" }).`); } const holdout = options.holdout ?? (await this.#cases("holdout.json")); if (!holdout?.length) throw new Error(`train needs held-out cases: pass holdout, or add machines/${this.name}/holdout.json.`); const trials = options.trials ?? 2; const concurrency = this.#options.concurrency ?? 4; const id = newRunId(); const log = (line: string) => this.#log(line);
const samples = options.samples ?? 0; if (samples > 0) { const cases = options.cases ?? (await this.#cases("cases.json")); if (!cases?.length) throw new Error(`samples needs training cases: pass cases, or add machines/${this.name}/cases.json.`); log(`sampling: ${cases.length} cases × ${samples} with ${live.id} (${def.model})`); await runMatrix(workspace, [{ machine: def, version: live.id }], cases, samples, concurrency, undefined, log); } const graded = await this.#gradeWith(workspace); if (graded.length) log(`graded ${graded.length} pending run${graded.length === 1 ? "" : "s"}`);
// Training data: good runs, never a held-out case. const stored = await this.store.listRuns(this.name, { limit: 100_000 }); const chosen = selectRuns(def, stored, { minScore: options.minScore ?? 1, bestOf: options.bestOf ?? 2, exclude: new Set(holdout.map((c) => c.input)), models: options.anyModel ? undefined : new Set([def.model, baseModel]), }); if (chosen.length === 0) throw new Error(`No graded runs of ${this.name} score at least ${options.minScore ?? 1} outside the held-out cases; sample some first (samples: 4).`); const lines = trainingLines(def, chosen, { wholeRuns: options.wholeRuns ?? wholeRunsFor(baseModel) }); const tokens = estimateTokens(lines); const epochs = options.epochs ?? 2; const estimate = trainingCost(baseModel, tokens, epochs); const byModel: Record<string, number> = {}; for (const run of chosen) byModel[run.model] = (byModel[run.model] ?? 0) + 1; const inputs = new Set(chosen.map((r) => r.input)).size; const unit = lines.some((l) => l.whole_run) ? "whole-run" : "step"; log(`data: ${chosen.length} runs on ${inputs} cases → ${lines.length} ${unit} examples, at most ~${tokens} tokens; training estimate $${estimate.toFixed(2)}`); if (budget.spentUsd + estimate > budget.limitUsd) { throw new BudgetExceeded(`Training would cost about $${estimate.toFixed(2)}, past the $${budget.limitUsd.toFixed(2)} budget ($${budget.spentUsd.toFixed(4)} spent).`); } const logDir = path.join(this.workspaceDir, ".machine", "training", `${this.name.replace(/\//g, "-")}-${id}`); const data = path.join(logDir, "data.jsonl"); await writeLines(data, lines); const trainer = options.trainer ?? tinkerSft(); log(`training ${baseModel} for ${epochs} epoch${epochs === 1 ? "" : "s"} → ${path.relative(this.workspaceDir, logDir)}`); const effort = effortScalar(this.#options.reasoningEffort); const result = await trainer({ data, logDir, baseModel, epochs, learningRate: options.learningRate, batchSize: options.batchSize, effort, log, signal: options.signal }); const trainedCost = result.tokens ? trainingCost(baseModel, result.tokens, 1) : estimate; budget.charge({ costUsd: trainedCost }, "Training"); priceCheckpointAs(result.checkpoint, baseModel); log(`checkpoint ${result.checkpoint}`);
// The candidate against the live version, on cases neither trained on. const candidate = withModel(def, result.checkpoint); const [liveRuns, candidateRuns] = await this.#holdoutRuns(workspace, live, def, candidate, `candidate:${id}`, holdout, trials, concurrency); const comparison = comparePaired(holdout, liveRuns, candidateRuns, log); const report: CandidateReport = { kind: "train", machine: this.name, live: { version: live.id, model: def.model }, candidate: { model: result.checkpoint, instructionsChanged: false }, data: { runs: chosen.length, examples: lines.length, inputs, tokens, byModel, ...(unit === "whole-run" ? { wholeRuns: true } : {}) }, training: { baseModel, checkpoint: result.checkpoint, steps: result.steps, epochs, ...(options.learningRate !== undefined ? { learningRate: options.learningRate } : {}), firstLoss: result.firstLoss, lastLoss: result.lastLoss, logDir: path.relative(this.workspaceDir, logDir), costUsd: trainedCost, }, holdout: summarizeComparison(comparison, trials), decision: promotionGate(comparison, options.minZ ?? 2), spentUsd: budget.spentUsd, }; return this.#saveCandidate({ id, machine: this.name, createdAt: new Date().toISOString(), from: live.id, change: { model: result.checkpoint, baseModel }, report, status: "pending" }); }
/** * Rewrite the instructions from the live version's failures (the built-in * optimizer machine) and measure the rewrite on the training and held-out * cases. Returns a Candidate; the live machine is unchanged. */ async optimize(options: OptimizeOptions): Promise<Candidate> { const budget = new Budget(options.budget); const workspace = this.#workspace(budget); const live = await this.version(); const def = this.#definition; const cases = options.cases ?? (await this.#cases("cases.json")); const holdout = options.holdout ?? (await this.#cases("holdout.json")); if (!cases?.length || !holdout?.length) throw new Error(`optimize needs cases and held-out cases (cases.json and holdout.json in machines/${this.name}/).`); const trials = options.trials ?? 2; const concurrency = this.#options.concurrency ?? 4; const id = newRunId(); const log = (line: string) => this.#log(line); const [liveRuns] = await runMatrix(workspace, [{ machine: def, version: live.id }], cases, trials, concurrency, undefined, log); const optimizerModel = options.optimizerModel ?? baseModelOf(def.model) ?? def.model; const optRun = await workspace.run(withModel(optimizer, optimizerModel), failureReport(def, liveRuns), { evaluate: true }); const proposal = parseProposal(optRun.output); if (!proposal || optRun.scorecard?.total !== 1) throw new Error(`The optimizer made no usable proposal (run ${optRun.id}).`); const candidate = withInstructions(def, proposal); const label = `candidate:${id}`; const [candidateRuns] = await runMatrix(workspace, [{ machine: candidate, version: label }], cases, trials, concurrency, undefined, log); const train = comparePaired(cases, liveRuns, candidateRuns, log); const [liveHold, candHold] = await this.#holdoutRuns(workspace, live, def, candidate, label, holdout, trials, concurrency); const held = comparePaired(holdout, liveHold, candHold, log); const report: CandidateReport = { kind: "optimize", machine: this.name, live: { version: live.id, model: def.model }, candidate: { model: def.model, instructionsChanged: true }, train: summarizeComparison(train, trials), holdout: summarizeComparison(held, trials), decision: decide(train, held, { minZ: options.minZ ?? 2 }), spentUsd: budget.spentUsd, }; return this.#saveCandidate({ id, machine: this.name, createdAt: new Date().toISOString(), from: live.id, change: { instructions: proposal }, report, status: "pending" }); }
/** * A change made by hand (another model, new instructions) as a candidate: * measured against the live version on held-out cases like a trained * checkpoint, and committed through the same gate. */ async propose(change: { model?: string; baseModel?: string; instructions?: string }, options: { budget: number; holdout?: Case[]; trials?: number; minZ?: number }): Promise<Candidate> { if (change.model === undefined && change.instructions === undefined) throw new Error("propose needs a model or instructions."); const budget = new Budget(options.budget); const workspace = this.#workspace(budget); const live = await this.version(); const def = this.#definition; const holdout = options.holdout ?? (await this.#cases("holdout.json")); if (!holdout?.length) throw new Error(`propose needs held-out cases: pass holdout, or add machines/${this.name}/holdout.json.`); if (change.model && change.baseModel) priceCheckpointAs(change.model, change.baseModel); let candidate = def; if (change.instructions !== undefined) candidate = withInstructions(candidate, change.instructions.trim()); if (change.model !== undefined) candidate = withModel(candidate, change.model); const trials = options.trials ?? 2; const id = newRunId(); const [liveRuns, candidateRuns] = await this.#holdoutRuns(workspace, live, def, candidate, `candidate:${id}`, holdout, trials, this.#options.concurrency ?? 4); const comparison = comparePaired(holdout, liveRuns, candidateRuns, (l) => this.#log(l)); const report: CandidateReport = { kind: "propose", machine: this.name, live: { version: live.id, model: def.model }, candidate: { model: candidate.model, instructionsChanged: candidate.ownInstructions !== def.ownInstructions }, holdout: summarizeComparison(comparison, trials), decision: promotionGate(comparison, options.minZ ?? 2), spentUsd: budget.spentUsd, }; return this.#saveCandidate({ id, machine: this.name, createdAt: new Date().toISOString(), from: live.id, change, report, status: "pending" }); }
/** * Have an author model write a child of this machine that specializes in * `focus`: machines/<name>/<slug>/ with defineChild, 1-3 new criteria with * graders, instructions, and rehearsal cases (cases.json to train on, * holdout.json for the promotion gate). The child is validated (it * typechecks, loads, and its check graders run) and case fixtures are * checked by running their tests. Returns what was written; load the child * with machine.load(result.name) to rehearse and train it. */ async specialize(options: Omit<SpecializeOptions, "authorModel" | "log"> & { budget: number; authorModel?: string }): Promise<SpecializeResult> { if (this.#persist !== "files") throw new Error("specialize writes a child directory, so it needs a machine loaded from machines/ (machine.load)."); const authorModel = options.authorModel ?? DEFAULT_AUTHOR_MODEL; const workspace = this.#workspace(new Budget(options.budget), authorModel); return specialize(workspace, this.workspaceDir, this.#definition, { ...options, authorModel, sandboxes: options.sandboxes ?? this.#options.sandboxes, log: (line) => this.#log(line), }); }
/** Stored candidates, newest first. */ async candidates(options: { status?: CandidateRecord["status"] } = {}): Promise<Candidate[]> { const dir = this.#candidateDir(); let files: string[]; try { files = (await readdir(dir)).filter((f) => f.endsWith(".json")).sort().reverse(); } catch { return []; } const records = await Promise.all(files.map(async (f) => JSON.parse(await readFile(path.join(dir, f), "utf8")) as CandidateRecord)); return records.filter((r) => !options.status || r.status === options.status).map((r) => this.#candidate(r)); }
/** One stored candidate by id (or "latest" for the newest pending one). */ async candidate(id = "latest"): Promise<Candidate> { if (id === "latest") { const [newest] = await this.candidates({ status: "pending" }); if (!newest) throw new Error(`${this.name} has no pending candidate; run train() or optimize() first.`); return newest; } nameSegments(id, "candidate id"); let record: CandidateRecord; try { record = JSON.parse(await readFile(path.join(this.#candidateDir(), `${id}.json`), "utf8")); } catch { throw new Error(`${this.name} has no candidate ${id}.`); } return this.#candidate(record); }
/** Release harness resources (Pi and opencode servers). */ async close(): Promise<void> { for (const h of this.#harnesses.values()) await h.close?.(); this.#harnesses.clear(); }
async #ensureVersion(): Promise<VersionEntry> { const history = await this.#versions.read(this.name); const found = findVersion(history, this.#definition); if (found) { this.#history = liveVersion(history)?.id === found.id ? history : { ...history, live: [...history.live, found.id] }; if (this.#history !== history) await this.#versions.write(this.#history); return found; } const source: VersionSource = history.versions.length ? { kind: "recorded", note: "changed outside machine" } : { kind: "initial" }; const { history: next, entry } = addVersion(history, this.#definition, source, { baseModel: baseModelOf(this.#definition.model) }); await this.#versions.write(next); this.#history = next; return entry; }
/** Make `next` the running definition; for a loaded machine, write its files. */ async #apply(next: Machine, baseModel: string | undefined): Promise<void> { if (this.#persist === "files") { const dir = path.join(this.workspaceDir, "machines", ...nameSegments(this.name, "machine name")); if (next.ownInstructions !== this.#definition.ownInstructions) { const file = path.join(dir, INSTRUCTIONS_FILE); const onDisk = await readFile(file, "utf8").then((t) => t.trim(), () => undefined); if (onDisk !== this.#definition.ownInstructions) throw new Error(`${this.name}'s instructions do not come from ${file}; load them with instructionsFrom(import.meta.url) to change them here.`); await writeFile(file, `${next.ownInstructions}\n`); } if (next.model !== this.#definition.model) { if (!usesModelFile(dir)) throw new Error(`machines/${this.name}/machine.ts must take its model from modelFrom(import.meta.url, …) to change it here.`); await writeFile(path.join(dir, MODEL_FILE), `${JSON.stringify({ model: next.model, ...(baseModel && next.model.startsWith("tinker://") ? { baseModel } : {}) }, null, 2)}\n`); } } this.#definition = next; }
/** Graded held-out runs of the live version (stored ones reused, newest first) and fresh runs of the candidate. */ async #holdoutRuns(workspace: Workspace, live: VersionEntry, def: Machine, candidate: Machine, label: string, holdout: Case[], trials: number, concurrency: number): Promise<[Run[][], Run[][]]> { const stored = (await this.store.listRuns(this.name, { limit: 100_000 })).filter( (r) => r.version === live.id && r.model === def.model && r.scorecard && !needsGrade(r, def.rubric) && r.history.length === 0, ); const liveRuns = holdout.map((c) => stored.filter((r) => r.input === c.input).slice(0, trials)); const reused = liveRuns.reduce((n, r) => n + r.length, 0); this.#log(`held out: ${holdout.length} cases × ${trials}; reusing ${reused} stored run${reused === 1 ? "" : "s"} of ${live.id}`); for (const [i, c] of holdout.entries()) { const missing = trials - liveRuns[i].length; if (missing > 0) liveRuns[i].push(...(await runMatrix(workspace, [{ machine: def, version: live.id }], [c], missing, concurrency, undefined, (l) => this.#log(l)))[0][0]); } const [candidateRuns] = await runMatrix(workspace, [{ machine: candidate, version: label }], holdout, trials, concurrency, undefined, (l) => this.#log(l)); return [liveRuns, candidateRuns]; }
#candidateDir(): string { return path.join(this.workspaceDir, ".machine", "candidates", ...nameSegments(this.name, "machine name")); }
async #saveCandidate(record: CandidateRecord): Promise<Candidate> { await mkdir(this.#candidateDir(), { recursive: true, mode: 0o700 }); await writeFile(path.join(this.#candidateDir(), `${record.id}.json`), `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 }); return this.#candidate(record); }
#candidate(record: CandidateRecord): Candidate { const handle = this; const save = (r: CandidateRecord) => this.#saveCandidate(r); return { id: record.id, machine: record.machine, from: record.from, change: record.change, report: record.report, status: record.status, async commit(options = {}) { const current = await handle.candidate(record.id); if (current.status !== "pending") throw new Error(`Candidate ${record.id} is already ${current.status}.`); const live = await handle.version(); if (live.id !== record.from && !options.force) { throw new CommitRefused(`The live version is now ${live.id}, but candidate ${record.id} was measured against ${record.from}. Measure it again, or force it (commit({ force: true }), machine commit --force).`, record.report); } if (!record.report.decision.keep && !options.force) { throw new CommitRefused(`Not committed: ${record.report.decision.reason}. Force it with commit({ force: true }) or machine commit --force.`, record.report); } const source: VersionSource = record.report.kind === "train" ? { kind: "train", candidate: record.id, checkpoint: record.change.model! } : record.report.kind === "optimize" ? { kind: "optimize", candidate: record.id } : { kind: "propose", candidate: record.id }; const h = record.report.holdout; const note = `held out ${h.live.mean.toFixed(3)} → ${h.candidate.mean.toFixed(3)}${options.force && !record.report.decision.keep ? " (forced past the gate)" : ""}`; const entry = await handle.update({ ...record.change, note }, source); // The candidate's held-out runs were made by what is now this version. for (const run of await handle.store.listRuns(handle.name, { limit: 100_000 })) { if (run.version === `candidate:${record.id}`) await handle.store.putRun({ ...run, version: entry.id }); } await save({ ...record, status: "committed", version: entry.id }); return entry; }, }; }
async #cases(file: string): Promise<Case[] | undefined> { if (this.#persist !== "files" || !(await hasCaseFile(this.workspaceDir, this.#definition, file))) return undefined; return loadCases(this.workspaceDir, this.#definition, file); }
#workspace(budget: Budget | undefined, model = this.model): Workspace { const chosen = this.#options.harness; let harness: Harness; if (typeof chosen === "object") harness = chosen; else { const kind = chosen ?? process.env.MACHINE_HARNESS ?? defaultHarness(model); harness = this.#harnesses.get(kind) ?? lazyHarness(kind); this.#harnesses.set(kind, harness); } const kind = harness.name; const judgeModel = this.#options.judgeModel ?? process.env.MACHINE_JUDGE_MODEL ?? (kind === "tinker" || kind === "pi" ? DEFAULT_TINKER_JUDGE : undefined); return createWorkspace({ name: this.#options.workspaceName ?? process.env.MACHINE_WORKSPACE ?? "local", root: path.join(this.workspaceDir, ".machine"), harness, judgeModel, judgeEffort: this.#options.judgeEffort ?? (kind === "tinker" ? "medium" : undefined), reasoningEffort: this.#options.reasoningEffort, budget, store: this.store, sandboxes: this.#options.sandboxes, events: this.#events, }); }
#log(line: string) { this.#options.log?.(line); }}
function summarizeComparison(c: Comparison, trials: number): ComparisonSummary { return { cases: c.cases.length, trials, live: c.a, candidate: c.b, diff: c.diff, stderr: c.stderr, better: c.better, worse: c.worse, tied: c.tied, perCase: c.cases.map((x) => ({ input: x.input, live: x.a.mean, candidate: x.b.mean })), };}
/** A candidate report as text, for terminals. */export function formatReport(report: CandidateReport): string { const out: string[] = []; const pct = (x: number) => `${Math.round(x * 100)}%`; const signed = (x: number) => `${x >= 0 ? "+" : ""}${x.toFixed(3)}`; out.push(`${report.kind === "train" ? "Trained" : report.kind === "optimize" ? "Optimized" : "Proposed"} candidate for ${report.machine}, measured against the live ${report.live.version} (${report.live.model})`); if (report.data) { const models = Object.entries(report.data.byModel).map(([m, n]) => `${n} by ${m}`).join(", "); out.push(` data ${report.data.runs} good runs on ${report.data.inputs} cases (${models}) → ${report.data.examples} ${report.data.wholeRuns ? "whole-run" : "step"} examples`); } if (report.training) { const t = report.training; const loss = t.firstLoss !== undefined && t.lastLoss !== undefined ? `, loss ${t.firstLoss.toFixed(3)} → ${t.lastLoss.toFixed(3)}` : ""; out.push(` training ${t.baseModel}, ${t.epochs} epoch${t.epochs === 1 ? "" : "s"}${t.learningRate ? ` at lr ${t.learningRate}` : ""}, ${t.steps} steps${loss}, $${t.costUsd.toFixed(2)}`); out.push(` checkpoint ${t.checkpoint}`); } const section = (title: string, s: ComparisonSummary) => { out.push(` ${title}: ${s.cases} cases × ${s.trials} trial${s.trials === 1 ? "" : "s"} each`); out.push(` ${"".padEnd(12)} ${"live".padStart(7)} ${"candidate".padStart(10)}`); out.push(` ${"mean".padEnd(12)} ${s.live.mean.toFixed(3).padStart(7)} ${s.candidate.mean.toFixed(3).padStart(10)} ${signed(s.diff)} ± ${s.stderr.toFixed(3)}`); out.push(` ${"pass rate".padEnd(12)} ${pct(s.live.passRate).padStart(7)} ${pct(s.candidate.passRate).padStart(10)}`); for (const id of Object.keys(s.live.criteria)) { out.push(` ${id.padEnd(12)} ${s.live.criteria[id].toFixed(3).padStart(7)} ${(s.candidate.criteria[id] ?? 0).toFixed(3).padStart(10)} ${signed((s.candidate.criteria[id] ?? 0) - s.live.criteria[id])}`); } out.push(` better on ${s.better} cases, worse on ${s.worse}, tied on ${s.tied}`); }; if (report.train) section("cases it learned from", report.train); section("held-out cases", report.holdout); out.push(` decision ${report.decision.keep ? "PASS" : "HOLD"}: ${report.decision.reason}`); out.push(` spent $${report.spentUsd.toFixed(4)}`); return out.join("\n");}
/** Load and create machines. */export const machine = { /** A machine from machines/<name>/machine.ts in the workspace, with its versions in machines/<name>/versions.json. */ async load(name: string, options: MachineOptions = {}): Promise<MachineHandle> { const workspaceDir = path.resolve(options.workspace ?? process.cwd()); const definition = await loadMachineFrom(workspaceDir, name); const versions = options.versions ?? versionFile(path.join(workspaceDir, "machines", ...nameSegments(name, "machine name"), "versions.json")); return new MachineHandle(definition, { machine: name, versions: [], live: [] }, versions, "files", { ...options, workspace: workspaceDir }); },
/** * A machine defined in code, without a directory. Its versions are kept in * .machine/versions/<name>.json; once it has some, the live version's * instructions and model are used over the spec's (change them with update()). */ async new<R extends Rubric>(spec: MachineSpec<R> | Machine<R>, options: MachineOptions = {}): Promise<MachineHandle> { const workspaceDir = path.resolve(options.workspace ?? process.cwd()); let definition: Machine = "ownInstructions" in spec ? spec : defineMachine(spec); const versions = options.versions ?? versionFile(path.join(workspaceDir, ".machine", "versions", `${nameSegments(definition.name, "machine name").join("--")}.json`)); const history = await versions.read(definition.name); const live = liveVersion(history); if (live) { if (live.baseModel) priceCheckpointAs(live.model, live.baseModel); definition = withModel(withInstructions(definition, live.instructions), live.model); } return new MachineHandle(definition, history, versions, "history", { ...options, workspace: workspaceDir }); },};