Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676// A machine writes its own specialist. An author model is given a parent// machine (its job, instructions, tools, rubric, and source) and a focus, and// writes a child machine for that focus: machine.ts with defineChild, new// criteria with their graders, and instructions. Then it writes rehearsal// cases for the focus: training cases (what `machine train` samples and// learns from) and held-out cases (what the promotion gate measures on).//// Nothing the author writes is trusted. The child must typecheck, load, be a// child of the parent on the requested model, and its check graders must run// on sample runs; failures go back to the author for a bounded number of// retries. A case with a test must fail before its reference fix and pass// after it, in the same sandbox runs use; cases that do not are discarded.
import { execFile } from "node:child_process";import { existsSync } from "node:fs";import { mkdir, readFile, rm, writeFile } from "node:fs/promises";import os from "node:os";import path from "node:path";import { exists, loadCases, type Case } from "./cases.ts";import { check, defineEvaluator } from "./evaluator.ts";import { defineMachine, type Machine } from "./machine.ts";import { defineRubric } from "./rubric.ts";import { localSandboxes, type SandboxProvider } from "./sandbox.ts";import { nameSegments } from "./store.ts";import { mapLimit } from "./trials.ts";import type { Message, Run } from "./types.ts";import type { Workspace } from "./workspace.ts";
/** The default author: a strong Tinker model that writes code and tests well. */export const DEFAULT_AUTHOR_MODEL = "thinkingmachines/Inkling";
const authorRubric = defineRubric({ parsed: { description: "Replies in the requested tagged format." },});
/** The author: writes a child machine, or cases for one. Its model is set at use. */export const specializer = defineMachine({ name: "specializer", intent: "Write a specialized child of a machine, and rehearsal cases for it.", instructions: [ "You design specialized AI agents, called machines. A machine is a TypeScript spec: instructions, a model, tools, and a rubric whose criteria are each graded by a deterministic check or a model judge.", "A child machine specializes a parent. It inherits every parent criterion with its grader, the parent's instructions (its own follow them), its tools, and its model unless it names its own. It adds criteria; it never redefines an inherited one.", "Write code that compiles under strict TypeScript, imports with .ts extensions, and uses only what the files you are shown export.", "Reply in exactly the tagged format asked for, with nothing else outside the tags.", ].join("\n"), model: "set-at-use", rubric: authorRubric, evaluator: defineEvaluator(authorRubric, { parsed: check("tagged", (run) => /<(machine_ts|case)>/.test(run.output)) }),});
export type SpecializeOptions = { /** What to specialize in, e.g. "Python asyncio and concurrency bugs". */ focus: string; /** The child's model. Default: the parent's. */ model?: string; /** Model that writes the child and its cases. */ authorModel: string; /** Child directory name under the parent. Default: one the author picks. */ slug?: string; /** Training cases to keep. Default 20. */ trainCases?: number; /** Held-out cases to keep. Default 8. */ holdoutCases?: number; /** Cases asked for per author call. Default 3. */ batchSize?: number; /** Times the author may fix a child that fails validation. Default 2. */ retries?: number; /** Reasoning effort for the author. Default "medium". */ authorEffort?: string | number; /** Author calls at once while writing cases. Default 4. */ concurrency?: number; /** Replace an existing child directory. Default false. */ overwrite?: boolean; /** Rewrite only the child (machine.ts, rubric.ts, instructions.md), keeping the cases already in its directory (needs slug). */ reuseCases?: boolean; /** Sandboxes for checking case fixtures. Default: local ones under .machine/sandboxes. */ sandboxes?: SandboxProvider; /** Validate the written child (typecheck, load, run graders). Default: validateChild. Tests pass a fake. */ validate?: (input: ChildCheckInput) => Promise<ChildCheck>; log?: (line: string) => void;};
export type ChildCheckInput = { workspaceDir: string; dir: string; name: string; parent: Machine; model: string; sampleRuns: Run[] };
/** What validating a written child found: its shape if it loaded, and every problem. */export type ChildCheck = { errors: string[]; /** Doubts worth one more look by the author (e.g. a check that scores every sample run the same); not fatal. */ warnings?: string[]; machine?: { name: string; parent?: string; model: string; criteria: Array<{ id: string; description: string; weight: number; grader: string }> };};
export type CaseDraft = Case & { test?: { command: string; args: string[] }; solution?: Record<string, string>; /** Held-back test files: run with the visible ones after the fix, never shown to the agent. */ hidden?: Record<string, string>; topic?: string;};
export type SpecializeResult = { name: string; dir: string; focus: string; model: string; authorModel: string; inherited: Array<{ id: string; grader: string }>; added: Array<{ id: string; description: string; weight: number; grader: string }>; attempts: number; topics: string[]; cases: number; holdout: number; /** Cases written and checked, and those discarded with the reason. */ drafted: number; discarded: Array<{ input: string; reason: string }>; /** Whether case fixtures were checked by running their tests. */ testedFixtures: boolean; /** Whether each case has held-back tests (hidden.json) for the child's hiddenTests grader. */ hiddenTests: boolean; /** Whether the cases were kept from an earlier run (reuseCases) rather than written now. */ reusedCases?: boolean; spentUsd: number;};
/** Write a child of `parent` for `focus`, validate it, and write its cases. */export async function specialize(workspace: Workspace, workspaceDir: string, parent: Machine, options: SpecializeOptions): Promise<SpecializeResult> { const log = options.log ?? (() => {}); const startSpent = workspace.budget?.spentUsd ?? 0; const model = options.model ?? parent.model; const author = { ...specializer, model: options.authorModel }; const effort = options.authorEffort ?? "medium"; const parentDir = path.join(workspaceDir, "machines", ...nameSegments(parent.name, "machine name")); const parentCases = await loadCases(workspaceDir, parent).catch(() => [] as Case[]); const withFiles = parentCases.some((c) => c.files && Object.keys(c.files).length);
// 1. The child machine, validated, with retries on what the author got wrong. const prompt = await designPrompt(workspaceDir, parent, parentDir, options.focus, model, options.slug, parentCases); let history: Message[] = []; let input = prompt; let design: ChildDesign | undefined; let checked: ChildCheck | undefined; let dir = ""; let name = ""; const retries = options.retries ?? 2; let attempts = 0; let warned = false; let topics: string[] = []; const sampleRuns = await sampleRunsOf(workspace, parent, parentCases); const CASE_FILES = ["cases.json", "holdout.json", "hidden.json"]; let saved: Record<string, string> = {}; if (options.reuseCases) { if (!options.slug) throw new Error("reuseCases needs the child's slug."); const existing = path.join(parentDir, options.slug); for (const f of ["machine.ts", "rubric.ts", "instructions.md", ...CASE_FILES]) { if (existsSync(path.join(existing, f))) saved[f] = await readFile(path.join(existing, f), "utf8"); } if (!saved["cases.json"] || !saved["holdout.json"]) throw new Error(`reuseCases: machines/${parent.name}/${options.slug}/ has no cases.json and holdout.json to keep.`); } const restoreCases = async () => { for (const f of CASE_FILES) if (saved[f] !== undefined) await writeFile(path.join(dir, f), saved[f]); }; for (; attempts <= retries; ) { attempts++; log(`author (${options.authorModel}) writing ${parent.name}/<child> for "${options.focus}"${attempts > 1 ? `, attempt ${attempts}` : ""}`); const run = await workspace.run(author, input, { history, reasoningEffort: effort }); history = [...history, ...run.trajectory]; const parsed = parseDesign(run.output, options.slug); if (typeof parsed === "string") { checked = { errors: [parsed] }; } else { design = parsed; if (design.topics.length) topics = design.topics; name = `${parent.name}/${design.slug}`; const next = path.join(parentDir, design.slug); if (dir && dir !== next) await rm(dir, { recursive: true, force: true }); if (dir !== next && existsSync(next) && !options.overwrite && !options.reuseCases) throw new Error(`${path.relative(workspaceDir, next)} exists; choose another slug or pass overwrite.`); dir = next; await rm(dir, { recursive: true, force: true }); await mkdir(dir, { recursive: true }); await writeFile(path.join(dir, "machine.ts"), ensureNewline(design.machineTs)); await writeFile(path.join(dir, "rubric.ts"), ensureNewline(design.rubricTs)); await writeFile(path.join(dir, "instructions.md"), ensureNewline(design.instructions)); await restoreCases(); checked = await (options.validate ?? validateChild)({ workspaceDir, dir, name, parent, model, sampleRuns }); } const warnings = checked.errors.length === 0 && !warned && attempts <= retries ? (checked.warnings ?? []) : []; if (checked.errors.length === 0 && warnings.length === 0) break; if (checked.errors.length) { log(` rejected: ${checked.errors.map((e) => e.split("\n")[0]).join("; ")}`); input = `Your child machine failed validation:\n${checked.errors.map((e) => `- ${e}`).join("\n")}\n\nFix every problem and reply again with all five tags (<slug>, <rubric_ts>, <machine_ts>, <instructions_md>, <topics>), complete.`; } else { warned = true; log(` asked to reconsider: ${warnings.join("; ")}`); input = `Your child machine is valid, but check these:\n${warnings.map((w) => `- ${w}`).join("\n")}\n\nA check should tell a good run from a bad one on the focus. Revise it if it cannot, or keep it if it is right. Reply again with all five tags, complete.`; } } if (!design || !checked?.machine || checked.errors.length) { if (dir) await rm(dir, { recursive: true, force: true }); if (options.reuseCases) { // Put the child back as it was. const before = path.join(parentDir, options.slug!); await mkdir(before, { recursive: true }); for (const [f, text] of Object.entries(saved)) await writeFile(path.join(before, f), text); } throw new Error(`The author's child machine failed validation after ${attempts} attempt${attempts === 1 ? "" : "s"}: ${checked?.errors.join("; ")}`); } log(` ${name} validated: typechecks, loads as a child of ${parent.name} on ${model}, graders run`); if (!topics.length) topics = [options.focus]; // A child graded by held-back tests needs them for every case. const usesHidden = withFiles && /\bhiddenTests\(/.test(design.machineTs);
if (options.reuseCases) { if (usesHidden && !saved["hidden.json"]) throw new Error("reuseCases: the new child grades held-back tests, but the kept cases have no hidden.json."); const train = JSON.parse(saved["cases.json"]) as Case[]; const held = JSON.parse(saved["holdout.json"]) as Case[]; log(` kept the ${train.length} training and ${held.length} held-out cases already in ${path.relative(workspaceDir, dir)}`); return { ...result(train.length, held.length, train.length + held.length, []), reusedCases: true }; }
// 2. Rehearsal cases, written in batches over the author's topics, each fixture checked; more batches while too few pass. const want = (options.trainCases ?? 20) + (options.holdoutCases ?? 8); const batchSize = options.batchSize ?? 3; const maxBatches = Math.ceil(want / batchSize) * 3; const sandboxes = options.sandboxes ?? localSandboxes(path.join(workspaceDir, ".machine", "sandboxes")); const example = parentCases.find((c) => c.files && Object.keys(c.files).length) ?? parentCases[0]; const drafts: CaseDraft[] = []; const kept: CaseDraft[] = []; const discarded: SpecializeResult["discarded"] = []; const seen = new Set<string>(); let batchesRun = 0; log(`author writing ${want} cases in batches of ${batchSize} over ${topics.length} topics; each test must fail before its fix and pass after`); while (kept.length < want && batchesRun < maxBatches) { const wave = Math.min(maxBatches - batchesRun, Math.max(1, Math.ceil(((want - kept.length) * 1.4) / batchSize))); const fresh: CaseDraft[] = []; await mapLimit( Array.from({ length: wave }, (_, k) => batchesRun + k), options.concurrency ?? 4, async (i) => { const batchTopics = [topics[(2 * i) % topics.length], topics[(2 * i + 1) % topics.length]]; let run: Run; try { run = await workspace.run(author, casesPrompt(options.focus, design!, batchTopics, batchSize, example, withFiles, usesHidden), { reasoningEffort: effort }); } catch (error) { if (error instanceof Error && error.name === "BudgetExceeded") throw error; log(` batch ${i + 1}: ${error instanceof Error ? error.message : error}`); return; } for (const draft of parseCases(run.output)) fresh.push({ ...draft, topic: batchTopics.join(" / ") }); }, ); batchesRun += wave; await mapLimit(fresh, 4, async (draft) => { const duplicate = seen.has(draft.input); seen.add(draft.input); const problem = duplicate ? "duplicate input" : await checkCase(draft, sandboxes, withFiles, usesHidden); if (problem) { discarded.push({ input: draft.input, reason: problem }); log(` discarded: ${problem.slice(0, 120)} (${draft.input.slice(0, 60)})`); } else kept.push(draft); }); drafts.push(...fresh); log(` ${kept.length} of ${drafts.length} cases pass their checks after ${batchesRun} batches`); } kept.sort((a, b) => drafts.indexOf(a) - drafts.indexOf(b)); if (kept.length < 2) throw new Error(`Only ${kept.length} of ${drafts.length} cases passed their checks; ${discarded.slice(0, 3).map((d) => d.reason).join("; ")}`); const { train, holdout } = splitCases(kept, options.trainCases ?? 20, options.holdoutCases ?? 8); const plain = (c: CaseDraft): Case => (c.files && Object.keys(c.files).length ? { input: c.input, files: c.files } : { input: c.input }); await writeFile(path.join(dir, "cases.json"), `${JSON.stringify(train.map(plain), null, 2)}\n`); await writeFile(path.join(dir, "holdout.json"), `${JSON.stringify(holdout.map(plain), null, 2)}\n`); if (usesHidden) { const hidden = Object.fromEntries([...train, ...holdout].map((c) => [c.input, { files: c.hidden!, command: c.test!.command, args: c.test!.args }])); await writeFile(path.join(dir, "hidden.json"), `${JSON.stringify(hidden, null, 2)}\n`); } log(` kept ${kept.length} of ${drafts.length} cases: ${train.length} training, ${holdout.length} held out`);
return result(train.length, holdout.length, drafts.length, discarded);
function result(cases: number, held: number, drafted: number, discarded: SpecializeResult["discarded"]): SpecializeResult { const own = new Set(checked!.machine!.criteria.map((c) => c.id)); return { name, dir: path.relative(workspaceDir, dir), focus: options.focus, model, authorModel: options.authorModel, inherited: Object.keys(parent.rubric).map((id) => ({ id, grader: parent.evaluator[id].name })), added: checked!.machine!.criteria.filter((c) => own.has(c.id)), attempts, topics, cases, holdout: held, drafted, discarded, testedFixtures: withFiles, hiddenTests: usesHidden, spentUsd: (workspace.budget?.spentUsd ?? 0) - startSpent, }; }}
type ChildDesign = { slug: string; rubricTs: string; machineTs: string; instructions: string; topics: string[] };
function tag(text: string, name: string): string | undefined { const match = text.match(new RegExp(`<${name}>([\\s\\S]*?)</${name}>`)); return match ? stripFence(match[1]) : undefined;}
function stripFence(text: string): string { const trimmed = text.trim(); const fenced = trimmed.match(/^```[\w-]*\n([\s\S]*?)\n?```$/); return fenced ? fenced[1] : trimmed;}
function ensureNewline(text: string): string { return text.endsWith("\n") ? text : `${text}\n`;}
/** The author's child machine, or what is missing from its reply. */export function parseDesign(output: string, slug?: string): ChildDesign | string { const got = { slug: slug ?? tag(output, "slug")?.trim().toLowerCase(), rubricTs: tag(output, "rubric_ts"), machineTs: tag(output, "machine_ts"), instructions: tag(output, "instructions_md"), }; const missing = Object.entries(got).filter(([, v]) => !v).map(([k]) => k); if (missing.length) return `Your reply is missing ${missing.join(", ")}.`; if (!/^[a-z][a-z0-9-]{1,30}$/.test(got.slug!)) return `The slug "${got.slug}" must be 2-31 lowercase letters, digits, or hyphens, starting with a letter.`; const topics = (tag(output, "topics") ?? "").split("\n").map((t) => t.replace(/^[-*\d.)\s]+/, "").trim()).filter(Boolean); return { slug: got.slug!, rubricTs: got.rubricTs!, machineTs: got.machineTs!, instructions: got.instructions!, topics };}
/** Cases in an author reply: <case> blocks with <input>, <file path>, <solution path>, and <test>. */export function parseCases(output: string): CaseDraft[] { const out: CaseDraft[] = []; for (const block of output.matchAll(/<case>([\s\S]*?)<\/case>/g)) { const body = block[1]; const input = body.match(/<input>([\s\S]*?)<\/input>/)?.[1]?.trim(); if (!input) continue; const files: Record<string, string> = {}; for (const f of body.matchAll(/<file path="([^"]+)">\n?([\s\S]*?)<\/file>/g)) files[f[1].trim()] = ensureNewline(stripFence(f[2])); const solution: Record<string, string> = {}; for (const f of body.matchAll(/<solution path="([^"]+)">\n?([\s\S]*?)<\/solution>/g)) solution[f[1].trim()] = ensureNewline(stripFence(f[2])); const hidden: Record<string, string> = {}; for (const f of body.matchAll(/<hidden path="([^"]+)">\n?([\s\S]*?)<\/hidden>/g)) hidden[f[1].trim()] = ensureNewline(stripFence(f[2])); const testLine = body.match(/<test>([\s\S]*?)<\/test>/)?.[1]?.trim(); const [command, ...args] = testLine ? testLine.split(/\s+/) : []; out.push({ input, ...(Object.keys(files).length ? { files } : {}), ...(Object.keys(solution).length ? { solution } : {}), ...(Object.keys(hidden).length ? { hidden } : {}), ...(command ? { test: { command, args } } : {}), }); } return out;}
const isTestFile = (file: string) => /(^|\/)(test_[^/]*|[^/]*_test\.[a-z]+|[^/]*\.test\.[a-z]+|tests?\/.*)$/.test(file);
/** * Why a drafted case is unusable, or undefined if it is fine. With a test, * the test must fail on the starting files and pass once the reference * solution is applied, and the solution may not change a test file. */export async function checkCase(draft: CaseDraft, sandboxes: SandboxProvider, requireTest: boolean, requireHidden = false): Promise<string | undefined> { for (const file of [...Object.keys(draft.files ?? {}), ...Object.keys(draft.solution ?? {}), ...Object.keys(draft.hidden ?? {})]) { if (!file || path.isAbsolute(file) || file.split(/[\\/]/).includes("..")) return `file "${file}" is not a relative path inside the project`; } if (!requireTest) return undefined; if (!draft.files || !Object.keys(draft.files).length) return "no files"; if (!draft.test) return "no test command"; if (!draft.solution || !Object.keys(draft.solution).length) return "no reference solution"; const touched = Object.keys(draft.solution).filter(isTestFile); if (touched.length) return `the solution changes a test file (${touched.join(", ")})`; if (!Object.keys(draft.files).some(isTestFile)) return "no test file among the files"; const hidden = draft.hidden ?? {}; if (requireHidden && !Object.keys(hidden).length) return "no hidden tests"; if (Object.keys(hidden).some((f) => !isTestFile(f) || f in draft.files!)) return "a hidden test must be a new test file"; const run = async (files: Record<string, string>) => { const box = await sandboxes.create(`case-check-${Math.random().toString(36).slice(2, 10)}`); try { for (const [file, text] of Object.entries(files)) await box.writeFile(file, text); return await box.exec(draft.test!.command, draft.test!.args, { timeoutMs: 20_000 }); } finally { await box.dispose(); } }; const before = await run(draft.files); if (before.timedOut) return "the test timed out before the fix"; if (before.exitCode === 0) return "the test already passes before the fix"; const after = await run({ ...draft.files, ...draft.solution, ...hidden }); if (after.timedOut) return "the test timed out after the fix"; if (after.exitCode !== 0) return `the tests fail after the reference fix: ${(after.stderr || after.stdout).trim().split("\n").slice(-2).join(" ").slice(0, 200)}`; if (Object.keys(hidden).length) { // Hidden tests the starting files already pass check nothing the fix does. const sources = Object.fromEntries(Object.entries(draft.files).filter(([f]) => !isTestFile(f))); const hiddenBefore = await run({ ...sources, ...hidden }); if (hiddenBefore.exitCode === 0 && !hiddenBefore.timedOut) return "the hidden tests already pass before the fix"; } return undefined;}
/** Spread held-out cases evenly through the kept cases (so each topic batch can land in both). */export function splitCases<T>(kept: T[], trainCount: number, holdoutCount: number): { train: T[]; holdout: T[] } { const total = Math.min(kept.length, trainCount + holdoutCount); const nHoldout = Math.min(holdoutCount, Math.max(1, Math.round((total * holdoutCount) / (trainCount + holdoutCount)))); const step = total / nHoldout; const holdoutIdx = new Set(Array.from({ length: nHoldout }, (_, i) => Math.floor(i * step + step / 2))); const train: T[] = []; const holdout: T[] = []; for (let i = 0; i < total; i++) (holdoutIdx.has(i) ? holdout : train).push(kept[i]); return { train, holdout };}
/** Runs to try the child's check graders on: the parent's stored runs, and an empty run per parent case. */async function sampleRunsOf(workspace: Workspace, parent: Machine, cases: Case[]): Promise<Run[]> { // Some runs that met every criterion and some that did not, so a check that cannot tell them apart shows. const graded = (await workspace.store.listRuns(parent.name, { limit: 200 })).filter((r) => r.harness !== "echo" && r.scorecard && !r.conversation); const stored = [...graded.filter((r) => r.scorecard!.total === 1).slice(0, 3), ...graded.filter((r) => r.scorecard!.total < 1).slice(0, 3)]; const empty = cases.slice(0, 3).map( (c, i): Run => ({ id: `sample-${i}`, machine: parent.name, workspace: workspace.name, input: c.input, ...(c.files ? { files: c.files } : {}), history: [], trajectory: [{ role: "user", content: c.input }, { role: "assistant", content: "Done." }], output: "Done.", model: parent.model, harness: "sample", startedAt: "", finishedAt: "", }), ); return [...stored, ...empty];}
async function designPrompt(workspaceDir: string, parent: Machine, parentDir: string, focus: string, model: string, slug: string | undefined, cases: Case[]): Promise<string> { const read = async (file: string) => ((await exists(file)) ? (await readFile(file, "utf8")).trim() : undefined); const parentSource = await read(path.join(parentDir, "machine.ts")); const parentRubric = await read(path.join(parentDir, "rubric.ts")); // machines/<parent segments>/<slug>/machine.ts → the workspace root is one level per segment, plus machines/. const src = `${"../".repeat(nameSegments(parent.name, "machine name").length + 2)}src/index.ts`; const caseExample = cases.find((c) => c.files) ?? cases[0]; return [ `Write a child machine of "${parent.name}" that specializes in: ${focus}`, "", `<parent name="${parent.name}">`, `intent: ${parent.intent}`, `model: ${parent.model}`, `tools: ${parent.tools.map((t) => `${t.name} (${t.description.split(". ")[0]})`).join("; ") || "none"}`, "criteria (inherited by the child, with their graders; do not redefine them):", ...Object.entries(parent.rubric).map(([id, c]) => `- ${id} [${parent.evaluator[id].name}]: ${c.description}`), "instructions (the child's own follow these):", parent.instructions, "</parent>", "", `<parent_machine_ts path="machines/${parent.name}/machine.ts">`, parentSource ?? "(not available)", "</parent_machine_ts>", parentRubric ? `\n<parent_rubric_ts>\n${parentRubric}\n</parent_rubric_ts>` : "", caseExample ? `\nA case of the parent, for the kind of task it gets:\n${JSON.stringify(caseExample).slice(0, 1500)}` : "", "", "An existing child machine, for the shape (julia/fast, a child of julia):", "<example_rubric_ts>", EXAMPLE_RUBRIC, "</example_rubric_ts>", "<example_machine_ts>", EXAMPLE_MACHINE, "</example_machine_ts>", "", "Requirements:", `- The child is machines/${parent.name}/${slug ?? "<slug>"}/ and its name is exactly "${parent.name}/${slug ?? "<slug>"}". Both machine.ts and rubric.ts import the core from "${src}"; machine.ts imports the parent (its default export) and any of its helpers above from "../machine.ts".`, `- machine.ts default-exports defineChild(parent, {...}) with: name, intent (one sentence), instructions: instructionsFrom(import.meta.url), model: modelFrom(import.meta.url, "${model}"), rubric, and evaluator: defineEvaluator(rubric, {...}). Import every function you use.`, "- rubric.ts exports `rubric = defineRubric({...})` with 1 to 3 NEW criteria for the focus (ids in snake_case, none equal to an inherited id), each with a one-sentence description a grader can apply to a run, and a weight from 1 to 2. A good criterion names something a generalist gets wrong in this focus and a specialist gets right (a common wrong fix, a missing safeguard, a pitfall), not a restatement of the inherited criteria.", "- Prefer deterministic graders: check(name, (run) => ({ score, reason })) that inspect what the run actually did, using the parent's helpers above (e.g. the files it left behind and how they changed, the tool calls it made, the commands it ran and their results). Use judge() only for what code cannot decide; a judge reads the run and the criterion's description, so write that description precisely.", ...(parentSource && /export function hiddenTests\b/.test(parentSource) ? [ "- The parent exports hiddenTests: if your machine.ts uses `hiddenTests(import.meta.url)` as a criterion's grader, every case will come with held-back tests of the focus's edge cases (run against the files the agent leaves behind). That is the strongest measure of specialization: whether the fix handles the cases a specialist would, not only the reported symptom. Prefer it for one criterion, described as e.g. \"The final code also passes held-back tests of the focus's edge cases that the request implies.\"", ] : []), "- run.files are the files the run STARTED with (the case's fixture), not what it did: never grade them alone. What the run left behind and changed is what the parent's helpers above return (e.g. its final files and changed files).", "- A check must measure its criterion: it must be able to give a good run 1 and a bad run 0 on the focus. Never score the run's JSON, the user's input, or keywords anywhere in the transcript. Look at specific code (e.g. lines a change added to a file) or specific command results. Type the run parameter with the Run type, not any. A check must never throw; return score 0 with a reason instead.", "- instructions.md: short, concrete guidance for the focus, in the second person, that adds to the parent's instructions without repeating them. Never mention graders, rubrics, or tests of the agent.", "- topics: 8 to 12 distinct kinds of task in the focus, one per line, each specific enough to write a case about (they will be used to write training and test cases).", "", "Reply with exactly these tags:", slug ? `<slug>${slug}</slug>` : "<slug>a short directory name for the child, e.g. asyncio</slug>", "<rubric_ts>the complete rubric.ts</rubric_ts>", "<machine_ts>the complete machine.ts</machine_ts>", "<instructions_md>the complete instructions.md</instructions_md>", "<topics>one topic per line</topics>", ].join("\n");}
function casesPrompt(focus: string, design: ChildDesign, topics: string[], count: number, example: Case | undefined, withFiles: boolean, withHidden: boolean): string { const shape = withFiles ? [ "Each case is a small, self-contained project with a real bug or missing feature in the focus, and a request from the user.", "- 1 to 3 source files plus one test file (test_*.py with unittest, or *.test.js run by node). Keep each file under 60 lines. Standard library only: no pip packages, no network.", "- The test must FAIL on the files as given and PASS once the bug is fixed. It must finish within 10 seconds either way (for concurrency bugs, use timeouts such as asyncio.wait_for, never a test that can hang).", "- The test must check the behavior the request asks for, not implementation details, and must be deterministic: run it many times and get the same result.", "- <solution> gives the fixed version of each source file it changes (complete files). It never changes a test file.", "- <test> is the command that runs the tests, with no shell syntax, e.g. python3 -m unittest -q", "- The <input> is what a user would type: describe the symptom or the change, sometimes naming the failing test, sometimes not. Never reveal the fix.", ...(withHidden ? [ "- <hidden> gives held-back test files the agent never sees (named test_hidden_*.py or *.hidden.test.js). They are run together with the visible test after the agent finishes. They check the SAME behavior the request asks for, on the edge cases of the focus that the visible test does not cover: the ones a specialist handles and a generalist who only fixes the reported symptom misses.", "- Be fair: the request must state the general requirement (e.g. \"for any timezone\", \"for every input the docstring allows\"), so a careful engineer could pass the hidden tests from the request and the code alone. Never test behavior the request and the code's documented contract do not ask for.", "- The hidden tests must FAIL on the files as given and PASS with the reference solution, which must handle the edge cases properly, not just the visible test.", ] : []), ] : ["Each case is a request a user would make in the focus, with a clear, checkable goal."]; return [ `Write ${count} cases for a coding agent that specializes in: ${focus}`, `Cover these kinds of task: ${topics.join("; ")}. Make every case different from the others in the bug and the code.`, "", "The agent's own instructions for the focus:", design.instructions, "", ...shape, example ? `\nA case of the general agent, for the format of a request (yours are harder and in the focus):\n${JSON.stringify(example).slice(0, 1200)}` : "", "", "Reply with only the cases, each in this format:", "<case>", "<input>the user's request</input>", ...(withFiles ? [ '<file path="relative/path.py">', "complete file contents", "</file>", '<solution path="relative/path.py">', "complete fixed file contents", "</solution>", ...(withHidden ? ['<hidden path="test_hidden_x.py">', "complete held-back test file", "</hidden>"] : []), "<test>python3 -m unittest -q</test>", ] : []), "</case>", ].join("\n");}
const EXAMPLE_RUBRIC = `import { defineRubric } from "../../../src/index.ts";
export const rubric = defineRubric({ measures: { description: "The final program times its main computation with @time, @elapsed, or @timed, after a warm-up call, and the run output shows the timing.", weight: 1, }, performant: { description: "The final program is written for speed: type-stable functions, no non-constant globals in hot code, preallocated outputs. Judge the code, not claims about it.", weight: 2, },});`;
const EXAMPLE_MACHINE = `import { check, defineChild, defineEvaluator, instructionsFrom, judge, modelFrom } from "../../../src/index.ts";import julia from "../machine.ts";import { codeOnly, finalProgram } from "../tools.ts";import { rubric } from "./rubric.ts";
/** Times its work with a timing macro, and the output shows a timing. */export const measures = check("measures", (run) => { const program = finalProgram(run); if (!program) return { score: 0, reason: "no program ran successfully" }; if (!/@(?:time|elapsed|timed|btime)\\b/.test(codeOnly(program.code))) return { score: 0, reason: "no @time, @elapsed, or @timed in the code" }; return { score: 1, reason: "timed and printed" };});
export default defineChild(julia, { name: "julia/fast", intent: "Write Julia optimized for runtime, and measure it.", instructions: instructionsFrom(import.meta.url), model: modelFrom(import.meta.url, "Qwen/Qwen3.6-35B-A3B"), rubric, evaluator: defineEvaluator(rubric, { measures, performant: judge() }),});`;
function runProcess(command: string, args: string[], cwd: string, timeoutMs = 120_000): Promise<{ code: number; stdout: string; stderr: string }> { return new Promise((resolve) => { execFile(command, args, { cwd, timeout: timeoutMs, maxBuffer: 16 * 1024 * 1024 }, (error, stdout, stderr) => { const code = error ? (typeof (error as { code?: unknown }).code === "number" ? ((error as { code: number }).code as number) : 1) : 0; resolve({ code, stdout: String(stdout), stderr: String(stderr) || (error && !String(stderr) ? error.message : "") }); }); });}
/** * Typecheck the child (when the workspace has a tsconfig.json), load it in a * fresh process, check its shape, and run each check grader on the sample * runs. Returns every problem found. */export async function validateChild({ workspaceDir, dir, name, parent, model, sampleRuns }: ChildCheckInput): Promise<ChildCheck> { const errors: string[] = []; const rel = path.relative(workspaceDir, dir); if (existsSync(path.join(workspaceDir, "tsconfig.json"))) { const tsc = path.join(workspaceDir, "node_modules", ".bin", "tsc"); if (existsSync(tsc)) { const result = await runProcess(tsc, ["--noEmit", "-p", workspaceDir], workspaceDir); const mine = `${result.stdout}\n${result.stderr}`.split("\n").filter((l) => l.startsWith(rel + path.sep)); for (const line of mine.slice(0, 10)) errors.push(`TypeScript: ${line}`); } } const scratch = path.join(os.tmpdir(), `machine-child-check-${process.pid}-${Date.now()}`); await mkdir(scratch, { recursive: true }); try { await writeFile(path.join(scratch, "runs.json"), JSON.stringify(sampleRuns)); const script = ` const mod = await import(${JSON.stringify(path.join(dir, "machine.ts"))}); const m = mod.default; const runs = JSON.parse(await Bun.file(${JSON.stringify(path.join(scratch, "runs.json"))}).text()); const own = Object.keys(m.rubric).filter((id) => !m.parent || !(id in m.parent.rubric)); const graderErrors = []; const constant = []; for (const id of own) { const g = m.evaluator[id]; if (!g.name.startsWith("check:")) continue; const scores = []; for (const run of runs) { try { const s = await g.grade({ run: { ...run, machine: m.name }, criterionId: id, criterion: m.rubric[id] }); if (!(s.score >= 0 && s.score <= 1)) graderErrors.push(id + " gave score " + s.score); scores.push(s.score); } catch (e) { graderErrors.push(id + " threw on a sample run: " + (e && e.message || e)); break; } } if (scores.length > 2 && scores.every((x) => x === scores[0])) constant.push(id + " gives every one of " + scores.length + " sample runs of the parent the same score (" + scores[0] + "), including runs that changed nothing"); } console.log(JSON.stringify({ name: m.name, parent: m.parent && m.parent.name, model: m.model, criteria: own.map((id) => ({ id, description: m.rubric[id].description, weight: m.rubric[id].weight ?? 1, grader: m.evaluator[id].name })), graderErrors, constant })); `; const result = await runProcess(process.execPath, ["-e", script], workspaceDir); const line = result.stdout.trim().split("\n").at(-1) ?? ""; let shape: (NonNullable<ChildCheck["machine"]> & { graderErrors: string[]; constant: string[] }) | undefined; try { shape = JSON.parse(line); } catch { errors.push(`machine.ts does not load: ${(result.stderr || result.stdout).trim().split("\n").filter((l) => /error|Error/.test(l)).slice(0, 3).join(" ").slice(0, 600) || "no output"}`); } if (!shape) return { errors }; if (shape.name !== name) errors.push(`machine.ts is named "${shape.name}"; it must be "${name}".`); if (shape.parent !== parent.name) errors.push(`machine.ts must be defineChild(${parent.name}, ...) of the parent imported from "../machine.ts".`); if (shape.model !== model) errors.push(`the child's model is "${shape.model}"; use modelFrom(import.meta.url, "${model}").`); if (shape.criteria.length < 1 || shape.criteria.length > 3) errors.push(`the child adds ${shape.criteria.length} criteria; add 1 to 3.`); for (const e of shape.graderErrors) errors.push(`grader ${e}`); const source = await readFile(path.join(dir, "machine.ts"), "utf8"); if (!/modelFrom\(\s*import\.meta\.url/.test(source)) errors.push("take the model from modelFrom(import.meta.url, ...) so training can change it."); if (!/instructionsFrom\(\s*import\.meta\.url\s*\)/.test(source)) errors.push("take the instructions from instructionsFrom(import.meta.url)."); if (/\brun\.files\b/.test(source) && !/\b(finalFiles|changedFiles|addedLines|toolCalls|trajectory)\b/.test(source)) { errors.push("a check reads run.files, the files the run started with, and nothing the run did: grade what it changed (e.g. the parent's finalFiles or changedFiles) instead."); } const { graderErrors: _, constant, ...machine } = shape; return { errors, warnings: constant, machine }; } finally { await rm(scratch, { recursive: true, force: true }); }}
/** The specialization as a short transcript for the terminal. */export function formatSpecialization(r: SpecializeResult): string { const out: string[] = []; out.push(`${r.authorModel} wrote ${r.name} (${r.model}) for "${r.focus}"${r.attempts > 1 ? ` in ${r.attempts} attempts` : ""}`); out.push(` inherits ${r.inherited.map((c) => c.id).join(", ")}`); for (const c of r.added) out.push(` adds ${c.id.padEnd(18)} ${(c.grader.split(":")[0] || "judge").padEnd(5)} ${c.description.length > 70 ? `${c.description.slice(0, 69)}…` : c.description}`); const checkedNote = r.testedFixtures ? "every test fails before its fix and passes after" : ""; out.push(` cases ${r.cases} training + ${r.holdout} held out (${r.reusedCases ? `kept from before${checkedNote ? `; ${checkedNote}` : ""}` : `kept ${r.cases + r.holdout} of ${r.drafted}${checkedNote ? `; ${checkedNote}` : ""}`})`); if (r.hiddenTests) out.push(` hidden each case has held-back edge-case tests (hidden.json), never shown to the agent`); out.push(` files ${r.dir}/{machine.ts, rubric.ts, instructions.md, cases.json, holdout.json${r.hiddenTests ? ", hidden.json" : ""}}`); out.push(` spent $${r.spentUsd.toFixed(4)}`); out.push("next:"); out.push(` machine suite ${r.name} --trials 2 --budget 1 # rehearse: baseline on the training cases`); out.push(` machine train ${r.name} --samples 4 --budget 8 # fine-tune on the best rehearsals, gate on held-out`); return out.join("\n");}