Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130import assert from "node:assert/strict";import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises";import { pathToFileURL } from "node:url";import os from "node:os";import path from "node:path";import { after, test } from "node:test";import dev, { addedLines, changedFiles, editFileTool, finalFiles, hiddenTests, listFilesTool, readFileTool, searchTool, verifiedAfterChange } from "../machines/dev/machine.ts";import { localSandboxes, type Message, type Run } from "../src/index.ts";
const dirs: string[] = [];after(() => Promise.all(dirs.map((d) => rm(d, { recursive: true, force: true }))));
async function project() { const root = await mkdtemp(path.join(os.tmpdir(), "machine-dev-test-")); dirs.push(root); const sandbox = await localSandboxes(root, { isolation: "none" }).create("p"); const at = (p: string) => path.join(sandbox.workdir, p); await mkdir(at("src"), { recursive: true }); await mkdir(at("node_modules/x"), { recursive: true }); await writeFile(at("src/a.py"), "def add(a, b):\n return a - b\n"); await writeFile(at("README.md"), "add numbers\n"); await writeFile(at("node_modules/x/index.js"), "def add\n"); return { ctx: { sandbox }, at, root };}
test("dev declares its tools and rubric", () => { assert.deepEqual(dev.tools.map((t) => t.name), ["list_files", "search", "read_file", "edit_file", "write_file", "run"]); assert.deepEqual(Object.keys(dev.rubric), ["verified", "does_task", "focused", "acts_on_request"]);});
test("list_files and search skip dependency folders", async () => { const { ctx } = await project(); assert.equal(await listFilesTool.execute({}, ctx), "README.md\nsrc/a.py"); assert.equal(await searchTool.execute({ pattern: "def add" }, ctx), "src/a.py:1: def add(a, b):"); assert.equal(await searchTool.execute({ pattern: "return", path: "src/a.py" }, ctx), "src/a.py:2: return a - b");});
test("tools refuse paths outside the project", async () => { const { ctx, at, root } = await project(); await writeFile(path.join(root, "secret.txt"), "key"); await symlink(root, at("escape")); await assert.rejects(listFilesTool.execute({ path: ".." }, ctx), /leaves the working directory/); await assert.rejects(searchTool.execute({ pattern: "key", path: "escape" }, ctx), /leaves the working directory/); await assert.rejects(readFileTool.execute({ path: "../secret.txt" }, ctx), /leaves the working directory/);});
test("read_file numbers lines and pages", async () => { const { ctx } = await project(); assert.equal(await readFileTool.execute({ path: "src/a.py", offset: 2, limit: 1 }, ctx), " 2 return a - b\n");});
test("edit_file replaces exactly one snippet", async () => { const { ctx, at } = await project(); assert.equal(await editFileTool.execute({ path: "src/a.py", old_text: "a - b", new_text: "a + b" }, ctx), "Edited src/a.py."); assert.equal(await readFile(at("src/a.py"), "utf8"), "def add(a, b):\n return a + b\n"); await assert.rejects(editFileTool.execute({ path: "src/a.py", old_text: "nope", new_text: "x" }, ctx), /not found/); await assert.rejects(editFileTool.execute({ path: "src/a.py", old_text: "a", new_text: "x" }, ctx), /appears \d+ times/); await editFileTool.execute({ path: "src/a.py", old_text: "a + b", new_text: "$& $1" }, ctx); assert.match(await readFile(at("src/a.py"), "utf8"), /return \$& \$1/, "replacement text is literal");});
function run(steps: Array<[string, unknown, string]>): Run { const trajectory: Message[] = [{ role: "user", content: "fix add" }]; steps.forEach(([name, input, result], i) => { trajectory.push({ role: "assistant", content: "", name, toolCallId: `c${i}`, toolInput: input }); trajectory.push({ role: "tool", content: result, name, toolCallId: `c${i}` }); }); return { id: "r", machine: "dev", workspace: "w", input: "fix add", history: [], trajectory, output: "", model: "m", harness: "h", startedAt: "", finishedAt: "" };}const ok = JSON.stringify({ exitCode: 0 });const bad = JSON.stringify({ exitCode: 1 });
test("verified needs a passing command after the last change", () => { assert.equal(verifiedAfterChange(run([])).score, 1); assert.equal(verifiedAfterChange(run([["edit_file", {}, "Edited a.py."], ["run", {}, ok]])).score, 1); assert.equal(verifiedAfterChange(run([["edit_file", {}, "Edited a.py."]])).score, 0); assert.equal(verifiedAfterChange(run([["run", {}, ok], ["edit_file", {}, "Edited a.py."]])).score, 0, "a pass before the change does not count"); assert.equal(verifiedAfterChange(run([["write_file", {}, "Wrote a.py (3 bytes)."], ["run", {}, bad]])).score, 0); assert.equal(verifiedAfterChange(run([["edit_file", {}, "Error: not found"]])).score, 1, "a failed edit changed nothing");});
test("finalFiles replays successful writes and edits over the case's files", () => { const call = (id: string, name: string, toolInput: unknown, result: string): Message[] => [ { role: "assistant", content: "", name, toolCallId: id, toolInput }, { role: "tool", content: result, name, toolCallId: id }, ]; const run = { id: "r", machine: "dev", workspace: "w", input: "fix", history: [], output: "", model: "m", harness: "h", startedAt: "", finishedAt: "", files: { "a.py": "x = 1\ny = 2\n", "b.py": "keep\n" }, trajectory: [ ...call("1", "edit_file", { path: "a.py", old_text: "x = 1", new_text: "x = 10" }, "Edited a.py."), ...call("2", "edit_file", { path: "b.py", old_text: "nope", new_text: "z" }, "old_text was not found"), ...call("3", "write_file", { path: "./test_a.py", content: "assert True\n" }, "Wrote ./test_a.py (12 bytes)."), ], } as Run; assert.deepEqual(finalFiles(run), { "a.py": "x = 10\ny = 2\n", "b.py": "keep\n", "test_a.py": "assert True\n" }); assert.deepEqual(changedFiles(run).map((c) => [c.path, c.before !== undefined]), [["a.py", true], ["test_a.py", false]]); assert.deepEqual(addedLines("x = 1\ny = 2\n", "x = 10\ny = 2\n"), ["x = 10"]);});
test("hiddenTests runs a case's held-back tests against the files the run left behind", async () => { const dir = await mkdtemp(path.join(os.tmpdir(), "machine-dev-hidden-")); dirs.push(dir); const hidden = { "fix mean": { files: { "test_hidden.py": "import unittest\nfrom stats import mean\n\n\nclass H(unittest.TestCase):\n def test_empty(self):\n self.assertEqual(mean([]), 0)\n" }, command: "python3", args: ["-m", "unittest", "-q"], }, }; await writeFile(path.join(dir, "hidden.json"), JSON.stringify(hidden)); const grader = hiddenTests(pathToFileURL(path.join(dir, "machine.ts")).href, { isolation: "none" }); const run = (content: string) => ({ id: `r${Math.random()}`, machine: "dev/x", workspace: "w", input: "fix mean", history: [], output: "", model: "m", harness: "h", startedAt: "", finishedAt: "", files: { "stats.py": "def mean(xs):\n return sum(xs) / len(xs)\n" }, trajectory: [ { role: "assistant", content: "", name: "write_file", toolCallId: "1", toolInput: { path: "stats.py", content } }, { role: "tool", content: "Wrote stats.py (1 bytes).", name: "write_file", toolCallId: "1" }, ], }) as Run; const ctx = (r: Run) => ({ run: r, criterionId: "edge_cases", criterion: { description: "d" } }); const fixed = await grader.grade(ctx(run("def mean(xs):\n return sum(xs) / len(xs) if xs else 0\n"))); assert.equal(fixed.score, 1, fixed.reason); const symptomOnly = await grader.grade(ctx(run("def mean(xs):\n return sum(xs) / len(xs)\n"))); assert.equal(symptomOnly.score, 0); assert.match(symptomOnly.reason, /hidden tests fail/); assert.equal((await grader.grade(ctx({ ...run(""), input: "something else" }))).reason, "no hidden tests for this input");});