Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566import assert from "node:assert/strict";import { execFileSync } from "node:child_process";import { mkdtemp, rm } from "node:fs/promises";import os from "node:os";import path from "node:path";import { after, test } from "node:test";import coder, { ranWrittenCode, runTool, writeFileTool } from "../machines/coder/machine.ts";import { localSandboxes, type Message, type Run } from "../src/index.ts";
const dirs: string[] = [];after(() => Promise.all(dirs.map((d) => rm(d, { recursive: true, force: true }))));async function sandbox(isolation: "auto" | "none") { const root = await mkdtemp(path.join(os.homedir(), ".machine-coder-test-")); dirs.push(root); return localSandboxes(root, { isolation }).create("r");}let hasBwrap = false;try { execFileSync("bwrap", ["--ro-bind", "/", "/", "--unshare-all", "--", "true"], { stdio: "ignore" }); hasBwrap = true;} catch {}
(hasBwrap ? test : test.skip)("write_file and run produce a real hello world inside the isolated sandbox", async () => { const sb = await sandbox("auto"); const ctx = { sandbox: sb }; assert.match(String(await writeFileTool.execute({ path: "hello.py", content: "print(\"Hello, world!\")\n" }, ctx)), /Wrote hello.py/); const result = await runTool.execute({ command: "python3", args: ["hello.py"] }, ctx); assert.deepEqual(result, { exitCode: 0, timedOut: false, stdout: "Hello, world!\n", stderr: "" }); const failed = (await runTool.execute({ command: "python3", args: ["-c", "raise SystemExit(4)"] }, ctx)) as { exitCode: number }; assert.equal(failed.exitCode, 4); await sb.dispose();});
test("run refuses to execute code when the sandbox is not isolated", async () => { const sb = await sandbox("none"); await assert.rejects(runTool.execute({ command: "python3", args: ["-c", "print(1)"] }, { sandbox: sb }), /not isolated/); await sb.dispose();});
function run(steps: Array<[string, unknown, string]>): Run { const trajectory: Message[] = [{ role: "user", content: "hello world" }]; steps.forEach(([name, input, result], i) => { trajectory.push({ role: "assistant", content: "", name, toolCallId: `c${i}`, toolInput: input }); trajectory.push({ role: "tool", content: result, name, toolCallId: `c${i}` }); }); return { id: "r", machine: "coder", workspace: "w", input: "hello world", history: [], trajectory, output: "", model: "m", harness: "h", startedAt: "", finishedAt: "" };}const ok = JSON.stringify({ exitCode: 0, stdout: "Hello, world!\n" });const bad = JSON.stringify({ exitCode: 1, stderr: "SyntaxError" });
test("ran_code needs a written file that was run and exited 0", () => { assert.equal(ranWrittenCode(run([["write_file", { path: "hello.py" }, "Wrote hello.py (23 bytes)."], ["run", { command: "python3", args: ["hello.py"] }, ok]])).score, 1); assert.equal(ranWrittenCode(run([["write_file", { path: "./hello.py" }, "Wrote ./hello.py (23 bytes)."], ["run", { command: "python3", args: ["hello.py"] }, ok]])).score, 1, "paths are normalized"); assert.equal(ranWrittenCode(run([["write_file", { path: "hello.py" }, "Wrote hello.py"], ["run", { command: "python3", args: ["hello.py"] }, bad], ["write_file", { path: "hello.py" }, "Wrote hello.py"], ["run", { command: "python3", args: ["hello.py"] }, ok]])).score, 1, "a fix-and-rerun counts"); assert.match(ranWrittenCode(run([])).reason, /no file was written/); assert.match(ranWrittenCode(run([["write_file", { path: "hello.py" }, "Wrote hello.py"]])).reason, /never ran/); assert.match(ranWrittenCode(run([["write_file", { path: "hello.py" }, "Wrote hello.py"], ["run", { command: "echo", args: ["Hello, world!"] }, ok]])).reason, /none of them used a written file/); assert.match(ranWrittenCode(run([["write_file", { path: "hello.py" }, "Wrote hello.py"], ["run", { command: "python3", args: ["hello.py"] }, bad]])).reason, /never exited 0/); assert.equal(ranWrittenCode(run([["write_file", { path: "../x.py" }, "Error: leaves the working directory"], ["run", { command: "python3", args: ["../x.py"] }, ok]])).score, 0, "a failed write does not count");});
test("coder declares its two tools", () => { assert.deepEqual(coder.tools.map((t) => t.name), ["write_file", "run"]); assert.deepEqual(Object.keys(coder.rubric), ["ran_code", "does_task"]);});