Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576// Cases are the inputs a machine is measured on: machines/<name>/cases.json// (what suites, optimize, and train learn from) and holdout.json (never shown// to an optimizer or a trainer). A case is an input string, or an object with// the input and the files its working directory starts with, so a coding// agent can be given a small project to change.
import { access, readFile } from "node:fs/promises";import path from "node:path";import type { Machine } from "./machine.ts";import { nameSegments } from "./store.ts";
export type Case = { input: string; /** Files the run's working directory starts with, by relative path. */ files?: Record<string, string>;};
/** Cases from parsed JSON: an array of strings or `{ input, files? }` objects. */export function parseCases(value: unknown, source = "cases"): Case[] { if (!Array.isArray(value) || value.length === 0) throw new Error(`${source} must be a non-empty JSON array of cases.`); return value.map((c, i) => { if (typeof c === "string" && c.trim()) return { input: c }; if (c && typeof c === "object" && typeof (c as Case).input === "string" && (c as Case).input.trim()) { const { input, files } = c as Case; if (files !== undefined) { if (!files || typeof files !== "object" || Array.isArray(files)) throw new Error(`${source}: case ${i + 1} has "files" that is not an object of path to text.`); for (const [file, text] of Object.entries(files)) { if (typeof text !== "string") throw new Error(`${source}: case ${i + 1} file "${file}" is not text.`); if (!file || path.isAbsolute(file) || file.split(/[\\/]/).includes("..")) throw new Error(`${source}: case ${i + 1} file "${file}" must be a relative path inside the project.`); } } return files ? { input, files } : { input }; } throw new Error(`${source}: case ${i + 1} must be an input string or an object with an "input" string.`); });}
export async function exists(file: string): Promise<boolean> { try { await access(file); return true; } catch { return false; }}
/** Whether the machine or one of its ancestors has this case file. */export async function hasCaseFile(workspaceDir: string, machine: Machine, name: string): Promise<boolean> { for (let m: Machine | undefined = machine; m; m = m.parent) { if (await exists(path.join(workspaceDir, "machines", ...nameSegments(m.name, "machine name"), name))) return true; } return false;}
/** * A machine's cases.json, or another case file in its directory (e.g. a * held-out set). A child without the file uses its nearest ancestor's, since * it does the parent's job and more. */export async function loadCases(workspaceDir: string, machine: Machine, casesFile = "cases.json"): Promise<Case[]> { const dir = path.join(workspaceDir, "machines", ...nameSegments(machine.name, "machine name")); let file = path.resolve(dir, casesFile); if (!file.startsWith(dir + path.sep)) throw new Error(`--cases must name a file in ${path.relative(workspaceDir, dir)}/.`); for (let m = machine.parent; !(await exists(file)) && m; m = m.parent) { const inherited = path.resolve(path.join(workspaceDir, "machines", ...nameSegments(m.name, "machine name")), casesFile); if (await exists(inherited)) file = inherited; } let cases: unknown; try { cases = JSON.parse(await readFile(file, "utf8")); } catch (error) { throw new Error(`Could not read ${file}: ${error instanceof Error ? error.message : error}`); } return parseCases(cases, file);}