// Cases are the inputs a machine is measured on: machines//cases.json // (what suites, optimize, and train learn from) and holdout.json (never shown // to an optimizer or a trainer). A case is an input string, or an object with // the input and the files its working directory starts with, so a coding // agent can be given a small project to change. import { access, readFile } from "node:fs/promises"; import path from "node:path"; import type { Machine } from "./machine.ts"; import { nameSegments } from "./store.ts"; export type Case = { input: string; /** Files the run's working directory starts with, by relative path. */ files?: Record; }; /** Cases from parsed JSON: an array of strings or `{ input, files? }` objects. */ export function parseCases(value: unknown, source = "cases"): Case[] { if (!Array.isArray(value) || value.length === 0) throw new Error(`${source} must be a non-empty JSON array of cases.`); return value.map((c, i) => { if (typeof c === "string" && c.trim()) return { input: c }; if (c && typeof c === "object" && typeof (c as Case).input === "string" && (c as Case).input.trim()) { const { input, files } = c as Case; if (files !== undefined) { if (!files || typeof files !== "object" || Array.isArray(files)) throw new Error(`${source}: case ${i + 1} has "files" that is not an object of path to text.`); for (const [file, text] of Object.entries(files)) { if (typeof text !== "string") throw new Error(`${source}: case ${i + 1} file "${file}" is not text.`); if (!file || path.isAbsolute(file) || file.split(/[\\/]/).includes("..")) throw new Error(`${source}: case ${i + 1} file "${file}" must be a relative path inside the project.`); } } return files ? { input, files } : { input }; } throw new Error(`${source}: case ${i + 1} must be an input string or an object with an "input" string.`); }); } export async function exists(file: string): Promise { try { await access(file); return true; } catch { return false; } } /** Whether the machine or one of its ancestors has this case file. */ export async function hasCaseFile(workspaceDir: string, machine: Machine, name: string): Promise { for (let m: Machine | undefined = machine; m; m = m.parent) { if (await exists(path.join(workspaceDir, "machines", ...nameSegments(m.name, "machine name"), name))) return true; } return false; } /** * A machine's cases.json, or another case file in its directory (e.g. a * held-out set). A child without the file uses its nearest ancestor's, since * it does the parent's job and more. */ export async function loadCases(workspaceDir: string, machine: Machine, casesFile = "cases.json"): Promise { const dir = path.join(workspaceDir, "machines", ...nameSegments(machine.name, "machine name")); let file = path.resolve(dir, casesFile); if (!file.startsWith(dir + path.sep)) throw new Error(`--cases must name a file in ${path.relative(workspaceDir, dir)}/.`); for (let m = machine.parent; !(await exists(file)) && m; m = m.parent) { const inherited = path.resolve(path.join(workspaceDir, "machines", ...nameSegments(m.name, "machine name")), casesFile); if (await exists(inherited)) file = inherited; } let cases: unknown; try { cases = JSON.parse(await readFile(file, "utf8")); } catch (error) { throw new Error(`Could not read ${file}: ${error instanceof Error ? error.message : error}`); } return parseCases(cases, file); }