Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183// A workspace binds machines to a store, a sandbox provider, and a default// harness. It is where runs happen: run a task once, keep a named chat going,// and grade runs against each machine's rubric.
import path from "node:path";import { evaluate } from "./evaluator.ts";import type { Harness } from "./harness.ts";import type { Machine } from "./machine.ts";import { noEvents, preview, type EventSink } from "./events.ts";import type { Budget } from "./pricing.ts";import type { Rubric } from "./rubric.ts";import { localSandboxes, type Sandbox, type SandboxProvider } from "./sandbox.ts";import type { HarnessEvent } from "./events.ts";import { diskStore, nameSegments, newRunId, type Store } from "./store.ts";import type { Message, Run } from "./types.ts";
export type WorkspaceOptions = { /** Workspace name, recorded on every run. */ name: string; /** Directory for stored runs, sessions, and local sandboxes. */ root: string; harness: Harness; store?: Store; sandboxes?: SandboxProvider; /** Model for judge graders that name none. Defaults to the run's own model. */ judgeModel?: string; /** Reasoning effort for judges that name none, e.g. "none" to skip thinking. */ judgeEffort?: string | number; /** Reasoning effort for runs, unless a run sets its own. */ reasoningEffort?: string | number; /** Spending limit: runs and gradings are refused once it is reached. */ budget?: Budget; /** Where to report what runs are doing, e.g. fileEvents(".machine/events.jsonl") for the dashboard. */ events?: EventSink; now?: () => Date;};
export type RunOptions = { history?: Message[]; model?: string; harness?: Harness; reasoningEffort?: string | number; /** Grade the run against the machine's rubric before storing it. */ evaluate?: boolean; /** Recorded on the run: which version of the machine this is. */ revision?: string; /** Recorded on the run: the machine version (see versions.ts), e.g. "v2". */ version?: string; /** Files to put in the run's working directory before it starts (a case's fixture), by relative path. Recorded on the run. */ files?: Record<string, string>; /** Work in this sandbox instead of a fresh one, e.g. to keep a session's files between turns. It is not disposed. */ sandbox?: Sandbox; /** Progress while the run is in progress, in addition to the workspace's event sink. */ onEvent?: (event: HarnessEvent) => void; signal?: AbortSignal;};
export type Workspace = { readonly name: string; readonly store: Store; /** The spending limit, for work done outside run and evaluate (such as grading a grader). */ readonly budget?: Budget; readonly events: EventSink; /** Run a machine once on one input and store the run. */ run<R extends Rubric>(machine: Machine<R>, input: string, options?: RunOptions): Promise<Run>; /** Continue the named conversation: the session's messages become the run's history. */ chat<R extends Rubric>(machine: Machine<R>, session: string, input: string, options?: Omit<RunOptions, "history">): Promise<Run>; /** Grade a stored or fresh run and store the scorecard on it. */ evaluate<R extends Rubric>(machine: Machine<R>, run: Run, options?: { signal?: AbortSignal }): Promise<Run>;};
export function createWorkspace(options: WorkspaceOptions): Workspace { nameSegments(options.name, "workspace name"); const root = path.resolve(options.root); const store = options.store ?? diskStore(root); const sandboxes = options.sandboxes ?? localSandboxes(path.join(root, "sandboxes")); const now = options.now ?? (() => new Date());
const workspace: Workspace = { name: options.name, store, budget: options.budget, events: options.events ?? noEvents,
async run(machine, input, runOptions = {}) { const harness = runOptions.harness ?? options.harness; options.budget?.check(`a run of ${machine.name}`); const startedAt = now(); const id = newRunId(startedAt); const history = runOptions.history ?? []; const events = options.events ?? noEvents; const model = runOptions.model ?? machine.model; events.emit({ type: "run_start", machine: machine.name, run: id, model, harness: harness.name, input: preview(input), revision: runOptions.revision }); const sandbox = runOptions.sandbox ?? (await sandboxes.create(id)); let result; try { for (const [file, content] of Object.entries(runOptions.files ?? {})) await sandbox.writeFile(file, content); result = await harness.run({ instructions: machine.instructions, history, input, model: runOptions.model ?? machine.model, tools: machine.tools, sandbox, maxSteps: machine.maxSteps, onEvent: (event) => { events.emit({ ...event, machine: machine.name, run: id }); runOptions.onEvent?.(event); }, ...((runOptions.reasoningEffort ?? options.reasoningEffort) === undefined ? {} : { reasoningEffort: runOptions.reasoningEffort ?? options.reasoningEffort }), signal: runOptions.signal, }); } catch (error) { events.emit({ type: "run_error", machine: machine.name, run: id, error: preview(error instanceof Error ? error.message : String(error), 300) }); throw error; } finally { if (!runOptions.sandbox) await sandbox.dispose(); } let run: Run = { id, machine: machine.name, ...(runOptions.revision ? { revision: runOptions.revision } : {}), ...(runOptions.version ? { version: runOptions.version } : {}), workspace: options.name, instructions: machine.instructions, input, ...(runOptions.files && Object.keys(runOptions.files).length ? { files: runOptions.files } : {}), history, trajectory: result.trajectory, output: result.output, model: result.model, harness: harness.name, usage: result.usage, stopReason: result.stopReason, startedAt: startedAt.toISOString(), finishedAt: now().toISOString(), }; await store.putRun(run); events.emit({ type: "run_end", machine: machine.name, run: id, model: run.model, stopReason: run.stopReason, costUsd: run.usage?.costUsd, output: preview(run.output) }); options.budget?.charge(run.usage, `A run of ${machine.name} (${run.model})`); if (runOptions.evaluate) run = await workspace.evaluate(machine, run, { signal: runOptions.signal }); return run; },
async chat(machine, sessionName, input, runOptions = {}) { const session = await store.getSession(machine.name, sessionName); const history = session?.messages ?? []; const run = await workspace.run(machine, input, { ...runOptions, history }); await store.putSession({ machine: machine.name, name: sessionName, messages: [...history, { role: "user", content: input }, { role: "assistant", content: run.output }], updatedAt: run.finishedAt, }); return run; },
async evaluate(machine, run, evalOptions = {}) { if (run.machine !== machine.name) throw new Error(`Run ${run.id} belongs to "${run.machine}", not "${machine.name}".`); options.budget?.check(`grading run ${run.id}`); const scorecard = await evaluate(run, machine.rubric, machine.evaluator, { harness: options.harness, judgeModel: options.judgeModel, judgeEffort: options.judgeEffort, signal: evalOptions.signal, now, }); const graded = { ...run, scorecard }; await store.putRun(graded); (options.events ?? noEvents).emit({ type: "graded", machine: machine.name, run: run.id, total: scorecard.total, below: Object.entries(scorecard.criteria).filter(([, c]) => c.score < 1).map(([id]) => id), }); options.budget?.charge(scorecard.usage, `Grading run ${run.id}`); return graded; }, }; return workspace;}