Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112// Core data shapes shared by every primitive. Plain JSON so runs can be// stored, diffed, and turned into training data without conversion.
export type Role = "system" | "user" | "assistant" | "tool";
export type Message = { role: Role; content: string; /** Tool name, for tool calls and tool results. */ name?: string; /** Tool call id, linking a tool result to its call. */ toolCallId?: string; /** Arguments of an assistant tool call. */ toolInput?: unknown; /** The model's reasoning before this assistant message, when the endpoint returns it separately. */ reasoning?: string;};
export type Usage = { /** All input tokens, cached ones included. */ inputTokens?: number; /** The part of inputTokens served from the provider's prompt cache (billed lower). */ cachedInputTokens?: number; outputTokens?: number; /** USD, when the harness knows the model's price. */ costUsd?: number;};
/** What a harness returns for one task: the full trajectory and the final answer. */export type HarnessResult = { trajectory: Message[]; output: string; model: string; usage?: Usage; stopReason?: string;};
export type Score = { /** 0 (fails the criterion) to 1 (fully meets it). */ score: number; reason: string; /** Model usage spent producing this score, for graders that call a model. */ usage?: Usage;};
export type Scorecard = { /** Weighted mean of criterion scores, 0 to 1. */ total: number; criteria: Record<string, Score & { weight: number; grader: string }>; evaluatedAt: string; /** Fingerprint of the rubric the run was graded under; a run graded under another rubric is due for regrading. */ rubric?: string; /** What grading cost: judge and other model-backed graders. */ usage?: Usage;};
/** One execution of a machine on one input. */export type Run = { id: string; machine: string; /** Which version of the machine produced the run, e.g. a git commit, when known. */ revision?: string; /** The machine version (instructions, model, rubric) that produced the run, e.g. "v3", when versions are kept. */ version?: string; workspace: string; /** The system instructions the model was given. */ instructions?: string; input: string; /** Files the run's working directory started with (a case's fixture), by relative path. */ files?: Record<string, string>; /** Prior conversation the input was answered in, if any. */ history: Message[]; trajectory: Message[]; output: string; model: string; harness: string; usage?: Usage; stopReason?: string; startedAt: string; finishedAt: string; scorecard?: Scorecard; /** What a person said about the run, e.g. from /good, /bad, or /correct in an interactive session. */ feedback?: Feedback[]; /** The interactive session the run belongs to, if any. */ session?: string; /** For a turn of a conversation between two machines, where it sits in that conversation. */ conversation?: ConversationTurn;};
/** One turn of a conversation between a machine and its counterpart (see converse). */export type ConversationTurn = { /** Shared by every run of the conversation, on both sides. */ id: string; /** The case the conversation was started from, e.g. a simulated user's scenario. */ case: string; /** The machine on the other side. */ with: string; /** 1-based turn of this side. */ turn: number; /** This side's last turn: for the agent, the run that holds the whole conversation and is graded. */ final: boolean; /** Why the conversation ended, on final turns: the agent finished, or it hit the turn limit. */ ended?: "done" | "max_turns";};
export type Feedback = { kind: "good" | "bad" | "correction"; /** For a correction, what should change ("too terse"); optional otherwise. */ note?: string; at: string;};