Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071// How far can a grader be trusted? Compare a candidate grader against a// reference (a deterministic check, a judge, or your own labels) on the same// runs, and report its error rates. A false pass (alpha) puts bad samples into// training data and rewards the wrong behavior; a false fail (beta) wastes// good samples. Both shrink the information each grade carries.
export type Agreement = { n: number; /** Reference passes, and how many the candidate also passed. */ positives: number; truePasses: number; /** Reference failures, and how many the candidate also failed. */ negatives: number; trueFails: number; /** Share of reference failures the candidate passed. */ falsePassRate: number; /** Share of reference passes the candidate failed. */ falseFailRate: number; /** Share of candidate passes that the reference also passed. */ precision: number; agreement: number; /** Cohen's kappa: agreement beyond what chance would give. */ kappa: number; /** Mutual information between reference and candidate labels, in bits per label. */ bitsPerLabel: number;};
/** `reference` and `candidate` are pass/fail labels for the same runs. */export function agreement(reference: boolean[], candidate: boolean[]): Agreement { if (reference.length !== candidate.length || reference.length === 0) throw new Error("Need the same, non-zero number of labels from both graders."); const n = reference.length; let tp = 0, tn = 0, fp = 0, fn = 0; reference.forEach((r, i) => { const c = candidate[i]; if (r && c) tp++; else if (!r && !c) tn++; else if (!r && c) fp++; else fn++; }); const agree = (tp + tn) / n; const pRef = (tp + fn) / n; const pCand = (tp + fp) / n; const chance = pRef * pCand + (1 - pRef) * (1 - pCand); const cells = [tp, fp, fn, tn].map((x) => x / n); const marginals = [pRef * pCand, (1 - pRef) * pCand, pRef * (1 - pCand), (1 - pRef) * (1 - pCand)]; const bits = cells.reduce((s, p, i) => (p > 0 && marginals[i] > 0 ? s + p * Math.log2(p / marginals[i]) : s), 0); return { n, positives: tp + fn, truePasses: tp, negatives: tn + fp, trueFails: tn, falsePassRate: tn + fp ? fp / (tn + fp) : 0, falseFailRate: tp + fn ? fn / (tp + fn) : 0, precision: tp + fp ? tp / (tp + fp) : 1, agreement: agree, kappa: chance < 1 ? (agree - chance) / (1 - chance) : 1, bitsPerLabel: bits, };}
/** * Of the samples a grader keeps, the share that are actually bad, given how * often samples are good (`passRate`) and the grader's error rates. */export function contamination(passRate: number, falsePassRate: number, falseFailRate: number): number { const keptBad = falsePassRate * (1 - passRate); const keptGood = (1 - falseFailRate) * passRate; return keptBad + keptGood > 0 ? keptBad / (keptBad + keptGood) : 0;}