Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
4.1 kB · 107 lines
Python
at letta-code
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108#!/usr/bin/env python3from __future__ import annotations
import argparseimport jsonimport statisticsimport sysfrom collections import defaultdictfrom pathlib import Pathfrom typing import Any
EXAMPLE_DIR = Path(__file__).resolve().parentsys.path.insert(0, str(EXAMPLE_DIR / "src"))
from euler_common import load_jsonl # noqa: E402
def ratio(numerator: int, denominator: int) -> float: return numerator / denominator if denominator else 0.0
def summarize(rows: list[dict[str, Any]]) -> dict[str, Any]: groups: dict[tuple[int, str, str, str], list[dict[str, Any]]] = defaultdict(list) for row in rows: groups[ ( int(row["problem_id"]), str(row["language"]), str(row["phase"]), str(row.get("strategy", "canonical")), ) ].append(row) breakdown = [] for (problem_id, language, phase, strategy), values in sorted(groups.items()): attempts = len(values) correct = sum(bool(value.get("correct")) for value in values) executed = sum(bool(value.get("successful_execution")) for value in values) formatted = sum(bool(value.get("format_valid")) for value in values) errors = sum(bool(value.get("error")) for value in values) durations = [float(value.get("duration_seconds", 0)) for value in values] breakdown.append( { "problem_id": problem_id, "language": language, "phase": phase, "strategy": strategy, "attempts": attempts, "correct": correct, "pass_at_1": ratio(correct, attempts), "pass_at_k_observed": float(correct > 0), "executed": executed, "execution_rate": ratio(executed, attempts), "format_rate": ratio(formatted, attempts), "error_rate": ratio(errors, attempts), "median_duration_seconds": statistics.median(durations) if durations else 0.0, } ) return { "attempts": len(rows), "correct": sum(bool(row.get("correct")) for row in rows), "breakdown": breakdown, }
def markdown(summary: dict[str, Any]) -> str: lines = [ "# Project Euler run report", "", f"- Attempts: {summary['attempts']}", f"- Correct: {summary['correct']}", f"- Overall pass@1 estimate: {ratio(summary['correct'], summary['attempts']):.1%}", "", "| Problem | Language | Phase | Strategy | Correct | Pass@1 | Any success | Executed | Boxed | Errors | Median seconds |", "|---:|---|---|---|---:|---:|---:|---:|---:|---:|---:|", ] for row in summary["breakdown"]: lines.append( f"| {row['problem_id']} | {row['language']} | {row['phase']} | {row['strategy']} | " f"{row['correct']}/{row['attempts']} | {row['pass_at_1']:.1%} | " f"{row['pass_at_k_observed']:.0%} | {row['execution_rate']:.1%} | " f"{row['format_rate']:.1%} | {row['error_rate']:.1%} | " f"{row['median_duration_seconds']:.1f} |" ) lines.extend( [ "", "> `pass@1` is the empirical success fraction. “Any success” is observed pass@k for this finite fan-out, not an unbiased estimator.", ] ) return "\n".join(lines) + "\n"
def main() -> None: parser = argparse.ArgumentParser(description="Build JSON and Markdown reports from evaluator results.") parser.add_argument("results", type=Path, nargs="+") parser.add_argument("--output-dir", type=Path, required=True) args = parser.parse_args() rows = [row for path in args.results for row in load_jsonl(path)] summary = summarize(rows) args.output_dir.mkdir(parents=True, exist_ok=True) (args.output_dir / "report.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") (args.output_dir / "report.md").write_text(markdown(summary), encoding="utf-8") print(args.output_dir / "report.md")
if __name__ == "__main__": main()