From 7eeca6e484869b6abdcbea33bc86901a393fc70b Mon Sep 17 00:00:00 2001 From: David Crawshaw Date: Sun, 2 Aug 2026 18:48:07 +0000 Subject: [PATCH] Correct demo provenance and viewer portability Co-authored-by: Shelley --- .../site-demo-commits-2026-08-02/README.md | 23 ++-- .../site-demo-commits-2026-08-02/runs.json | 118 +++++++++++++++--- .../site-demo-commits-2026-08-02/verify.py | 19 ++- .../site-demo-commits-2026-08-02/viewer.html | 8 +- 4 files changed, 134 insertions(+), 34 deletions(-) diff --git a/analysis/site-demo-commits-2026-08-02/README.md b/analysis/site-demo-commits-2026-08-02/README.md index 8443a67..23f348d 100644 --- a/analysis/site-demo-commits-2026-08-02/README.md +++ b/analysis/site-demo-commits-2026-08-02/README.md @@ -2,11 +2,13 @@ Fresh runs on **August 2, 2026** using `gpt-5.6-sol`, `-no-cache`, and upstream repository context. The canonical checked-in outputs were produced by -Meat commit `85fd905c4957203dcae7be146de745d5f93fbac3`; three earlier repeat -runs used behavior-equivalent commit -`38d6b8f832030c7da90e4138452540788cadecf2`. Both exposed rubric hash -`3c17e8412d288ebc`. [`runs.json`](runs.json) contains every full response, -stderr timing/token line, input SHA-256, model, flags, and executable commit. +Meat commit `85fd905c4957203dcae7be146de745d5f93fbac3` with rubric hash +`3c17e8412d288ebc`. Three earlier exploratory runs used commit +`38d6b8f832030c7da90e4138452540788cadecf2` with rubric hash +`e6866ffaafb898ce`; because the model-visible surfaces differ, treat those as +cross-revision robustness observations rather than identical reruns. +[`runs.json`](runs.json) contains every full response, stderr timing/token line, +input SHA-256, model, flags, executable commit, and per-run rubric hash. Run `python3 verify.py` to recount all metrics, verify JSON/diff identity, check repeat-run retention, and confirm the standalone viewer's embedded data. @@ -53,7 +55,8 @@ text. Meat keeps one representative correction and drops seven repetitions. > Meat turns 16 changed lines across 8 hunks into the only 2 lines you need. **Metrics:** 16 → 2 changed rows; 76 → 10 physical rows; 2,895 → 458 bytes. -The result was 2/16 in each of four fresh runs. +All four recorded evaluation runs across the two Meat revisions above retained +2/16 changed rows. Files: [`go.original.diff`](go.original.diff) · [`go.meat.diff`](go.meat.diff) · [`go.meat.json`](go.meat.json) @@ -81,7 +84,8 @@ themselves as generated. Meat removes every generated copy. > Meat keeps the modular source fix and removes eight generated copies. **Metrics:** 18 → 2 changed rows; 117 → 11 physical rows; 7,812 → 774 -bytes. The result was 2/18 in each of four fresh runs. +bytes. All four recorded evaluation runs across the two Meat revisions above +retained 2/18 changed rows. Files: [`python.original.diff`](python.original.diff) · [`python.meat.diff`](python.meat.diff) · @@ -110,8 +114,9 @@ imports and ten repetitive hunks, leaving one concurrency decision. > else. **Metrics:** 22 → 2 changed rows; 106 → 13 physical rows; 5,090 → 546 -bytes. Across four fresh runs, Meat retained 2 rows three times and 4 rows once; -the checked-in canonical result is the 2-row output. +bytes. Across four recorded evaluation runs spanning the two Meat revisions, +Meat retained 2 rows three times and 4 rows once; the checked-in canonical +result is the 2-row output. Files: [`rust.original.diff`](rust.original.diff) · [`rust.meat.diff`](rust.meat.diff) · [`rust.meat.json`](rust.meat.json) diff --git a/analysis/site-demo-commits-2026-08-02/runs.json b/analysis/site-demo-commits-2026-08-02/runs.json index c89e885..cd1b34b 100644 --- a/analysis/site-demo-commits-2026-08-02/runs.json +++ b/analysis/site-demo-commits-2026-08-02/runs.json @@ -1,13 +1,6 @@ { "date": "2026-08-02", "model": "gpt-5.6-sol", - "rubric_hash": "3c17e8412d288ebc", - "flags": [ - "-no-cache", - "-model", - "gpt-5.6-sol", - "-json" - ], "candidates": { "go": { "upstream_commit": "0b6ea6bb04e49e5eba614d39bb1334c87fff4f62", @@ -24,7 +17,13 @@ "input_tokens": 5866, "output_tokens": 337, "elision": "kept 2/16 changed lines in 1/1 files" - } + }, + "rubric_hash": "e6866ffaafb898ce", + "model": "gpt-5.6-sol", + "flags": [ + "-no-cache", + "-json" + ] }, { "run": 2, @@ -36,7 +35,13 @@ "input_tokens": 5866, "output_tokens": 375, "elision": "kept 2/16 changed lines in 1/1 files" - } + }, + "rubric_hash": "e6866ffaafb898ce", + "model": "gpt-5.6-sol", + "flags": [ + "-no-cache", + "-json" + ] }, { "run": 3, @@ -48,7 +53,13 @@ "input_tokens": 5866, "output_tokens": 285, "elision": "kept 2/16 changed lines in 1/1 files" - } + }, + "rubric_hash": "e6866ffaafb898ce", + "model": "gpt-5.6-sol", + "flags": [ + "-no-cache", + "-json" + ] }, { "run": 4, @@ -60,7 +71,15 @@ "input_tokens": 5620, "output_tokens": 301, "elision": "kept 2/16 changed lines in 1/1 files" - } + }, + "rubric_hash": "3c17e8412d288ebc", + "model": "gpt-5.6-sol", + "flags": [ + "-no-cache", + "-model", + "gpt-5.6-sol", + "-json" + ] } ] }, @@ -79,7 +98,13 @@ "input_tokens": 35659, "output_tokens": 626, "elision": "kept 2/18 changed lines in 1/9 files" - } + }, + "rubric_hash": "e6866ffaafb898ce", + "model": "gpt-5.6-sol", + "flags": [ + "-no-cache", + "-json" + ] }, { "run": 2, @@ -91,7 +116,13 @@ "input_tokens": 49086, "output_tokens": 917, "elision": "kept 2/18 changed lines in 1/9 files" - } + }, + "rubric_hash": "e6866ffaafb898ce", + "model": "gpt-5.6-sol", + "flags": [ + "-no-cache", + "-json" + ] }, { "run": 3, @@ -103,7 +134,13 @@ "input_tokens": 61129, "output_tokens": 1009, "elision": "kept 2/18 changed lines in 1/9 files" - } + }, + "rubric_hash": "e6866ffaafb898ce", + "model": "gpt-5.6-sol", + "flags": [ + "-no-cache", + "-json" + ] }, { "run": 4, @@ -115,7 +152,15 @@ "input_tokens": 49861, "output_tokens": 979, "elision": "kept 2/18 changed lines in 1/9 files" - } + }, + "rubric_hash": "3c17e8412d288ebc", + "model": "gpt-5.6-sol", + "flags": [ + "-no-cache", + "-model", + "gpt-5.6-sol", + "-json" + ] } ] }, @@ -134,7 +179,13 @@ "input_tokens": 13860, "output_tokens": 736, "elision": "kept 2/22 changed lines in 1/3 files" - } + }, + "rubric_hash": "e6866ffaafb898ce", + "model": "gpt-5.6-sol", + "flags": [ + "-no-cache", + "-json" + ] }, { "run": 2, @@ -146,7 +197,13 @@ "input_tokens": 22754, "output_tokens": 1687, "elision": "kept 4/22 changed lines in 2/3 files" - } + }, + "rubric_hash": "e6866ffaafb898ce", + "model": "gpt-5.6-sol", + "flags": [ + "-no-cache", + "-json" + ] }, { "run": 3, @@ -158,7 +215,13 @@ "input_tokens": 13749, "output_tokens": 657, "elision": "kept 2/22 changed lines in 1/3 files" - } + }, + "rubric_hash": "e6866ffaafb898ce", + "model": "gpt-5.6-sol", + "flags": [ + "-no-cache", + "-json" + ] }, { "run": 4, @@ -170,9 +233,24 @@ "input_tokens": 13098, "output_tokens": 412, "elision": "kept 2/22 changed lines in 1/3 files" - } + }, + "rubric_hash": "3c17e8412d288ebc", + "model": "gpt-5.6-sol", + "flags": [ + "-no-cache", + "-model", + "gpt-5.6-sol", + "-json" + ] } ] } - } + }, + "canonical_rubric_hash": "3c17e8412d288ebc", + "canonical_flags": [ + "-no-cache", + "-model", + "gpt-5.6-sol", + "-json" + ] } diff --git a/analysis/site-demo-commits-2026-08-02/verify.py b/analysis/site-demo-commits-2026-08-02/verify.py index 4838d9d..e0db2dd 100644 --- a/analysis/site-demo-commits-2026-08-02/verify.py +++ b/analysis/site-demo-commits-2026-08-02/verify.py @@ -40,16 +40,29 @@ def metrics(text: str) -> tuple[int, int, int, int, int]: return changed, len(lines), len(text.encode()), files, hunks +EXPECTED_RUBRICS = { + "38d6b8f832030c7da90e4138452540788cadecf2": "e6866ffaafb898ce", + "85fd905c4957203dcae7be146de745d5f93fbac3": "3c17e8412d288ebc", +} + + def main() -> None: runs = json.loads((ROOT / "runs.json").read_text()) assert runs["model"] == "gpt-5.6-sol" - assert runs["rubric_hash"] == "3c17e8412d288ebc" + assert runs["canonical_rubric_hash"] == "3c17e8412d288ebc" embedded_text = (ROOT / "data.js").read_text().strip() prefix = "window.__DEMO_FILES__ = " assert embedded_text.startswith(prefix) and embedded_text.endswith(";") embedded = json.loads(embedded_text[len(prefix) : -1]) + viewer = (ROOT / "viewer.html").read_text() + assert '' in viewer + assert "fetch(" not in viewer + assert "grid-template-columns:46px minmax(0,1fr)" in viewer + for key in embedded: + assert key in viewer + for name, expected in EXPECTED.items(): raw_path = ROOT / f"{name}.original.diff" out_path = ROOT / f"{name}.meat.diff" @@ -69,6 +82,10 @@ def main() -> None: assert run_data["input_sha256"] == hashlib.sha256(raw.encode()).hexdigest() kept = [metrics(run["response"]["smart_diff"])[0] for run in run_data["runs"]] assert kept == expected["runs"], (name, "runs", kept) + for run in run_data["runs"]: + assert run["model"] == "gpt-5.6-sol" + assert run["rubric_hash"] == EXPECTED_RUBRICS[run["meat_commit"]] + assert "-no-cache" in run["flags"] and "-json" in run["flags"] assert run_data["runs"][-1]["response"] == response assert embedded[f"{name}-original.diff"] == raw diff --git a/analysis/site-demo-commits-2026-08-02/viewer.html b/analysis/site-demo-commits-2026-08-02/viewer.html index e20b6e8..5fe1933 100644 --- a/analysis/site-demo-commits-2026-08-02/viewer.html +++ b/analysis/site-demo-commits-2026-08-02/viewer.html @@ -28,16 +28,16 @@ -- knotmirror 0.0.1