Something went wrong. Try again.
My version of https://github.com/vercel-labs/fx
Something went wrong. Try again.
6.5 kB · 170 lines
Zig
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171const std = @import("std");const TranscriptRuntime = @import("benchmark_exports").TranscriptRuntime;
const Kind = enum { baseline, candidate };const call_ids = [_][]const u8{ "first", "second" };const samples_per_batch = 100;const comparison_batches = 9;const growth_batches = 5;const ratio_scale = 1_000;const max_ratio = 2 * ratio_scale;const Harness = struct { transcript: TranscriptRuntime = .{ .max_retained_transcript_bytes = 256 }, kind: Kind, entry_ids: [call_ids.len]u32 = undefined,
fn init(alloc: std.mem.Allocator, kind: Kind) !Harness { var self = Harness{ .kind = kind }; errdefer self.transcript.deinit(alloc); _ = try self.transcript.appendSemanticNotice(alloc, .{ .topic = "system", .tone = .information, .body = "unrelated output retained near the configured transcript cap", }); for (call_ids, 0..) |call_id, index| switch (kind) { .baseline => self.entry_ids[index] = try self.transcript.appendRawTranscriptEntryClassified( alloc, "● read_file\n", .tool_status, ), .candidate => _ = try self.transcript.applyToolLifecycle( alloc, .{ .authoritative_started = .{ .id = .{ .turn_id = 1, .call_id = call_id }, .reconciles_provisional_call_id = null, .tool_name = "read_file", .activity_kind = .read, } }, ), }; return self; }
fn update(self: *Harness, alloc: std.mem.Allocator, index: usize) !void { const slot = index % call_ids.len; switch (self.kind) { .baseline => { if (!try self.transcript.updateRawBytesEntry( alloc, self.entry_ids[slot], "● Reading progress\n", )) return error.MissingBaselineTranscriptEntry; }, .candidate => _ = try self.transcript.applyToolLifecycle( alloc, .{ .progress = .{ .id = .{ .turn_id = 1, .call_id = call_ids[slot] }, .text = "● Reading progress", } }, ), } }};fn runUpdates(harness: *Harness, alloc: std.mem.Allocator, count: usize) !void { for (0..count) |index| try harness.update(alloc, index);}
fn measureP95(io: std.Io, harness: *Harness, alloc: std.mem.Allocator) !u64 { var samples: [samples_per_batch]u64 = undefined; for (&samples, 0..) |*sample, index| { const started = std.Io.Timestamp.now(io, .awake).nanoseconds; try harness.update(alloc, index); sample.* = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds - started); } std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); return samples[94];}
fn ratio(numerator: u64, denominator: u64) u64 { if (denominator == 0) return std.math.maxInt(u64); return @intCast(@min( @as(u128, numerator) * ratio_scale / denominator, std.math.maxInt(u64), ));}fn comparisonRatio(io: std.Io, alloc: std.mem.Allocator, reverse: bool) !u64 { var baseline = try Harness.init(alloc, .baseline); defer baseline.transcript.deinit(alloc); var candidate = try Harness.init(alloc, .candidate); defer candidate.transcript.deinit(alloc); try runUpdates(&baseline, alloc, 200); try runUpdates(&candidate, alloc, 200);
var baseline_p95: u64 = undefined; var candidate_p95: u64 = undefined; if (reverse) { candidate_p95 = try measureP95(io, &candidate, alloc); baseline_p95 = try measureP95(io, &baseline, alloc); } else { baseline_p95 = try measureP95(io, &baseline, alloc); candidate_p95 = try measureP95(io, &candidate, alloc); } return ratio(candidate_p95, baseline_p95);}fn growthRatio(io: std.Io, alloc: std.mem.Allocator) !u64 { var candidate = try Harness.init(alloc, .candidate); defer candidate.transcript.deinit(alloc); try runUpdates(&candidate, alloc, 200); const initial = try measureP95(io, &candidate, alloc); try runUpdates(&candidate, alloc, 5_000); return ratio(try measureP95(io, &candidate, alloc), initial);}
const Gate = struct { median: u64, breaches: usize, passed: bool,};fn evaluate(ratios: []u64, breach_quorum: usize) Gate { var breaches: usize = 0; for (ratios) |value| if (value > max_ratio) { breaches += 1; }; std.mem.sort(u64, ratios, {}, std.sort.asc(u64)); const midpoint = ratios[ratios.len / 2]; return .{ .median = midpoint, .breaches = breaches, .passed = midpoint <= max_ratio or breaches < breach_quorum, };}pub fn main(init: std.process.Init) !void { const alloc = std.heap.c_allocator; var comparison_ratios: [comparison_batches]u64 = undefined; for (&comparison_ratios, 0..) |*value, index| { value.* = try comparisonRatio(init.io, alloc, index % 2 == 1); } var growth_ratios: [growth_batches]u64 = undefined; for (&growth_ratios) |*value| value.* = try growthRatio(init.io, alloc); const comparison = evaluate(&comparison_ratios, 5); const growth = evaluate(&growth_ratios, 3);
var buffer: [512]u8 = undefined; var writer = std.Io.File.stdout().writer(init.io, &buffer); try writer.interface.print( "comparison_median_ratio={d}.{d:0>3}x breaches={d}/{d} passed={any}\n" ++ "growth_median_ratio={d}.{d:0>3}x breaches={d}/{d} passed={any}\n", .{ comparison.median / ratio_scale, comparison.median % ratio_scale, comparison.breaches, comparison_batches, comparison.passed, growth.median / ratio_scale, growth.median % ratio_scale, growth.breaches, growth_batches, growth.passed, }, ); try writer.interface.flush(); if (!comparison.passed or !growth.passed) return error.UiActivityProgressRegression;}
test "benchmark gate handles isolated and repeated timing breaches" { var isolated = [_]u64{ 1_000, 1_100, 2_500, 1_050, 1_000 }; var repeated = [_]u64{ 2_100, 2_200, 1_900, 2_300, 2_400 }; try std.testing.expect(evaluate(&isolated, 3).passed); try std.testing.expect(!evaluate(&repeated, 3).passed);}