Something went wrong. Try again.
zig 0.17 driver for libhegel, with the harness parts for stateful property tests
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332//! thin zig driver for libhegel, the property-testing engine behind Hegel//! (https://github.com/hegeldev/hegel-rust, MIT).//!//! libhegel owns generation, shrinking, and replay. this file owns the loop//! that feeds it test cases, the draw calls a property makes, and the failure//! report.
const std = @import("std");pub const c = @import("c");
const gpa = std.heap.smp_allocator;
pub const Settings = struct { /// null defers to libhegel: `HEGEL_TEST_CASES`, a `hegel.toml` profile, /// or its default of 100. test_cases: ?u64 = null, /// null defers to `HEGEL_SEED`, or a fresh seed per run. seed: ?u64 = null, /// directory for libhegel's failure database. null disables it. database: ?[*:0]const u8 = null, /// print every note as it is made, for every case. the only way to see /// the history leading to a crash, since a crashed case never reports. /// defaults to on when `HEGEL_ZIG_ECHO` is set. echo: ?bool = null,};
pub const TestCase = struct { ctx: *c.hegel_context_t, tc: *c.hegel_test_case_t, capture: bool, echo: bool, trace: std.ArrayList(u8) = .empty, origin: [*:0]const u8 = "property failed",
pub fn int(self: *TestCase, comptime T: type, min: T, max: T) error{StopTest}!T { var out: i64 = 0; try self.draw(c.hegel_generate_integer(self.ctx, self.tc, @intCast(min), @intCast(max), &out)); return @intCast(out); }
/// an index into a collection of `len` alternatives. shrinks toward 0. pub fn index(self: *TestCase, len: usize) error{StopTest}!usize { return self.int(usize, 0, len - 1); }
pub fn boolean(self: *TestCase) error{StopTest}!bool { return try self.int(u8, 0, 1) == 1; }
pub fn bytes(self: *TestCase, buf: []u8, min_len: usize, max_len: usize) error{StopTest}![]u8 { std.debug.assert(max_len <= buf.len); var result: c.hegel_generate_bytes_result_t = .{ .data = null, .len = 0 }; try self.draw(c.hegel_generate_bytes(self.ctx, self.tc, min_len, max_len, &result)); defer check(self.ctx, c.hegel_generate_bytes_result_free(self.ctx, &result)); @memcpy(buf[0..result.len], result.data[0..result.len]); return buf[0..result.len]; }
/// a libhegel-sized sequence of steps. each step's draws sit in their own /// span, so the shrinker can delete or reorder whole steps. pub fn steps(self: *TestCase, min: u64, max: u64) error{StopTest}!Steps { var collection: ?*c.hegel_collection_t = null; try self.draw(c.hegel_new_collection(self.ctx, self.tc, min, max, &collection)); var label: u64 = 0; check(self.ctx, c.hegel_label_from_name(self.ctx, "hegel-zig.step", &label)); return .{ .case = self, .collection = collection.?, .label = label }; }
/// an engine-owned state machine over the tags of `Rule`. libhegel picks /// each rule, decides when the history ends (at most `step_count` rules), /// and applies swarm testing: each case enables a random subset of the /// rules, so some histories are saturated with a few of them. `weights` /// biases selection among the enabled rules, in tag order. pub fn machine( self: *TestCase, comptime Rule: type, weights: ?*const [std.meta.tags(Rule).len]f64, step_count: u32, ) error{StopTest}!Machine(Rule) { const tags = comptime std.meta.tags(Rule); const names = comptime blk: { var out: [tags.len][*c]const u8 = undefined; for (tags, 0..) |tag, i| out[i] = @tagName(tag).ptr; break :blk out; }; const groups: [tags.len]i64 = @splat(0); var handle: ?*c.hegel_state_machine_t = null; var concurrency: i64 = 0; try self.draw(c.hegel_new_state_machine( self.ctx, self.tc, &names, &groups, if (weights) |w| w else null, tags.len, null, null, 0, 1, 1, step_count, &handle, &concurrency, )); return .{ .case = self, .handle = handle.? }; }
/// record a line of the case's history. shown only for the minimal /// failing case, so the format arguments must be cheap to compute. pub fn note(self: *TestCase, comptime fmt: []const u8, args: anytype) void { if (self.echo) std.debug.print(fmt ++ "\n", args); if (!self.capture) return; self.trace.print(gpa, fmt ++ "\n", args) catch @panic("out of memory"); }
/// fail the case. `origin` names the violated property: libhegel shrinks /// each distinct origin as its own bug, so it must not vary with the data. pub fn fail(self: *TestCase, comptime origin: [*:0]const u8) error{PropertyFailed} { self.origin = origin; return error.PropertyFailed; }
fn draw(self: *TestCase, rc: c.hegel_result_t) error{StopTest}!void { if (rc == c.HEGEL_E_STOP_TEST) return error.StopTest; check(self.ctx, rc); }};
pub const Steps = struct { case: *TestCase, collection: *c.hegel_collection_t, label: u64, in_step: bool = false,
pub fn next(self: *Steps) error{StopTest}!bool { const case = self.case; if (self.in_step) { check(case.ctx, c.hegel_stop_span(case.ctx, case.tc, false)); self.in_step = false; } var more = false; try case.draw(c.hegel_collection_more(case.ctx, case.tc, self.collection, &more)); if (!more) return false; check(case.ctx, c.hegel_start_span(case.ctx, case.tc, self.label)); self.in_step = true; return true; }
pub fn deinit(self: *Steps) void { check(self.case.ctx, c.hegel_collection_free(self.case.ctx, self.collection)); }};
pub fn Machine(comptime Rule: type) type { return struct { case: *TestCase, handle: *c.hegel_state_machine_t, in_round: bool = false,
/// the next rule to apply, or null when the history is over. pub fn next(self: *@This()) error{StopTest}!?Rule { const case = self.case; while (true) { if (!self.in_round) { var group: i64 = 0; try case.draw(c.hegel_state_machine_next_group(case.ctx, case.tc, self.handle, &group)); if (group == c.HEGEL_STATE_MACHINE_DONE) return null; self.in_round = true; } var rule: i64 = 0; try case.draw(c.hegel_state_machine_next_rule(case.ctx, case.tc, self.handle, 0, &rule)); if (rule != c.HEGEL_STATE_MACHINE_DONE) return std.meta.tags(Rule)[@intCast(rule)]; self.in_round = false; } }
pub fn deinit(self: *@This()) void { check(self.case.ctx, c.hegel_state_machine_free(self.case.ctx, self.handle)); } };}
/// what a run exercised, by rule. "no failures" over an empty state space/// and over a real one print the same line; this is what tells them apart./// counts every case the engine ran, including the ones spent shrinking.pub fn Tally(comptime Rule: type) type { return struct { cases: u64 = 0, finished: u64 = 0, steps: u64 = 0, rules: [std.meta.tags(Rule).len]u64 = @splat(0), /// summed over finished cases final_size: u64 = 0,
pub fn case(self: *@This()) void { self.cases += 1; }
pub fn step(self: *@This(), rule: Rule) void { self.steps += 1; for (std.meta.tags(Rule), &self.rules) |tag, *count| count.* += @intFromBool(tag == rule); }
/// the case ran to the end of its history. `size` is whatever measures /// how much state it built: keys in the tree, tasks scheduled. pub fn finish(self: *@This(), size: u64) void { self.finished += 1; self.final_size += size; }
pub fn print(self: *const @This()) void { const cases: f64 = @floatFromInt(@max(self.cases, 1)); const finished: f64 = @floatFromInt(@max(self.finished, 1)); std.debug.print("{d} cases ({d} finished), {d} steps ({d:.1} per case), mean final size {d:.1}\n ", .{ self.cases, self.finished, self.steps, @as(f64, @floatFromInt(self.steps)) / cases, @as(f64, @floatFromInt(self.final_size)) / finished, }); for (std.meta.tags(Rule), self.rules) |rule, count| std.debug.print(" {s}={d}", .{ @tagName(rule), count }); std.debug.print("\n", .{}); for (std.meta.tags(Rule), self.rules) |rule, count| { if (count == 0) std.debug.print("never ran: {s}\n", .{@tagName(rule)}); } } };}
/// run `property` against generated cases. on failure, prints each distinct/// failure's minimal history and returns `error.PropertyFailed`.////// the property returns normally to pass, `case.fail(...)` to fail, and/// `error.Reject` to discard a case that broke a precondition. any other/// error also fails the case, with the error name as its origin.pub fn run( settings: Settings, context: anytype, comptime property: fn (@TypeOf(context), *TestCase) anyerror!void,) error{PropertyFailed}!void { const ctx = c.hegel_context_new() orelse @panic("hegel_context_new failed"); defer _ = c.hegel_context_free(ctx);
var s: ?*c.hegel_settings_t = null; check(ctx, c.hegel_settings_new(ctx, &s)); defer check(ctx, c.hegel_settings_free(ctx, s)); if (settings.test_cases) |n| check(ctx, c.hegel_settings_set_test_cases(ctx, s, n)); check(ctx, c.hegel_settings_set_database(ctx, s, settings.database orelse "")); if (settings.seed) |seed| check(ctx, c.hegel_settings_set_seed(ctx, s, seed, true));
var hegel_run: ?*c.hegel_run_t = null; check(ctx, c.hegel_run_start(ctx, s, null, null, &hegel_run)); defer check(ctx, c.hegel_run_free(ctx, hegel_run));
const echo = settings.echo orelse (std.c.getenv("HEGEL_ZIG_ECHO") != null);
var traces: std.StringArrayHashMapUnmanaged([]u8) = .empty; defer { for (traces.values()) |trace| gpa.free(trace); traces.deinit(gpa); }
while (true) { var tc: ?*c.hegel_test_case_t = null; check(ctx, c.hegel_next_test_case(ctx, hegel_run, &tc)); const handle = tc orelse break; defer check(ctx, c.hegel_test_case_free(ctx, handle));
var case: TestCase = .{ .ctx = ctx, .tc = handle, .capture = false, .echo = echo }; defer case.trace.deinit(gpa); check(ctx, c.hegel_test_case_should_capture(ctx, handle, &case.capture)); if (case.echo) std.debug.print("--- case ---\n", .{});
const status: u32, const origin: ?[*:0]const u8 = if (property(context, &case)) |_| .{ c.HEGEL_STATUS_VALID, null } else |err| switch (err) { error.StopTest => .{ c.HEGEL_STATUS_OVERRUN, null }, error.Reject => .{ c.HEGEL_STATUS_INVALID, null }, error.PropertyFailed => .{ c.HEGEL_STATUS_INTERESTING, case.origin }, else => .{ c.HEGEL_STATUS_INTERESTING, @errorName(err).ptr }, };
if (origin) |o| { if (case.capture) { const entry = traces.getOrPut(gpa, std.mem.span(o)) catch @panic("out of memory"); if (entry.found_existing) gpa.free(entry.value_ptr.*); entry.value_ptr.* = case.trace.toOwnedSlice(gpa) catch @panic("out of memory"); } } check(ctx, c.hegel_mark_complete(ctx, handle, status, origin)); }
var result: ?*c.hegel_run_result_t = null; check(ctx, c.hegel_run_result(ctx, hegel_run, &result)); defer check(ctx, c.hegel_run_result_free(ctx, result));
var run_status: c.hegel_run_status_t = undefined; check(ctx, c.hegel_run_result_status(ctx, result, &run_status)); if (run_status == c.HEGEL_RUN_STATUS_PASSED) return;
var run_error: [*c]const u8 = null; check(ctx, c.hegel_run_result_error(ctx, result, &run_error)); if (run_error != null) std.debug.print("hegel run error: {s}\n", .{run_error});
var failure_count: usize = 0; check(ctx, c.hegel_run_result_failure_count(ctx, result, &failure_count)); for (0..failure_count) |i| { var failure: ?*c.hegel_failure_t = null; check(ctx, c.hegel_run_result_failure(ctx, result, i, &failure)); defer check(ctx, c.hegel_failure_free(ctx, failure));
var origin: [*c]const u8 = null; var blob: [*c]const u8 = null; var caveat: [*c]const u8 = null; check(ctx, c.hegel_failure_origin(ctx, failure, &origin)); check(ctx, c.hegel_failure_reproduction_blob(ctx, failure, &blob)); check(ctx, c.hegel_failure_caveat(ctx, failure, &caveat));
const name = std.mem.span(origin); std.debug.print("\nFAILED: {s}\n", .{name}); if (traces.get(name)) |trace| std.debug.print("minimal history:\n{s}", .{trace}); if (caveat != null) std.debug.print("caveat: {s}\n", .{caveat}); if (blob != null) std.debug.print("reproduce: {s}\n", .{blob}); } return error.PropertyFailed;}
fn check(ctx: *c.hegel_context_t, rc: c.hegel_result_t) void { if (rc == c.HEGEL_OK) return; std.debug.panic("libhegel call failed ({d}): {s}", .{ rc, c.hegel_context_last_error(ctx) });}