From 28ac004f48a79439f12ed37779dc3668af7dda18 Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Fri, 7 Aug 2026 03:21:08 -0500 Subject: [PATCH] seal: right-size per-block DID blooms to actual block cardinality (upstream f02919c) Per-block blooms were sized for the 4096-event block cap; real blocks hold 1-3 unique DIDs, making the resident bloom region ~90% of server heap (53.6 GiB observed). Port upstream's fix: the footer builder sizes every per-block bloom to the segment's max per-block unique-DID count (clamped >= 1), keeping the equal-size region invariant; the region header is self-describing, so legacy fixed-size segments coexist with no format change. The rewrite path already right-sized; its redundant event-count bound moves into the builder. Regression test asserts a cardinality-1 segment's blooms are >10x smaller than cap-sized ones with unchanged lookup behavior. --- receipts/ec8393c.json | 47 ++++++++++++++++++++++++ src/internal/storage/segment_footer.zig | 21 ++++++++--- src/internal/storage/segment_rewrite.zig | 8 ++-- src/internal/storage/segment_writer.zig | 37 ++++++++++++++++++- 4 files changed, 102 insertions(+), 11 deletions(-) create mode 100644 receipts/ec8393c.json diff --git a/receipts/ec8393c.json b/receipts/ec8393c.json new file mode 100644 index 0000000..bc6f5bd --- /dev/null +++ b/receipts/ec8393c.json @@ -0,0 +1,47 @@ +{ + "created_utc": "20260807T054506Z", + "image": { + "local_id": "sha256:f2c17353c68ac9d3d253708702b462b931e450432e986a95b529d684d794f08a", + "registry_digest": "sha256:f2c17353c68ac9d3d253708702b462b931e450432e986a95b529d684d794f08a", + "tag": "atcr.io/zat.dev/stream:ec8393c" + }, + "revisions": { + "dashboard_sha256": "ef6033c182d8c3f9d88af7d91e61126724efbe72e6b940984c9de12e9530cd91", + "dependencies": { + "otel": "git+https://github.com/zzstoatzz/otel-zig.git#158b32d87762be4f6b65a7d723d08b20d74d1294", + "rocksdb": "https://github.com/zzstoatzz/rocksdb-zig/archive/9d2ebd8.tar.gz", + "websocket": "https://github.com/zzstoatzz/websocket.zig/archive/73429df.tar.gz", + "xxhash": "https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.3.tar.gz", + "zat": "https://tangled.org/zat.dev/zat/archive/409711c0ff5465211f3ff3eb8a5332332da09b3b.tar.gz", + "zstd": "https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz" + }, + "stream": "ec8393c796a9404c085d55794bf17982e602083f", + "stream_describe": "ec8393c", + "upstream_jetstream": "f29815c391fc2644f8a3dd36b899fb3697dd1ea6", + "upstream_jetstream_dirty": false, + "zig": "0.16.0" + }, + "schema": "stream-admission/1", + "suites": { + "archive-contract": "pass", + "compaction-config-contract": "pass", + "cursor-lookback-contract": "pass", + "dashboard-test": "pass", + "differential-oracle": "pass", + "environment-contract": "pass", + "http-metrics-contract": "pass", + "lifecycle-oracle": "pass", + "listener-contract": "pass", + "logging-contract": "pass", + "plan-config-contract": "pass", + "powerloss-oracle": "pass", + "process-metrics-contract": "pass", + "retry-config-contract": "pass", + "shutdown-contract": "pass", + "status-contract": "pass", + "subscribe-config-contract": "pass", + "subscribe-read-batch-contract": "pass", + "unit-debug": "pass", + "unit-releasesafe": "pass" + } +} diff --git a/src/internal/storage/segment_footer.zig b/src/internal/storage/segment_footer.zig index b674385..9cd2b37 100644 --- a/src/internal/storage/segment_footer.zig +++ b/src/internal/storage/segment_footer.zig @@ -30,8 +30,6 @@ pub const Envelope = struct { pub const Builder = struct { /// arena; the caller owns its lifetime (must outlive finish) scratch: Allocator, - /// per-block DID blooms are all sized for this many entries - bloom_capacity: usize, infos: std.ArrayList(segment.BlockIndexEntry) = .empty, block_dids: std.ArrayList(std.StringHashMapUnmanaged(void)) = .empty, block_collections: std.ArrayList(std.AutoHashMapUnmanaged(u32, void)) = .empty, @@ -41,8 +39,8 @@ pub const Builder = struct { table_counts: std.ArrayList(u32) = .empty, table_ids: std.StringHashMapUnmanaged(u32) = .empty, - pub fn init(scratch: Allocator, bloom_capacity: usize) Builder { - return .{ .scratch = scratch, .bloom_capacity = bloom_capacity }; + pub fn init(scratch: Allocator) Builder { + return .{ .scratch = scratch }; } /// record one block. `info` arrives fully populated (offset, sizes, @@ -103,15 +101,26 @@ pub const Builder = struct { try buf.appendSlice(allocator, blob); } - // === section 3: per-block DID blooms (all sized for bloom_capacity) === + // === section 3: per-block DID blooms === + // Sized for the segment's actual max per-block unique-DID + // cardinality, not the 4096-event block cap (upstream f02919c): + // real blocks hold runs of a few repos (median 1-3 unique DIDs), + // and fixed-capacity blooms made the resident region >90% of + // server heap. The segment-wide max keeps every bloom identical + // in size — the invariant that lets readers index the region by + // multiplication — and the region header is self-describing, so + // right-sized and legacy fixed-size segments coexist freely. const block_did_bloom_offset = buf.items.len; { + var bloom_capacity: usize = 1; + for (self.block_dids.items) |*dids| + bloom_capacity = @max(bloom_capacity, dids.count()); var region_header: [8]u8 = undefined; std.mem.writeInt(u32, region_header[0..4], @intCast(self.infos.items.len), .little); var bloom_size: u32 = 0; var blobs: std.ArrayList([]const u8) = .empty; for (self.block_dids.items) |*dids| { - var filter = try gloom.Filter.init(scratch, self.bloom_capacity, 0.001); + var filter = try gloom.Filter.init(scratch, bloom_capacity, 0.001); var keys = dids.keyIterator(); while (keys.next()) |k| filter.add(k.*); const blob = try filter.marshal(scratch); diff --git a/src/internal/storage/segment_rewrite.zig b/src/internal/storage/segment_rewrite.zig index f7aeec5..b80161e 100644 --- a/src/internal/storage/segment_rewrite.zig +++ b/src/internal/storage/segment_rewrite.zig @@ -78,10 +78,10 @@ pub fn rewrite( try out.appendSlice(allocator, segment.magic); try out.appendNTimes(allocator, 0, segment.header_size - segment.magic.len); - // per-block blooms are uniformly sized; keep the historical capacity - var bloom_capacity: usize = 1; - for (sealed.block_index) |e| bloom_capacity = @max(bloom_capacity, e.event_count); - var builder = footer_mod.Builder.init(scratch, bloom_capacity); + // the footer builder sizes per-block blooms from the surviving rows' + // actual unique-DID cardinality (tighter than the source segment's + // event-count bound it previously carried forward) + var builder = footer_mod.Builder.init(scratch); var kept_events: std.ArrayList(segment.Event) = .empty; defer kept_events.deinit(allocator); diff --git a/src/internal/storage/segment_writer.zig b/src/internal/storage/segment_writer.zig index 788aa09..2583ae2 100644 --- a/src/internal/storage/segment_writer.zig +++ b/src/internal/storage/segment_writer.zig @@ -13,6 +13,7 @@ const std = @import("std"); const footer_mod = @import("segment_footer.zig"); +const gloom = @import("gloom.zig"); const segment = @import("segment.zig"); const zstd = @import("zstd.zig"); @@ -218,7 +219,7 @@ pub const ActiveWriter = struct { const scratch = builder_arena.allocator(); // walk our own frames (mirrors upstream walkActiveFrames) - var builder = footer_mod.Builder.init(scratch, self.max_events_per_block); + var builder = footer_mod.Builder.init(scratch); var env: footer_mod.Envelope = .{}; var saw_any = false; @@ -416,3 +417,37 @@ test "seal: our reader accepts our sealed file" { try testing.expectEqual(@as(u32, 0), ci.entries[2].count); // sentinels never count try testing.expectEqual(@as(u32, 250 - 10), ci.entries[0].count + ci.entries[1].count); } + +test "seal: per-block blooms are sized to actual DID cardinality, not the block cap" { + var w = try ActiveWriter.init(testing.allocator); + defer w.deinit(); + w.max_events_per_block = 100; + + // three blocks, every event the same DID: max per-block unique-DID + // cardinality is 1. A fixed-capacity bloom (block cap 100, let alone + // the production 4096) marshals kilobytes; a right-sized one is tiny. + var seq: u64 = 1; + while (seq <= 250) : (seq += 1) try w.append(sampleEvent(seq)); + try w.seal(); + + var sealed = try segment.Sealed.parse(testing.allocator, w.bytes()); + defer sealed.deinit(testing.allocator); + + const blobs = try sealed.blockBloomBlobs(testing.allocator); + defer testing.allocator.free(blobs); + try testing.expectEqual(@as(usize, 3), blobs.len); + + // the pre-fix sizing used the production block cap (4096); right-sized + // blooms for cardinality-1 blocks must be far below it + var reference = try gloom.Filter.init(testing.allocator, default_max_events_per_block, 0.001); + defer reference.deinit(testing.allocator); + const fixed_size = (try reference.marshal(testing.allocator)); + defer testing.allocator.free(fixed_size); + try testing.expect(blobs[0].len * 10 < fixed_size.len); + + // right-sizing must not change lookup behavior + for (blobs) |blob| { + try testing.expect(try gloom.containsMarshaled(blob, "did:plc:sampleaccount123")); + try testing.expect(!try gloom.containsMarshaled(blob, "did:plc:someoneelse")); + } +} -- knotmirror 0.0.1