Something went wrong. Try again.
atproto relay in zig zlay.waow.tech
relay zig atproto
Something went wrong. Try again.
Zig
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083//! relay subscriber — per-host firehose worker//!//! connects to a single PDS or relay upstream, decodes firehose frames//! using the zat SDK's CBOR codec, validates commit frames, and persists//! all events to disk with relay-assigned sequence numbers before broadcast.//!//! managed by the Slurper, which spawns one Subscriber per tracked host.
const std = @import("std");const websocket = @import("websocket");const zat = @import("zat");const broadcaster = @import("broadcaster.zig");const validator_mod = @import("validator.zig");const event_log_mod = @import("event_log.zig");const collection_index_mod = @import("collection_index.zig");const resync_mod = @import("resync.zig");const frame_worker_mod = @import("frame_worker.zig");const host_ops_mod = @import("host_ops.zig");
const Allocator = std.mem.Allocator;const Io = std.Io;const log = std.log.scoped(.relay);
const max_consecutive_failures = 15;const cursor_flush_interval_sec = 4; // flush cursor to DB every N seconds (Go relay: 4s)const ping_interval_sec = 30; // keepalive ping interval (Go relay: 30s)const max_ping_failures = 4; // close connection after N consecutive ping failures (Go relay: 4)
// per-host rate limits — backpressure thresholds (matches indigo: 50/s, 2500/hr, 20k/day)// blocking instead of dropping means these trigger TCP backpressure on the upstream PDSconst default_per_second_limit: u64 = 50;const default_per_hour_limit: u64 = 2_500;const default_per_day_limit: u64 = 20_000;
// trusted hosts get much higher limits (Go relay: 5000/s, 50M/hr, 500M/day)const trusted_per_second_limit: u64 = 5_000;const trusted_per_hour_limit: u64 = 50_000_000;const trusted_per_day_limit: u64 = 500_000_000;
// Go relay: TrustedDomains config — hosts matching these suffixes get trusted limitsconst trusted_suffixes: []const []const u8 = &.{".host.bsky.network"};
pub fn isTrustedHost(hostname: []const u8) bool { for (trusted_suffixes) |suffix| { if (std.mem.endsWith(u8, hostname, suffix)) return true; } return false;}
/// compute rate limits scaled by account count (matches Go relay: slurper.go ComputeLimiterCounts)pub fn computeLimits(trusted: bool, account_count: u64) struct { sec: u64, hour: u64, day: u64 } { if (trusted) return .{ .sec = trusted_per_second_limit, .hour = trusted_per_hour_limit, .day = trusted_per_day_limit, }; return .{ .sec = default_per_second_limit + account_count / 1000, .hour = default_per_hour_limit + account_count, .day = default_per_day_limit + account_count * 10, };}
pub const Options = struct { hostname: []const u8 = "bsky.network", max_message_size: usize = 5 * 1024 * 1024, host_id: u64 = 0, account_count: u64 = 0, ca_bundle: ?std.crypto.Certificate.Bundle = null,};
fn timestamp(io: Io) i64 { return @intCast(@divFloor(Io.Timestamp.now(io, .real).nanoseconds, std.time.ns_per_s));}
fn milliTimestamp(io: Io) i64 { return Io.Timestamp.now(io, .real).toMilliseconds();}
fn microTimestamp(io: Io) i64 { return Io.Timestamp.now(io, .real).toMicroseconds();}
fn nanoTimestamp(io: Io) i96 { return Io.Timestamp.now(io, .real).toNanoseconds();}
/// simple sliding window rate limiter — tracks event counts per second/hour/day./// Sliding window rate limiter (same algorithm as Go relay's github.com/RussellLuo/slidingwindow)./// Uses millisecond timestamps for sub-second precision (critical for the 1-second window)./// Effective count = weight * prev_count + curr_count/// where weight = (window_size - elapsed) / window_size.const RateLimiter = struct { sec: SlidingWindow = .{ .size_ms = 1_000 }, hour: SlidingWindow = .{ .size_ms = 3_600_000 }, day: SlidingWindow = .{ .size_ms = 86_400_000 },
sec_limit: std.atomic.Value(u64) = .{ .raw = default_per_second_limit }, hour_limit: std.atomic.Value(u64) = .{ .raw = default_per_hour_limit }, day_limit: std.atomic.Value(u64) = .{ .raw = default_per_day_limit },
const SlidingWindow = struct { size_ms: i64, curr_start: i64 = 0, curr_count: u64 = 0, prev_count: u64 = 0,
fn advance(self: *SlidingWindow, now_ms: i64) void { const new_start = now_ms - @mod(now_ms, self.size_ms); if (new_start <= self.curr_start) return; const diff = @divTrunc(new_start - self.curr_start, self.size_ms); self.prev_count = if (diff == 1) self.curr_count else 0; self.curr_start = new_start; self.curr_count = 0; }
fn effectiveCount(self: *const SlidingWindow, now_ms: i64) u64 { const elapsed = now_ms - self.curr_start; const remaining = self.size_ms - elapsed; if (remaining <= 0 or self.prev_count == 0) return self.curr_count; const weighted_prev = self.prev_count * @as(u64, @intCast(remaining)) / @as(u64, @intCast(self.size_ms)); return weighted_prev + self.curr_count; } };
const Result = enum { allowed, sec, hour, day };
fn allow(self: *RateLimiter, now_ms: i64) Result { self.sec.advance(now_ms); self.hour.advance(now_ms); self.day.advance(now_ms);
if (self.sec.effectiveCount(now_ms) >= self.sec_limit.load(.monotonic)) return .sec; if (self.hour.effectiveCount(now_ms) >= self.hour_limit.load(.monotonic)) return .hour; if (self.day.effectiveCount(now_ms) >= self.day_limit.load(.monotonic)) return .day;
self.sec.curr_count += 1; self.hour.curr_count += 1; self.day.curr_count += 1; return .allowed; }
/// Block until all rate limit windows allow the event. /// Checks every 100ms, matching indigo's waitForLimiter behavior. /// Returns which tier (if any) caused a wait, for metrics. fn waitForAllow(self: *RateLimiter, shutdown: *std.atomic.Value(bool), io: Io) Result { // fast path: no waiting needed const now_ms = milliTimestamp(io); self.sec.advance(now_ms); self.hour.advance(now_ms); self.day.advance(now_ms);
if (self.sec.effectiveCount(now_ms) < self.sec_limit.load(.monotonic) and self.hour.effectiveCount(now_ms) < self.hour_limit.load(.monotonic) and self.day.effectiveCount(now_ms) < self.day_limit.load(.monotonic)) { self.sec.curr_count += 1; self.hour.curr_count += 1; self.day.curr_count += 1; return .allowed; }
// slow path: poll every 100ms until allowed (creates TCP backpressure) var waited: Result = .sec; while (!shutdown.load(.acquire)) { io.sleep(Io.Duration.fromMilliseconds(100), .awake) catch {};
const t = milliTimestamp(io); self.sec.advance(t); self.hour.advance(t); self.day.advance(t);
if (self.sec.effectiveCount(t) >= self.sec_limit.load(.monotonic)) { waited = .sec; continue; } if (self.hour.effectiveCount(t) >= self.hour_limit.load(.monotonic)) { waited = .hour; continue; } if (self.day.effectiveCount(t) >= self.day_limit.load(.monotonic)) { waited = .day; continue; }
self.sec.curr_count += 1; self.hour.curr_count += 1; self.day.curr_count += 1; return waited; } return waited; }
/// update rate limits from another thread (e.g. admin API). pub fn updateLimits(self: *RateLimiter, sec: u64, hour: u64, day: u64) void { self.sec_limit.store(sec, .monotonic); self.hour_limit.store(hour, .monotonic); self.day_limit.store(day, .monotonic); }};
/// semaphore that limits concurrent connection attempts across all subscriber threads./// bounds the DNS/TLS storm during startup and reconnect storms.pub const ConnectGate = struct { active: std.atomic.Value(u32) = .{ .raw = 0 }, limit: u32,};
pub const Subscriber = struct { allocator: Allocator, io: Io, options: Options, bc: *broadcaster.Broadcaster, validator: *validator_mod.Validator, persist: ?*event_log_mod.DiskPersist, collection_index: ?*collection_index_mod.CollectionIndex = null, resyncer: ?*resync_mod.Resyncer = null, pool: ?*frame_worker_mod.FramePool = null, /// dedicated Threaded io for frame workers — safe from plain OS threads pool_io: ?Io = null, /// host ops queue — pushes rare DB ops to a background thread host_ops: ?*host_ops_mod.HostOpsQueue = null, /// coalescing cursor map — subscriber writes latest seq atomically, worker sweeps every 5s cursor_map: ?*host_ops_mod.CursorMap = null, cursor_slot: ?u32 = null, /// connect gate — limits concurrent DNS/TLS handshakes across all subscribers connect_gate: ?*ConnectGate = null, shutdown: *std.atomic.Value(bool), last_upstream_seq: ?u64 = null, last_cursor_flush: i64 = 0, rate_limiter: RateLimiter = .{}, /// exponential backoff (seconds) between reconnect attempts — persists across retries, /// reset to 1 on successful connect. max 1800s (30 min) for dormant-eligible hosts. backoff: u64 = 1,
// per-host shutdown (e.g. FutureCursor — stops only this subscriber) host_shutdown: std.atomic.Value(bool) = .{ .raw = false },
pub fn init( allocator: Allocator, io: Io, bc: *broadcaster.Broadcaster, val: *validator_mod.Validator, persist: ?*event_log_mod.DiskPersist, shutdown: *std.atomic.Value(bool), options: Options, ) Subscriber { const trusted = isTrustedHost(options.hostname); const limits = computeLimits(trusted, options.account_count); return .{ .allocator = allocator, .io = io, .options = options, .bc = bc, .validator = val, .persist = persist, .shutdown = shutdown, .rate_limiter = .{ .sec_limit = .{ .raw = limits.sec }, .hour_limit = .{ .raw = limits.hour }, .day_limit = .{ .raw = limits.day }, }, }; }
/// check if this subscriber should stop (global or per-host shutdown) fn shouldStop(self: *Subscriber) bool { return self.shutdown.load(.acquire) or self.host_shutdown.load(.acquire); }
/// release connect gate permit (idempotent-safe: only decrements if gate exists) fn releaseConnectGate(self: *Subscriber) void { if (self.connect_gate) |gate| _ = gate.active.fetchSub(1, .monotonic); }
/// run the subscriber loop. reconnects with exponential backoff. /// blocks until shutdown or host marked dormant. pub fn run(self: *Subscriber) void { const max_backoff: u64 = 1800;
// cursor is set at spawn time by slurper if (self.last_upstream_seq) |seq| { log.info("host {s}: resuming from cursor {d}", .{ self.options.hostname, seq }); }
while (!self.shouldStop()) { // acquire connect permit — limits concurrent DNS/TLS handshakes. // released immediately after connectAndRead returns (success or failure) // so the permit is not held during backoff sleep. if (self.connect_gate) |gate| { while (gate.active.load(.acquire) >= gate.limit) { if (self.shouldStop()) return; self.io.sleep(Io.Duration.fromMilliseconds(100), .awake) catch {}; } _ = gate.active.fetchAdd(1, .monotonic); }
log.info("host {s}: connecting...", .{self.options.hostname});
self.connectAndRead() catch |err| { if (self.shouldStop()) { self.releaseConnectGate(); return; } log.err("host {s}: error: {s}, reconnecting in {d}s...", .{ self.options.hostname, @errorName(err), self.backoff }); }; // release permit after connectAndRead completes (success or failure) — // connected subscribers reading frames don't count against the limit, // and failed subscribers shouldn't hold permits during backoff sleep self.releaseConnectGate();
if (self.shouldStop()) return;
// track failures for this host (pushed to background thread via host_ops queue) if (self.options.host_id > 0) { if (self.host_ops) |hq| { hq.push(.{ .host_id = self.options.host_id, .kind = .increment_failures, .payload = .{ .host_shutdown = &self.host_shutdown }, }); } }
// backoff sleep in small increments so we can check shutdown var remaining: u64 = self.backoff; while (remaining > 0 and !self.shouldStop()) { const chunk = @min(remaining, 1); self.io.sleep(Io.Duration.fromSeconds(@intCast(chunk)), .awake) catch {}; remaining -= chunk; } self.backoff = @min(self.backoff * 2, max_backoff); } }
/// flush cursor position (atomic store to coalescing map — worker sweeps every 5s) fn flushCursor(self: *Subscriber) void { if (self.options.host_id == 0) return; const seq = self.last_upstream_seq orelse return; if (self.cursor_map) |cm| { if (self.cursor_slot) |slot| { cm.update(slot, seq); } } }
fn connectAndRead(self: *Subscriber) !void { var path_buf: [256]u8 = undefined; var w: std.Io.Writer = .fixed(&path_buf);
try w.writeAll("/xrpc/com.atproto.sync.subscribeRepos"); if (self.last_upstream_seq) |cursor| { try w.print("?cursor={d}", .{cursor}); } const path = w.buffered();
// DNS + TCP connect through pool_io (Threaded — has working netLookup). // The resulting fd is used by the Evented io for TLS + WebSocket I/O. const dns_io = self.pool_io orelse self.io; const host_name = try Io.net.HostName.init(self.options.hostname); const net_stream = try host_name.connect(dns_io, 443, .{ .mode = .stream });
var client = try websocket.Client.initWithStream(self.io, self.allocator, net_stream, .{ .host = self.options.hostname, .port = 443, .tls = true, .max_size = self.options.max_message_size, .ca_bundle = self.options.ca_bundle, }); defer client.deinit();
var host_header_buf: [256]u8 = undefined; const host_header = std.fmt.bufPrint( &host_header_buf, "Host: {s}\r\n", .{self.options.hostname}, ) catch self.options.hostname;
try client.handshake(path, .{ .headers = host_header }); log.info("host {s}: connected", .{self.options.hostname});
// reset failures + backoff on successful connect (via host_ops queue) self.backoff = 1; if (self.options.host_id > 0) { if (self.host_ops) |hq| { hq.push(.{ .host_id = self.options.host_id, .kind = .reset_failures, .payload = .{ .none = {} }, }); } }
var handler = FrameHandler{ .subscriber = self, };
// heartbeat read loop: merged keepalive + frame reading in a single task. // SO_RCVTIMEO fires after interval_ms, triggering a ping. closes after // max_failures consecutive idle intervals with no frames received. try client.readLoopWithHeartbeat(&handler, .{ .interval_ms = ping_interval_sec * 1000, .max_failures = max_ping_failures, }); }};
const FrameHandler = struct { subscriber: *Subscriber,
pub fn serverMessage(self: *FrameHandler, data: []const u8) !void { const sub = self.subscriber; const io = sub.io;
// lightweight header decode for cursor tracking + routing var arena = std.heap.ArenaAllocator.init(sub.allocator); defer arena.deinit(); const alloc = arena.allocator();
const header_result = zat.cbor.decode(alloc, data) catch |err| { log.debug("frame header decode failed: {s} (len={d})", .{ @errorName(err), data.len }); _ = sub.bc.stats.decode_errors.fetchAdd(1, .monotonic); return; }; const header = header_result.value; const payload_data = data[header_result.consumed..];
// check op field (1 = message, -1 = error) const op = header.getInt("op") orelse return; if (op == -1) { // error frame from upstream — check for FutureCursor // Go relay: slurper.go — sets host to idle and disconnects if (zat.cbor.decodeAll(alloc, payload_data) catch null) |err_payload| { const err_name = err_payload.getString("error") orelse "unknown"; const err_msg = err_payload.getString("message") orelse ""; log.warn("host {s}: error frame: {s}: {s}", .{ sub.options.hostname, err_name, err_msg }); if (std.mem.eql(u8, err_name, "FutureCursor")) { // our cursor is ahead of the PDS — set host to idle, stop this subscriber only if (sub.options.host_id > 0) { if (sub.host_ops) |hq| { hq.push(.{ .host_id = sub.options.host_id, .kind = .update_status, .payload = .{ .status = host_ops_mod.HostOp.Payload.Status.init("idle") }, }); } } sub.host_shutdown.store(true, .release); } } return; }
const frame_type = header.getString("t") orelse return; const payload = zat.cbor.decodeAll(alloc, payload_data) catch |err| { log.debug("frame payload decode failed: {s} (type={s})", .{ @errorName(err), frame_type }); _ = sub.bc.stats.decode_errors.fetchAdd(1, .monotonic); return; };
// count every successfully decoded event (matches Go relay's events_received_counter) _ = sub.bc.stats.frames_in.fetchAdd(1, .monotonic);
// extract seq for cursor tracking (deferred until after pool accepts) const upstream_seq = payload.getUint("seq");
// time-based cursor flush (Go relay: every 4 seconds) { const now = timestamp(io); if (now - sub.last_cursor_flush >= cursor_flush_interval_sec) { sub.flushCursor(); sub.last_cursor_flush = now; } }
// per-host rate limiting — block until window opens (matches indigo's waitForLimiter) // blocking here stalls the websocket reader → TCP backpressure → PDS slows down switch (sub.rate_limiter.waitForAllow(sub.shutdown, io)) { .allowed => {}, .sec => { _ = sub.bc.stats.rate_limited.fetchAdd(1, .monotonic); _ = sub.bc.stats.rate_limited_sec.fetchAdd(1, .monotonic); }, .hour => { _ = sub.bc.stats.rate_limited.fetchAdd(1, .monotonic); _ = sub.bc.stats.rate_limited_hour.fetchAdd(1, .monotonic); }, .day => { _ = sub.bc.stats.rate_limited.fetchAdd(1, .monotonic); _ = sub.bc.stats.rate_limited_day.fetchAdd(1, .monotonic); }, }
// filter unknown frame types before submitting to pool (forward-compat) const is_commit = std.mem.eql(u8, frame_type, "#commit"); const is_sync = std.mem.eql(u8, frame_type, "#sync"); const is_account = std.mem.eql(u8, frame_type, "#account"); const is_identity = std.mem.eql(u8, frame_type, "#identity");
if (!is_commit and !is_sync and !is_account and !is_identity) { // advance cursor for intentionally skipped frames — // we won't process these on reconnect either if (upstream_seq) |s| sub.last_upstream_seq = s; return; }
// submit to frame pool for heavy processing (CBOR re-decode, validation, persist, broadcast) // route by DID so commits for the same account serialize (prevents chain break races) // blocks when queue is full — TCP backpressure propagates to upstream PDS (matches indigo) if (sub.pool) |pool| { const did_key: u64 = blk: { const d = payload.getString("repo") orelse payload.getString("did"); break :blk if (d) |s| std.hash.Wyhash.hash(0, s) else sub.options.host_id; }; const duped = sub.allocator.dupe(u8, data) catch return; // dupe hostname per-frame: subscriber teardown (slurper.runWorker) // frees sub.options.hostname after sub.run() returns, but FrameWorks // can still be queued in the pool. borrowing the slice would be a // use-after-free (corrupt hostnames in chain-break logs, etc.). const hostname_dup = sub.allocator.dupe(u8, sub.options.hostname) catch { sub.allocator.free(duped); return; }; const t0 = nanoTimestamp(io); if (pool.submit(did_key, .{ .data = duped, .host_id = sub.options.host_id, .hostname = hostname_dup, .allocator = sub.allocator, .io = sub.pool_io orelse sub.io, // pool_io (Threaded) for worker-safe ops .bc = sub.bc, .validator = sub.validator, .persist = sub.persist, .collection_index = sub.collection_index, .resyncer = sub.resyncer, }, sub.shutdown)) { // pool accepted — advance cursor past this frame _ = sub.bc.stats.pool_queued_bytes.fetchAdd(duped.len, .monotonic); if (upstream_seq) |s| sub.last_upstream_seq = s; if (nanoTimestamp(io) - t0 > 1_000_000) { // >1ms = had to wait _ = sub.bc.stats.pool_backpressure.fetchAdd(1, .monotonic); } } else { // shutdown requested — don't advance cursor so reconnect replays this frame sub.allocator.free(duped); sub.allocator.free(hostname_dup); } return; }
// fallback: no pool, process inline (original path for tests / standalone use) self.processInline(sub, alloc, data, payload, upstream_seq, frame_type, is_commit, is_sync, is_account, is_identity); }
/// inline processing path — used when no frame pool is configured (tests, standalone). /// this is the original serverMessage heavy processing logic. fn processInline( _: *FrameHandler, sub: *Subscriber, alloc: Allocator, data: []const u8, payload: zat.cbor.Value, upstream_seq: ?u64, _: []const u8, is_commit: bool, is_sync: bool, is_account: bool, is_identity: bool, ) void { const io = sub.io;
// extract DID: "repo" for commits, "did" for identity/account const did: ?[]const u8 = if (is_commit) payload.getString("repo") else payload.getString("did");
// on #identity event, evict cached signing key so next commit re-resolves if (is_identity) { if (did) |d| sub.validator.evictKey(d); }
// resolve DID → numeric UID for event header (host-aware) const uid: u64 = if (sub.persist) |dp| blk: { if (did) |d| { const result = dp.uidForDidFromHost(d, sub.options.host_id) catch break :blk @as(u64, 0);
// host authority enforcement (mirrors frame_worker path) if ((result.is_new or result.host_changed) and !is_identity) { switch (sub.validator.resolveHostAuthority(d, sub.options.host_id)) { .migrate => { dp.setAccountHostId(result.uid, sub.options.host_id) catch {}; if (result.host_changed) { log.info("host {s}: account migrated uid={d} did={s}", .{ sub.options.hostname, result.uid, d, }); } }, .reject => { log.info("host {s}: dropping event, host authority failed uid={d} did={s}", .{ sub.options.hostname, result.uid, d, }); _ = sub.bc.stats.failed.fetchAdd(1, .monotonic); _ = sub.bc.stats.failed_host_authority.fetchAdd(1, .monotonic); return; }, .accept => {}, } }
break :blk result.uid; } else break :blk @as(u64, 0); } else 0;
// process #account events: update upstream status // Go relay: ingest.go processAccountEvent if (is_account) { if (sub.persist) |dp| { if (uid > 0) { const is_active = payload.getBool("active") orelse false; const status_str = payload.getString("status"); const new_status: []const u8 = if (is_active) "active" else (status_str orelse "inactive"); dp.updateAccountUpstreamStatus(uid, new_status) catch |err| { log.debug("upstream status update failed: {s}", .{@errorName(err)}); };
// on any non-active status, remove all collection index entries // (covers deleted, takendown, suspended, deactivated, and future unknown statuses) if (!is_active) { if (sub.collection_index) |ci| { if (did) |d| { ci.removeAll(d) catch |err| { log.debug("collection removeAll failed: {s}", .{@errorName(err)}); }; } } } } } }
// for commits and syncs: check account is active, validate, extract state var commit_data_cid: ?[]const u8 = null; var commit_rev: ?[]const u8 = null; if (is_commit or is_sync) { // drop for inactive accounts if (sub.persist) |dp| { if (uid > 0) { const active = dp.isAccountActive(uid) catch true; if (!active) { _ = sub.bc.stats.skipped.fetchAdd(1, .monotonic); return; } } }
// rev checks: stale, future, chain continuity if (is_commit and uid > 0) { if (payload.getString("rev")) |incoming_rev| { // future-rev rejection if (zat.Tid.parse(incoming_rev)) |tid| { const rev_us: i64 = @intCast(tid.timestamp()); const now_us = microTimestamp(io); const skew_us: i64 = sub.validator.config.rev_clock_skew * 1_000_000; if (rev_us > now_us + skew_us) { log.info("host {s}: dropping future rev uid={d} rev={s}", .{ sub.options.hostname, uid, incoming_rev, }); _ = sub.bc.stats.failed.fetchAdd(1, .monotonic); _ = sub.bc.stats.failed_future_rev.fetchAdd(1, .monotonic); return; } }
if (sub.persist) |dp| { if (dp.getAccountState(uid, alloc) catch null) |prev| { // stale rev check if (std.mem.order(u8, incoming_rev, prev.rev) != .gt) { log.debug("host {s}: dropping stale commit uid={d} rev={s} <= {s}", .{ sub.options.hostname, uid, incoming_rev, prev.rev, }); _ = sub.bc.stats.skipped.fetchAdd(1, .monotonic); return; }
// chain continuity: since should match stored rev if (payload.getString("since")) |since| { if (!std.mem.eql(u8, since, prev.rev)) { log.info("host {s}: chain break uid={d} since={s} stored_rev={s}", .{ sub.options.hostname, uid, since, prev.rev, }); _ = sub.bc.stats.chain_breaks.fetchAdd(1, .monotonic); } }
// chain continuity: prevData CID should match stored data_cid if (payload.get("prevData")) |pd| { if (pd == .cid) { const prev_data_encoded = zat.multibase.encode(alloc, .base32lower, pd.cid.raw) catch ""; if (prev_data_encoded.len > 0 and prev.data_cid.len > 0 and !std.mem.eql(u8, prev_data_encoded, prev.data_cid)) { log.info("host {s}: chain break uid={d} prevData mismatch", .{ sub.options.hostname, uid, }); _ = sub.bc.stats.chain_breaks.fetchAdd(1, .monotonic); } } } } } } }
if (is_commit) { const result = sub.validator.validateCommit(payload); if (!result.valid) return; commit_data_cid = result.data_cid; commit_rev = result.commit_rev;
// track collections from commit ops (phase 1 live indexing) if (sub.collection_index) |ci| { if (did) |d| { if (payload.get("ops")) |ops| { ci.trackCommitOps(d, ops); } } } } else { // #sync: signature verification only, no ops/MST const result = sub.validator.validateSync(payload); if (!result.valid) return; commit_data_cid = result.data_cid; commit_rev = result.commit_rev;
// enqueue collection index resync — #sync means repo discontinuity if (sub.resyncer) |r| { if (did) |d| r.enqueue(d, sub.options.hostname); } } }
// determine event kind for persistence const kind: event_log_mod.EvtKind = if (is_commit) .commit else if (is_sync) .sync else if (is_account) .account else // is_identity (unknown types already filtered above) .identity;
// persist + resequence + enqueue under ordering lock to guarantee // broadcast_queue insertion order matches seq assignment order. if (sub.persist) |dp| { var spins: u64 = 0; while (sub.bc.persist_order.cmpxchgWeak(0, 1, .acquire, .monotonic) != null) { spins += 1; std.atomic.spinLoopHint(); } if (spins > 0) { _ = sub.bc.stats.persist_order_spins.fetchAdd(spins, .monotonic); }
const relay_seq = dp.persist(kind, uid, data) catch |err| { sub.bc.persist_order.store(0, .release); log.warn("persist failed: {s}", .{@errorName(err)}); return; }; sub.bc.stats.relay_seq.store(relay_seq, .release);
const broadcast_data = broadcaster.resequenceFrame(alloc, data, relay_seq) orelse data; const owned = sub.allocator.dupe(u8, broadcast_data) catch { sub.bc.persist_order.store(0, .release); return; }; sub.bc.broadcast_queue.push(relay_seq, owned, &sub.bc.stats); sub.bc.persist_order.store(0, .release);
// update per-DID state outside the ordering lock (Postgres round-trip) if ((is_commit or is_sync) and uid > 0) { if (commit_rev) |rev| { const cid_str: []const u8 = if (commit_data_cid) |cid_raw| zat.multibase.encode(alloc, .base32lower, cid_raw) catch "" else ""; if (dp.updateAccountState(uid, rev, cid_str)) |updated| { if (!updated) { log.debug("host {s}: stale state update lost race uid={d} rev={s}", .{ sub.options.hostname, uid, rev, }); } } else |err| { log.debug("account state update failed: {s}", .{@errorName(err)}); } } } } else { const seq = upstream_seq orelse 0; const owned = sub.allocator.dupe(u8, data) catch return; sub.bc.broadcast_queue.push(seq, owned, &sub.bc.stats); } }
pub fn close(self: *FrameHandler) void { log.info("host {s}: connection closed", .{self.subscriber.options.hostname}); // flush cursor on disconnect self.subscriber.flushCursor(); }};
// --- tests ---
test "decode frame via SDK and extract fields" { const cbor = zat.cbor;
var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator();
// build a commit frame using SDK encoder const header: cbor.Value = .{ .map = &.{ .{ .key = "op", .value = .{ .unsigned = 1 } }, .{ .key = "t", .value = .{ .text = "#commit" } }, } }; const payload: cbor.Value = .{ .map = &.{ .{ .key = "repo", .value = .{ .text = "did:plc:test123" } }, .{ .key = "seq", .value = .{ .unsigned = 12345 } }, .{ .key = "rev", .value = .{ .text = "3k2abc000000" } }, .{ .key = "time", .value = .{ .text = "2024-01-15T10:30:00Z" } }, } };
const header_bytes = try cbor.encodeAlloc(alloc, header); const payload_bytes = try cbor.encodeAlloc(alloc, payload);
var frame = try alloc.alloc(u8, header_bytes.len + payload_bytes.len); @memcpy(frame[0..header_bytes.len], header_bytes); @memcpy(frame[header_bytes.len..], payload_bytes);
// decode using SDK (same path as FrameHandler.serverMessage) const h_result = try cbor.decode(alloc, frame); const h = h_result.value; const p_data = frame[h_result.consumed..]; const p = try cbor.decodeAll(alloc, p_data);
try std.testing.expectEqualStrings("#commit", h.getString("t").?); try std.testing.expectEqual(@as(i64, 1), h.getInt("op").?); try std.testing.expectEqual(@as(i64, 12345), p.getInt("seq").?); try std.testing.expectEqualStrings("did:plc:test123", p.getString("repo").?);}
test "decode identity frame via SDK" { const cbor = zat.cbor;
var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator();
const header: cbor.Value = .{ .map = &.{ .{ .key = "op", .value = .{ .unsigned = 1 } }, .{ .key = "t", .value = .{ .text = "#identity" } }, } }; const payload: cbor.Value = .{ .map = &.{ .{ .key = "did", .value = .{ .text = "did:plc:alice" } }, .{ .key = "seq", .value = .{ .unsigned = 99 } }, .{ .key = "time", .value = .{ .text = "2024-01-15T10:30:00Z" } }, } };
const header_bytes = try cbor.encodeAlloc(alloc, header); const payload_bytes = try cbor.encodeAlloc(alloc, payload);
var frame = try alloc.alloc(u8, header_bytes.len + payload_bytes.len); @memcpy(frame[0..header_bytes.len], header_bytes); @memcpy(frame[header_bytes.len..], payload_bytes);
const h_result = try cbor.decode(alloc, frame); const h = h_result.value; const p_data = frame[h_result.consumed..]; const p = try cbor.decodeAll(alloc, p_data);
try std.testing.expectEqualStrings("#identity", h.getString("t").?); try std.testing.expectEqualStrings("did:plc:alice", p.getString("did").?); try std.testing.expectEqual(@as(i64, 99), p.getInt("seq").?);}
test "rate limiter enforces per-second limit" { var rl: RateLimiter = .{ .sec_limit = .{ .raw = 3 }, .hour_limit = .{ .raw = 1000 }, .day_limit = .{ .raw = 10000 } };
const now: i64 = 1_000_000_000; try std.testing.expectEqual(RateLimiter.Result.allowed, rl.allow(now)); try std.testing.expectEqual(RateLimiter.Result.allowed, rl.allow(now)); try std.testing.expectEqual(RateLimiter.Result.allowed, rl.allow(now)); try std.testing.expectEqual(RateLimiter.Result.sec, rl.allow(now));
// at start of next second, prev carries over fully → still blocked try std.testing.expectEqual(RateLimiter.Result.sec, rl.allow(now + 1000));
// two seconds later prev is forgotten try std.testing.expectEqual(RateLimiter.Result.allowed, rl.allow(now + 2000));}
test "rate limiter enforces per-hour limit" { var rl: RateLimiter = .{ .sec_limit = .{ .raw = 1000 }, .hour_limit = .{ .raw = 5 }, .day_limit = .{ .raw = 10000 } };
const now: i64 = 3_600_000 * 100; for (0..5) |_| { try std.testing.expectEqual(RateLimiter.Result.allowed, rl.allow(now)); } try std.testing.expectEqual(RateLimiter.Result.hour, rl.allow(now + 100));
// next hour, prev carries over → still blocked try std.testing.expectEqual(RateLimiter.Result.hour, rl.allow(now + 3_600_000));
// two hours later prev is forgotten try std.testing.expectEqual(RateLimiter.Result.allowed, rl.allow(now + 7_200_000));}
test "sliding window interpolates previous count by elapsed time" { var rl: RateLimiter = .{ .sec_limit = .{ .raw = 10 }, .hour_limit = .{ .raw = 1_000_000 }, .day_limit = .{ .raw = 1_000_000 } };
const base: i64 = 1_000_000_000; for (0..8) |_| { try std.testing.expectEqual(RateLimiter.Result.allowed, rl.allow(base)); }
// next second: prev=8, curr=0. effective=8, room for 2 try std.testing.expectEqual(RateLimiter.Result.allowed, rl.allow(base + 1000)); try std.testing.expectEqual(RateLimiter.Result.allowed, rl.allow(base + 1000)); try std.testing.expectEqual(RateLimiter.Result.sec, rl.allow(base + 1000));
// halfway through: prev weight = 500/1000 = 0.5, so weighted_prev = 4. curr=2. eff=6, room for 4 try std.testing.expectEqual(RateLimiter.Result.allowed, rl.allow(base + 1500)); try std.testing.expectEqual(RateLimiter.Result.allowed, rl.allow(base + 1500)); try std.testing.expectEqual(RateLimiter.Result.allowed, rl.allow(base + 1500)); try std.testing.expectEqual(RateLimiter.Result.allowed, rl.allow(base + 1500)); try std.testing.expectEqual(RateLimiter.Result.sec, rl.allow(base + 1500));}
test "waitForAllow blocks then allows after window advances" { const io = std.testing.io;
// verify that waitForAllow returns a non-.allowed result when the limit was hit, // indicating it had to wait. We use a tiny limit so the fast path is exhausted. var rl: RateLimiter = .{ .sec_limit = .{ .raw = 1 }, .hour_limit = .{ .raw = 1000 }, .day_limit = .{ .raw = 10000 } }; var shutdown = std.atomic.Value(bool){ .raw = false };
// first call takes the fast path try std.testing.expectEqual(RateLimiter.Result.allowed, rl.waitForAllow(&shutdown, io));
// second call must block (sec limit = 1), then return .sec after the window advances // this will sleep ~100ms+ until the sliding window allows it const before = milliTimestamp(io); const result = rl.waitForAllow(&shutdown, io); const elapsed = milliTimestamp(io) - before;
try std.testing.expectEqual(RateLimiter.Result.sec, result); try std.testing.expect(elapsed >= 100); // must have slept at least one 100ms poll}
test "waitForAllow respects shutdown" { const io = std.testing.io;
var rl: RateLimiter = .{ .sec_limit = .{ .raw = 1 }, .hour_limit = .{ .raw = 1000 }, .day_limit = .{ .raw = 10000 } }; var shutdown = std.atomic.Value(bool){ .raw = false };
// exhaust the limit _ = rl.waitForAllow(&shutdown, io);
// set shutdown before the next call shutdown.store(true, .release);
// should return immediately without blocking const before = milliTimestamp(io); _ = rl.waitForAllow(&shutdown, io); const elapsed = milliTimestamp(io) - before;
try std.testing.expect(elapsed < 50); // should not have slept}
test "trusted host detection" { try std.testing.expect(isTrustedHost("pds-123.host.bsky.network")); try std.testing.expect(isTrustedHost("abc.host.bsky.network")); try std.testing.expect(!isTrustedHost("bsky.network")); try std.testing.expect(!isTrustedHost("evil.bsky.network")); try std.testing.expect(!isTrustedHost("pds.example.com"));}
test "error frame (op=-1) is detected" { const cbor = zat.cbor;
var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator();
const header: cbor.Value = .{ .map = &.{ .{ .key = "op", .value = .{ .negative = -1 } }, .{ .key = "t", .value = .{ .text = "#info" } }, } };
const header_bytes = try cbor.encodeAlloc(alloc, header); const h_result = try cbor.decode(alloc, header_bytes); const h = h_result.value;
try std.testing.expectEqual(@as(i64, -1), h.getInt("op").?);}
// --- spec conformance tests ---
test "spec: unknown frame type (op=1, t=#unknown) is ignored" { // event stream spec: unknown t values must be ignored for forward-compat const cbor = zat.cbor;
var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator();
const header: cbor.Value = .{ .map = &.{ .{ .key = "op", .value = .{ .unsigned = 1 } }, .{ .key = "t", .value = .{ .text = "#unknown" } }, } }; const payload: cbor.Value = .{ .map = &.{ .{ .key = "did", .value = .{ .text = "did:plc:test123" } }, .{ .key = "seq", .value = .{ .unsigned = 1 } }, } };
const header_bytes = try cbor.encodeAlloc(alloc, header); const payload_bytes = try cbor.encodeAlloc(alloc, payload);
// decode header — verify it's a valid message with unknown type const h_result = try cbor.decode(alloc, header_bytes); const h = h_result.value;
try std.testing.expectEqual(@as(i64, 1), h.getInt("op").?); const frame_type = h.getString("t").?; try std.testing.expectEqualStrings("#unknown", frame_type);
// verify unknown type is NOT one of the known types (this is the filter logic) const is_commit = std.mem.eql(u8, frame_type, "#commit"); const is_sync = std.mem.eql(u8, frame_type, "#sync"); const is_account = std.mem.eql(u8, frame_type, "#account"); const is_identity = std.mem.eql(u8, frame_type, "#identity"); try std.testing.expect(!is_commit and !is_sync and !is_account and !is_identity);
// verify payload still decodes (frame is valid, just ignored) const p = try cbor.decodeAll(alloc, payload_bytes); try std.testing.expectEqualStrings("did:plc:test123", p.getString("did").?);}
test "spec: error frame (op=-1) is handled, not persisted" { // event stream spec: op=-1 frames are error notifications from upstream const cbor = zat.cbor;
var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator();
const header: cbor.Value = .{ .map = &.{ .{ .key = "op", .value = .{ .negative = -1 } }, .{ .key = "t", .value = .{ .text = "#error" } }, } }; const err_payload: cbor.Value = .{ .map = &.{ .{ .key = "error", .value = .{ .text = "FutureCursor" } }, .{ .key = "message", .value = .{ .text = "cursor is ahead of server" } }, } };
const header_bytes = try cbor.encodeAlloc(alloc, header); const h_result = try cbor.decode(alloc, header_bytes); const h = h_result.value;
// verify op=-1 is detected const op = h.getInt("op").?; try std.testing.expectEqual(@as(i64, -1), op);
// verify error payload decodes correctly const payload_bytes = try cbor.encodeAlloc(alloc, err_payload); const p = try cbor.decodeAll(alloc, payload_bytes); try std.testing.expectEqualStrings("FutureCursor", p.getString("error").?); try std.testing.expectEqualStrings("cursor is ahead of server", p.getString("message").?);}