Something went wrong. Try again.
atproto relay in zig zlay.waow.tech
relay zig atproto
Something went wrong. Try again.
Zig
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643//! disk persistence matching indigo's diskpersist format//!//! append-only log files with relay-assigned sequence numbers.//! Postgres metadata index for fast cursor→file lookup.//!//! on-disk entry format (28-byte LE header + CBOR payload)://! [4B flags LE] [4B kind LE] [4B payload_len LE] [8B uid LE] [8B seq LE] [payload]//!//! file naming: evts-{startSeq} (rotated every events_per_file events)//!//! see: indigo cmd/relay/stream/persist/diskpersist/diskpersist.go
const std = @import("std");const Io = std.Io;const pg = @import("pg");const lru = @import("lru.zig");
const Allocator = std.mem.Allocator;const log = std.log.scoped(.relay);
// --- constants matching indigo ---
pub const header_size: usize = 28; // 4 + 4 + 4 + 8 + 8
pub const EvtKind = enum(u32) { commit = 1, handle = 2, // deprecated tombstone = 3, // deprecated identity = 4, account = 5, sync = 6,};
pub const EvtFlags = struct { pub const takedown: u32 = 1; pub const rebased: u32 = 2;};
const default_events_per_file: u32 = 10_000;const default_flush_interval_ms: u64 = 100;const default_flush_threshold: usize = 400;
// --- header ---
pub const EvtHeader = struct { flags: u32, kind: u32, len: u32, // payload length (not including header) uid: u64, seq: u64,
pub fn encode(self: EvtHeader, buf: *[header_size]u8) void { std.mem.writeInt(u32, buf[0..4], self.flags, .little); std.mem.writeInt(u32, buf[4..8], self.kind, .little); std.mem.writeInt(u32, buf[8..12], self.len, .little); std.mem.writeInt(u64, buf[12..20], self.uid, .little); std.mem.writeInt(u64, buf[20..28], self.seq, .little); }
pub fn decode(buf: *const [header_size]u8) EvtHeader { return .{ .flags = std.mem.readInt(u32, buf[0..4], .little), .kind = std.mem.readInt(u32, buf[4..8], .little), .len = std.mem.readInt(u32, buf[8..12], .little), .uid = std.mem.readInt(u64, buf[12..20], .little), .seq = std.mem.readInt(u64, buf[20..28], .little), }; }};
// --- persist job (write buffer entry) ---
const PersistJob = struct { data: []u8, // header + payload, owned seq: u64, // assigned seq (for broadcast ordering)};
// --- disk persistence ---
/// playback request — caller posts, pool_io worker executespub const PlaybackRequest = struct { since: u64, allocator: Allocator, entries: std.ArrayListUnmanaged(PlaybackEntry) = .empty, err: ?anyerror = null, done: std.atomic.Value(bool) = .{ .raw = false }, next: std.atomic.Value(?*PlaybackRequest) = .{ .raw = null },};
/// DB request — caller posts, pool_io worker executes./// callers define typed structs embedding DbRequest + @fieldParentPtr.pub const DbRequest = struct { callback: *const fn (*DbRequest, *DiskPersist) void, done: std.atomic.Value(bool) = .{ .raw = false }, err: ?anyerror = null,
/// spin-wait for completion, yielding via io.sleep when available. /// never returns until `done` is true — the request struct lives on the /// caller's stack, so we must outlive the worker's callback execution. /// the queue's shutdown drain sets done+err on unprocessed requests, /// and workers set done after the callback returns, so this always terminates. pub fn wait(self: *DbRequest, io: Io, shutdown: *std.atomic.Value(bool)) void { _ = shutdown; while (!self.done.load(.acquire)) { io.sleep(Io.Duration.fromMicroseconds(100), .awake) catch { // sleep failed (Io shutting down) — busy-wait for worker to finish while (!self.done.load(.acquire)) { std.atomic.spinLoopHint(); } break; }; } }};
/// MPSC FIFO ring buffer for general DB traffic./// multiple producers push via spinlock,/// multiple consumers (pool_io worker threads) pop via CAS.pub const DbRequestQueue = struct { const CAPACITY = 4096;
items: [CAPACITY]*DbRequest = undefined, head: std.atomic.Value(u32) = .{ .raw = 0 }, tail: std.atomic.Value(u32) = .{ .raw = 0 }, push_lock: std.atomic.Value(u32) = .{ .raw = 0 }, // producer spinlock shutdown: *std.atomic.Value(bool), persist: *DiskPersist,
pub fn push(self: *DbRequestQueue, req: *DbRequest) void { // acquire producer spinlock while (self.push_lock.cmpxchgWeak(0, 1, .acquire, .monotonic) != null) { std.atomic.spinLoopHint(); } defer self.push_lock.store(0, .release);
const tail = self.tail.load(.monotonic); const head = self.head.load(.acquire); // if full, spin until space opens (workers are fast, this shouldn't happen) if (tail -% head >= CAPACITY) { self.push_lock.store(0, .release); while (self.tail.load(.monotonic) -% self.head.load(.acquire) >= CAPACITY) { std.atomic.spinLoopHint(); } while (self.push_lock.cmpxchgWeak(0, 1, .acquire, .monotonic) != null) { std.atomic.spinLoopHint(); } }
self.items[self.tail.load(.monotonic) % CAPACITY] = req; self.tail.store(self.tail.load(.monotonic) +% 1, .release); }
pub fn pop(self: *DbRequestQueue) ?*DbRequest { while (true) { const head = self.head.load(.acquire); if (head == self.tail.load(.acquire)) return null; const item = self.items[head % CAPACITY]; if (self.head.cmpxchgWeak(head, head +% 1, .acq_rel, .acquire) == null) return item; } }
pub fn run(self: *DbRequestQueue, pool_io: Io) void { while (!self.shutdown.load(.acquire)) { if (self.pop()) |req| { req.callback(req, self.persist); req.done.store(true, .release); } else { pool_io.sleep(Io.Duration.fromMilliseconds(5), .awake) catch return; } } // shutdown drain — signal error on unprocessed requests while (self.pop()) |req| { req.err = error.ShuttingDown; req.done.store(true, .release); } }};
pub const DiskPersist = struct { allocator: Allocator, dir_path: []const u8, dir: Io.Dir, db: *pg.Pool, current_file: ?Io.File = null, current_file_path: ?[]const u8 = null, current_file_pos: u64 = 0,
// sequence state cur_seq: u64 = 1, event_counter: u64 = 0,
// config events_per_file: u32 = default_events_per_file, retention_hours: u64 = 72, // 3 days max_dir_bytes: u64 = 100 * 1024 * 1024 * 1024, // 100 GB
// DID → UID cache (matches indigo's bidirectional ARC cache) did_cache: lru.LruCache(u64),
// write buffer (flushed periodically or when threshold hit) outbuf: std.ArrayListUnmanaged(u8) = .empty, evtbuf: std.ArrayListUnmanaged(PersistJob) = .empty, mutex: Io.Mutex = Io.Mutex.init,
// flush task flush_future: ?Io.Future(void) = null, alive: std.atomic.Value(bool) = .{ .raw = true },
io: Io,
/// last successful DB interaction (epoch seconds, set by Threaded workers). /// read by metrics server to report health without direct pg.Pool access. last_db_success: std.atomic.Value(i64) = .{ .raw = 0 },
/// MPSC queue for playback requests (caller → pool_io worker) playback_head: std.atomic.Value(?*PlaybackRequest) = .{ .raw = null },
/// current evtbuf entry count (for metrics — non-blocking, returns 0 if lock is contended) pub fn evtbufLen(self: *DiskPersist) usize { if (!self.mutex.tryLock()) return 0; defer self.mutex.unlock(self.io); return self.evtbuf.items.len; }
/// current DID cache entry count (for metrics) pub fn didCacheLen(self: *DiskPersist) u32 { return self.did_cache.count(); }
/// DID cache hashmap backing capacity (for memory attribution) pub fn didCacheMapCap(self: *DiskPersist) u32 { return self.did_cache.mapCapacity(); }
/// evtbuf allocated capacity in jobs (for memory attribution — non-blocking) pub fn evtbufCap(self: *DiskPersist) usize { if (!self.mutex.tryLock()) return 0; defer self.mutex.unlock(self.io); return self.evtbuf.capacity; }
/// outbuf allocated capacity in bytes (for memory attribution — non-blocking) pub fn outbufCap(self: *DiskPersist) usize { if (!self.mutex.tryLock()) return 0; defer self.mutex.unlock(self.io); return self.outbuf.capacity; }
pub fn init(allocator: Allocator, dir_path: []const u8, database_url: []const u8, db_pool_size: u16, io: Io) !DiskPersist { // ensure directory exists try Io.Dir.cwd().createDirPath(io, dir_path);
var dir = try Io.Dir.cwd().openDir(io, dir_path, .{ .iterate = true }); errdefer dir.close(io);
// connect to Postgres const uri = std.Uri.parse(database_url) catch return error.InvalidDatabaseUrl; const pool = try pg.Pool.initUri(allocator, io, uri, .{ .size = db_pool_size }); errdefer pool.deinit();
// create tables (matching indigo's Go relay schema) _ = try pool.exec( \\CREATE TABLE IF NOT EXISTS log_file_refs ( \\ id BIGSERIAL PRIMARY KEY, \\ path TEXT NOT NULL, \\ archived BOOLEAN NOT NULL DEFAULT false, \\ seq_start BIGINT NOT NULL, \\ created_at TIMESTAMPTZ NOT NULL DEFAULT now() \\) , .{});
_ = try pool.exec( \\CREATE TABLE IF NOT EXISTS account ( \\ uid BIGSERIAL PRIMARY KEY, \\ did TEXT NOT NULL UNIQUE, \\ host_id BIGINT NOT NULL DEFAULT 0, \\ status TEXT NOT NULL DEFAULT 'active', \\ upstream_status TEXT NOT NULL DEFAULT 'active', \\ created_at TIMESTAMPTZ NOT NULL DEFAULT now() \\) , .{});
// migration: add columns if they don't exist (for existing deployments) _ = pool.exec("ALTER TABLE account ADD COLUMN IF NOT EXISTS host_id BIGINT NOT NULL DEFAULT 0", .{}) catch {}; _ = pool.exec("ALTER TABLE account ADD COLUMN IF NOT EXISTS upstream_status TEXT NOT NULL DEFAULT 'active'", .{}) catch {};
_ = try pool.exec( \\CREATE TABLE IF NOT EXISTS account_repo ( \\ uid BIGINT PRIMARY KEY REFERENCES account(uid), \\ rev TEXT NOT NULL, \\ commit_data_cid TEXT NOT NULL \\) , .{});
_ = try pool.exec( \\CREATE TABLE IF NOT EXISTS domain_ban ( \\ id BIGSERIAL PRIMARY KEY, \\ domain TEXT NOT NULL UNIQUE, \\ created_at TIMESTAMPTZ NOT NULL DEFAULT now() \\) , .{});
_ = try pool.exec( \\CREATE TABLE IF NOT EXISTS host ( \\ id BIGSERIAL PRIMARY KEY, \\ hostname TEXT NOT NULL UNIQUE, \\ status TEXT NOT NULL DEFAULT 'active', \\ last_seq BIGINT NOT NULL DEFAULT 0, \\ failed_attempts INTEGER NOT NULL DEFAULT 0, \\ account_limit BIGINT, \\ created_at TIMESTAMPTZ NOT NULL DEFAULT now(), \\ updated_at TIMESTAMPTZ NOT NULL DEFAULT now() \\) , .{});
// migration: add account_limit column to existing host tables _ = try pool.exec("ALTER TABLE host ADD COLUMN IF NOT EXISTS account_limit BIGINT", .{});
_ = try pool.exec( \\CREATE TABLE IF NOT EXISTS backfill_progress ( \\ collection TEXT NOT NULL, \\ source TEXT NOT NULL, \\ cursor TEXT NOT NULL DEFAULT '', \\ imported_count BIGINT NOT NULL DEFAULT 0, \\ completed_at TIMESTAMPTZ, \\ created_at TIMESTAMPTZ NOT NULL DEFAULT now(), \\ PRIMARY KEY (collection, source) \\) , .{});
// migrate: old schema had collection as sole PK — add source to composite PK _ = pool.exec( \\DO $$ BEGIN \\ IF EXISTS ( \\ SELECT 1 FROM pg_constraint \\ WHERE conname = 'backfill_progress_pkey' \\ AND conrelid = 'backfill_progress'::regclass \\ AND array_length(conkey, 1) = 1 \\ ) THEN \\ ALTER TABLE backfill_progress DROP CONSTRAINT backfill_progress_pkey; \\ ALTER TABLE backfill_progress ADD PRIMARY KEY (collection, source); \\ END IF; \\END $$ , .{}) catch {};
var self = DiskPersist{ .allocator = allocator, .dir_path = try allocator.dupe(u8, dir_path), .dir = dir, .db = pool, .did_cache = lru.LruCache(u64).init(allocator, 500_000, io), .io = io, };
// recover from existing log files try self.resumeLog();
return self; }
pub fn deinit(self: *DiskPersist) void { // stop flush task (loop checks alive after each sleep interval) self.alive.store(false, .release); if (self.flush_future) |*f| f.cancel(self.io);
// flush remaining self.mutex.lockUncancelable(self.io); self.flushLocked() catch {}; self.mutex.unlock(self.io);
// free write buffer for (self.evtbuf.items) |job| self.allocator.free(job.data); self.evtbuf.deinit(self.allocator); self.outbuf.deinit(self.allocator);
self.did_cache.deinit();
if (self.current_file) |f| f.close(self.io); if (self.current_file_path) |p| self.allocator.free(p); self.dir.close(self.io); self.db.deinit(); self.allocator.free(self.dir_path); }
/// start the background flush thread pub fn start(self: *DiskPersist) !void { self.flush_future = try self.io.concurrent(flushLoop, .{self}); }
pub const UidResult = struct { uid: u64, host_changed: bool = false, is_new: bool = false, };
/// resolve a DID to a numeric UID, checking host association. /// does NOT set host_id — caller must verify authority first via DID doc /// resolution, then call setAccountHostId on success. /// returns is_new=true when no host_id is set yet (first encounter), /// host_changed=true when host_id differs from stored value. pub fn uidForDidFromHost(self: *DiskPersist, did: []const u8, host_id: u64) !UidResult { const uid = try self.uidForDid(did); if (host_id > 0) { const current_host = self.getAccountHostId(uid) catch 0; if (current_host == 0) { return .{ .uid = uid, .is_new = true }; } else if (current_host != host_id) { log.info("account {s} (uid={d}) host mismatch: current={d} new={d}", .{ did, uid, current_host, host_id }); return .{ .uid = uid, .host_changed = true }; } } return .{ .uid = uid }; }
/// check DB health without touching pg.Pool — safe from any Io context. /// returns true if a Threaded worker successfully queried the DB within the last 30s. pub fn isDbHealthy(self: *DiskPersist) bool { const last = self.last_db_success.load(.acquire); if (last == 0) return false; var ts: std.c.timespec = undefined; _ = std.c.clock_gettime(.REALTIME, &ts); return (@as(i64, ts.sec) - last) < 30; }
fn markDbSuccess(self: *DiskPersist) void { var ts: std.c.timespec = undefined; _ = std.c.clock_gettime(.REALTIME, &ts); self.last_db_success.store(@as(i64, ts.sec), .release); }
/// resolve a DID to a numeric UID. creates a new account row on first encounter. /// matches indigo's Relay.DidToUid → Account.UID mapping. pub fn uidForDid(self: *DiskPersist, did: []const u8) !u64 { // fast path: check in-memory cache if (self.did_cache.get(did)) |uid| return uid;
// check database if (try self.db.rowUnsafe( "SELECT uid FROM account WHERE did = $1", .{did}, )) |row| { var r = row; defer r.deinit() catch {}; const uid: u64 = @intCast(r.get(i64, 0)); self.didCachePut(did, uid); self.markDbSuccess(); return uid; }
// create new account row (ignore if already exists from concurrent insert) _ = self.db.exec( "INSERT INTO account (did) VALUES ($1) ON CONFLICT (did) DO NOTHING", .{did}, ) catch |err| { log.warn("failed to create account for {s}: {s}", .{ did, @errorName(err) }); return err; };
// read back the UID (whether we just created it or it already existed) var row = try self.db.rowUnsafe( "SELECT uid FROM account WHERE did = $1", .{did}, ) orelse return error.AccountCreationFailed; defer row.deinit() catch {}; const uid: u64 = @intCast(row.get(i64, 0));
self.didCachePut(did, uid); return uid; }
/// insert into did_cache (LRU handles eviction at capacity). fn didCachePut(self: *DiskPersist, did: []const u8, uid: u64) void { self.did_cache.put(did, uid) catch {}; }
/// per-DID sync state for chain tracking pub const AccountState = struct { rev: []const u8, data_cid: []const u8, };
/// get stored sync state for a user (from account_repo table) pub fn getAccountState(self: *DiskPersist, uid: u64, allocator: Allocator) !?AccountState { var row = (try self.db.rowUnsafe( "SELECT rev, commit_data_cid FROM account_repo WHERE uid = $1", .{@as(i64, @intCast(uid))}, )) orelse return null; defer row.deinit() catch {}; const rev = row.get([]const u8, 0); const data_cid = row.get([]const u8, 1); if (rev.len == 0 or data_cid.len == 0) return null; return .{ .rev = try allocator.dupe(u8, rev), .data_cid = try allocator.dupe(u8, data_cid), }; }
/// update stored sync state after a verified commit (conditional upsert into account_repo). /// uses WHERE clause to ensure rev only moves forward, preventing race conditions /// when concurrent workers process commits for the same DID. /// returns true if the state was updated, false if a newer rev was already stored. pub fn updateAccountState(self: *DiskPersist, uid: u64, rev: []const u8, data_cid: []const u8) !bool { const rows_affected = try self.db.exec( "INSERT INTO account_repo (uid, rev, commit_data_cid) VALUES ($1, $2, $3) ON CONFLICT (uid) DO UPDATE SET rev = EXCLUDED.rev, commit_data_cid = EXCLUDED.commit_data_cid WHERE account_repo.rev < EXCLUDED.rev", .{ @as(i64, @intCast(uid)), rev, data_cid }, ); return (rows_affected orelse 0) > 0; }
// --- account status ---
/// get the host_id for an account. returns 0 if not set. pub fn getAccountHostId(self: *DiskPersist, uid: u64) !u64 { var row = (try self.db.rowUnsafe( "SELECT host_id FROM account WHERE uid = $1", .{@as(i64, @intCast(uid))}, )) orelse return 0; defer row.deinit() catch {}; const hid = row.get(i64, 0); return if (hid > 0) @intCast(hid) else 0; }
/// count accounts on a host (Threaded pool) pub fn getHostAccountCount(self: *DiskPersist, host_id: u64) u64 { return getHostAccountCountImpl(host_id, self.db); }
fn getHostAccountCountImpl(host_id: u64, db: *pg.Pool) u64 { var row = (db.rowUnsafe( "SELECT COUNT(*) FROM account WHERE host_id = $1", .{@as(i64, @intCast(host_id))}, ) catch return 0) orelse return 0; defer row.deinit() catch {}; const count = row.get(i64, 0); return if (count > 0) @intCast(count) else 0; }
/// effective account count (Threaded pool) pub fn getEffectiveAccountCount(self: *DiskPersist, host_id: u64) u64 { return getEffectiveAccountCountImpl(host_id, self.db); }
/// uses admin-configured limit if set, otherwise actual COUNT(*). fn getEffectiveAccountCountImpl(host_id: u64, db: *pg.Pool) u64 { var row = (db.rowUnsafe( "SELECT COALESCE(h.account_limit, COUNT(a.uid)) FROM host h LEFT JOIN account a ON a.host_id = h.id WHERE h.id = $1 GROUP BY h.id", .{@as(i64, @intCast(host_id))}, ) catch return 0) orelse return 0; defer row.deinit() catch {}; const count = row.get(i64, 0); return if (count > 0) @intCast(count) else 0; }
/// set host account limit (Threaded pool) pub fn setHostAccountLimit(self: *DiskPersist, host_id: u64, limit: ?u64) !void { return setHostAccountLimitImpl(host_id, limit, self.db); }
/// pass null to clear the override and revert to actual COUNT(*). fn setHostAccountLimitImpl(host_id: u64, limit: ?u64, db: *pg.Pool) !void { if (limit) |l| { const clamped: i64 = if (l > @as(u64, @intCast(std.math.maxInt(i64)))) std.math.maxInt(i64) else @intCast(l); _ = try db.exec( "UPDATE host SET account_limit = $2, updated_at = now() WHERE id = $1", .{ @as(i64, @intCast(host_id)), clamped }, ); } else { _ = try db.exec( "UPDATE host SET account_limit = NULL, updated_at = now() WHERE id = $1", .{@as(i64, @intCast(host_id))}, ); } }
/// set the host_id for an account (first encounter or migration) pub fn setAccountHostId(self: *DiskPersist, uid: u64, host_id: u64) !void { _ = try self.db.exec( "UPDATE account SET host_id = $2 WHERE uid = $1", .{ @as(i64, @intCast(uid)), @as(i64, @intCast(host_id)) }, ); }
/// update the upstream status for an account (from #account events). /// Go relay: account.go UpdateAccountUpstreamStatus pub fn updateAccountUpstreamStatus(self: *DiskPersist, uid: u64, upstream_status: []const u8) !void { _ = try self.db.exec( "UPDATE account SET upstream_status = $2 WHERE uid = $1", .{ @as(i64, @intCast(uid)), upstream_status }, ); }
/// check if an account is active (both local status and upstream status). /// Go relay: models.Account.IsActive() pub fn isAccountActive(self: *DiskPersist, uid: u64) !bool { var row = (try self.db.rowUnsafe( "SELECT status, upstream_status FROM account WHERE uid = $1", .{@as(i64, @intCast(uid))}, )) orelse return false; defer row.deinit() catch {}; const status = row.get([]const u8, 0); const upstream = row.get([]const u8, 1); // active if local is active AND upstream is active const local_ok = std.mem.eql(u8, status, "active"); const upstream_ok = std.mem.eql(u8, upstream, "active"); return local_ok and upstream_ok; }
// --- host management ---
pub const Host = struct { id: u64, hostname: []const u8, status: []const u8, last_seq: u64, failed_attempts: u32, account_limit: ?u64 = null, };
const HostResult = struct { id: u64, last_seq: u64 };
/// get or create a host row (Threaded pool) pub fn getOrCreateHost(self: *DiskPersist, hostname: []const u8) !HostResult { return getOrCreateHostImpl(hostname, self.db); }
fn getOrCreateHostImpl(hostname: []const u8, db: *pg.Pool) !HostResult { _ = db.exec( "INSERT INTO host (hostname) VALUES ($1) ON CONFLICT (hostname) DO NOTHING", .{hostname}, ) catch |err| { log.warn("failed to create host {s}: {s}", .{ hostname, @errorName(err) }); return err; };
var row = try db.rowUnsafe( "SELECT id, last_seq FROM host WHERE hostname = $1", .{hostname}, ) orelse return error.HostCreationFailed; defer row.deinit() catch {}; return .{ .id = @intCast(row.get(i64, 0)), .last_seq = @intCast(row.get(i64, 1)), }; }
/// check if a host is banned or blocked by status (Threaded pool) pub fn isHostBanned(self: *DiskPersist, hostname: []const u8) bool { return isHostBannedImpl(hostname, self.db); }
fn isHostBannedImpl(hostname: []const u8, db: *pg.Pool) bool { var row = db.rowUnsafe( "SELECT status FROM host WHERE hostname = $1", .{hostname}, ) catch return false; if (row) |*r| { defer r.deinit() catch {}; const status = r.get([]const u8, 0); return std.mem.eql(u8, status, "banned") or std.mem.eql(u8, status, "blocked"); } return false; }
/// update cursor position for a host pub fn updateHostSeq(self: *DiskPersist, host_id: u64, seq: u64) !void { _ = try self.db.exec( "UPDATE host SET last_seq = $2, updated_at = now() WHERE id = $1", .{ @as(i64, @intCast(host_id)), @as(i64, @intCast(seq)) }, ); }
/// look up host ID by hostname (Threaded pool) pub fn getHostIdForHostname(self: *DiskPersist, hostname: []const u8) !?u64 { return getHostIdForHostnameImpl(hostname, self.db); }
fn getHostIdForHostnameImpl(hostname: []const u8, db: *pg.Pool) !?u64 { var row = (try db.rowUnsafe( "SELECT id FROM host WHERE hostname = $1", .{hostname}, )) orelse return null; defer row.deinit() catch {}; return @intCast(row.get(i64, 0)); }
/// update host status (Threaded pool) pub fn updateHostStatus(self: *DiskPersist, host_id: u64, status: []const u8) !void { return updateHostStatusImpl(host_id, status, self.db); }
fn updateHostStatusImpl(host_id: u64, status: []const u8, db: *pg.Pool) !void { _ = try db.exec( "UPDATE host SET status = $2, updated_at = now() WHERE id = $1", .{ @as(i64, @intCast(host_id)), status }, ); }
/// list all active hosts (Threaded pool) pub fn listActiveHosts(self: *DiskPersist, allocator: Allocator) ![]Host { return listActiveHostsImpl(allocator, self.db); }
fn listActiveHostsImpl(allocator: Allocator, db: *pg.Pool) ![]Host { var hosts: std.ArrayListUnmanaged(Host) = .empty; errdefer { for (hosts.items) |host| { allocator.free(host.hostname); allocator.free(host.status); } hosts.deinit(allocator); }
var result = try db.query( "SELECT id, hostname, status, last_seq, failed_attempts, account_limit FROM host WHERE status = 'active' ORDER BY id ASC", .{}, ); defer result.deinit();
while (result.nextUnsafe() catch null) |row| { try hosts.append(allocator, .{ .id = @intCast(row.get(i64, 0)), .hostname = try allocator.dupe(u8, row.get([]const u8, 1)), .status = try allocator.dupe(u8, row.get([]const u8, 2)), .last_seq = @intCast(row.get(i64, 3)), .failed_attempts = @intCast(row.get(i32, 4)), .account_limit = if (row.get(?i64, 5)) |v| @as(?u64, @intCast(v)) else null, }); }
return try hosts.toOwnedSlice(allocator); }
/// list all hosts (Threaded pool) pub fn listAllHosts(self: *DiskPersist, allocator: Allocator) ![]Host { return listAllHostsImpl(allocator, self.db); }
fn listAllHostsImpl(allocator: Allocator, db: *pg.Pool) ![]Host { var hosts: std.ArrayListUnmanaged(Host) = .empty; errdefer { for (hosts.items) |host| { allocator.free(host.hostname); allocator.free(host.status); } hosts.deinit(allocator); }
var result = try db.query( "SELECT id, hostname, status, last_seq, failed_attempts, account_limit FROM host ORDER BY id ASC", .{}, ); defer result.deinit();
while (result.nextUnsafe() catch null) |row| { try hosts.append(allocator, .{ .id = @intCast(row.get(i64, 0)), .hostname = try allocator.dupe(u8, row.get([]const u8, 1)), .status = try allocator.dupe(u8, row.get([]const u8, 2)), .last_seq = @intCast(row.get(i64, 3)), .failed_attempts = @intCast(row.get(i32, 4)), .account_limit = if (row.get(?i64, 5)) |v| @as(?u64, @intCast(v)) else null, }); }
return try hosts.toOwnedSlice(allocator); }
/// increment failure count and return new value pub fn incrementHostFailures(self: *DiskPersist, host_id: u64) !u32 { _ = try self.db.exec( "UPDATE host SET failed_attempts = failed_attempts + 1, updated_at = now() WHERE id = $1", .{@as(i64, @intCast(host_id))}, ); var row = try self.db.rowUnsafe( "SELECT failed_attempts FROM host WHERE id = $1", .{@as(i64, @intCast(host_id))}, ) orelse return 0; defer row.deinit() catch {}; return @intCast(row.get(i32, 0)); }
/// check if a hostname (or any parent domain) is banned (Threaded pool). pub fn isDomainBanned(self: *DiskPersist, hostname: []const u8) bool { return self.isDomainBannedImpl(hostname, self.db); }
/// Go relay: domain_ban.go DomainIsBanned — suffix-based check. fn isDomainBannedImpl(_: *DiskPersist, hostname: []const u8, db: *pg.Pool) bool { // check each suffix: "pds.host.example.com", "host.example.com", "example.com" var offset: usize = 0; while (offset < hostname.len) { const suffix = hostname[offset..]; var row = db.rowUnsafe( "SELECT 1 FROM domain_ban WHERE domain = $1", .{suffix}, ) catch return false; if (row) |*r| { r.deinit() catch {}; return true; } // advance past next dot if (std.mem.indexOfScalarPos(u8, hostname, offset, '.')) |dot| { offset = dot + 1; } else break; } return false; }
/// reset failure count (Threaded pool) pub fn resetHostFailures(self: *DiskPersist, host_id: u64) !void { return resetHostFailuresImpl(host_id, self.db); }
fn resetHostFailuresImpl(host_id: u64, db: *pg.Pool) !void { _ = try db.exec( "UPDATE host SET failed_attempts = 0, updated_at = now() WHERE id = $1", .{@as(i64, @intCast(host_id))}, ); }
/// enqueue a playback request for the pool_io worker to execute. /// uses MPSC push (lock-free Treiber stack). pub fn enqueuePlayback(self: *DiskPersist, req: *PlaybackRequest) void { while (true) { const head = self.playback_head.load(.acquire); req.next.store(head, .monotonic); if (self.playback_head.cmpxchgWeak(@as(?*PlaybackRequest, head), req, .release, .monotonic) == null) { return; } } }
/// pop all pending playback requests (single-consumer — pool_io worker only). /// returns a singly-linked list via .next pointers, or null if empty. pub fn popPlaybackBatch(self: *DiskPersist) ?*PlaybackRequest { return self.playback_head.swap(null, .acq_rel); }
/// persist an event. assigns a sequence number. returns the assigned seq. /// the event is buffered and will be flushed to disk asynchronously. pub fn persist(self: *DiskPersist, kind: EvtKind, uid: u64, payload: []const u8) !u64 { // build the on-disk record: header + payload const total_len = header_size + payload.len; const data = try self.allocator.alloc(u8, total_len); errdefer self.allocator.free(data);
// write header (seq filled in later under lock) const header = EvtHeader{ .flags = 0, .kind = @intFromEnum(kind), .len = @intCast(payload.len), .uid = uid, .seq = 0, // placeholder }; header.encode(data[0..header_size]); @memcpy(data[header_size..], payload);
self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io);
// assign seq const seq = self.cur_seq; self.cur_seq += 1; std.mem.writeInt(u64, data[20..28], seq, .little);
try self.evtbuf.append(self.allocator, .{ .data = data, .seq = seq }); try self.outbuf.appendSlice(self.allocator, data); self.event_counter += 1;
// flush if threshold hit if (self.evtbuf.items.len >= default_flush_threshold) { try self.flushLocked(); }
return seq; }
/// playback events with seq > since. calls cb for each event. pub fn playback(self: *DiskPersist, since: u64, allocator: Allocator, entries: *std.ArrayListUnmanaged(PlaybackEntry)) !void { self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io);
const since_i: i64 = @intCast(since);
// find the log file containing `since` var start_files: std.ArrayListUnmanaged(LogFileRef) = .empty; defer start_files.deinit(allocator);
if (since > 0) { // find file whose seq_start is just before `since` if (try self.db.rowUnsafe( "SELECT id, path, seq_start FROM log_file_refs WHERE seq_start <= $1 ORDER BY seq_start DESC LIMIT 1", .{since_i}, )) |row| { var r = row; defer r.deinit() catch {}; try start_files.append(allocator, .{ .path = try allocator.dupe(u8, r.get([]const u8, 1)), .seq_start = @intCast(r.get(i64, 2)), }); } }
// find all subsequent files { var result = try self.db.query( "SELECT id, path, seq_start FROM log_file_refs WHERE seq_start > $1 ORDER BY seq_start ASC", .{since_i}, ); defer result.deinit(); while (result.nextUnsafe() catch null) |r| { try start_files.append(allocator, .{ .path = try allocator.dupe(u8, r.get([]const u8, 1)), .seq_start = @intCast(r.get(i64, 2)), }); } }
defer for (start_files.items) |f| allocator.free(f.path);
// read events from each file for (start_files.items) |ref| { var file = self.dir.openFile(self.io, ref.path, .{}) catch continue; defer file.close(self.io); try readEventsFrom(allocator, file, self.io, since, entries); } }
/// last assigned sequence number, or null if empty pub fn lastSeq(self: *DiskPersist) ?u64 { if (self.cur_seq <= 1) return null; return self.cur_seq - 1; }
/// oldest available sequence number (Threaded pool) pub fn firstSeq(self: *DiskPersist) ?u64 { return firstSeqImpl(self.db); }
fn firstSeqImpl(db: *pg.Pool) ?u64 { if (db.rowUnsafe( "SELECT seq_start FROM log_file_refs ORDER BY seq_start ASC LIMIT 1", .{}, ) catch null) |row| { var r = row; defer r.deinit() catch {}; const v = r.get(i64, 0); return if (v > 0) @intCast(v) else null; } return null; }
/// garbage-collect log files older than the retention period pub fn gc(self: *DiskPersist) !void { self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io);
const cutoff_interval = try std.fmt.allocPrint(self.allocator, "{d} hours", .{self.retention_hours}); defer self.allocator.free(cutoff_interval);
// find expired refs var expired: std.ArrayListUnmanaged(GcRef) = .empty; defer { for (expired.items) |e| self.allocator.free(e.path); expired.deinit(self.allocator); }
{ var result = try self.db.query( "SELECT id, path FROM log_file_refs WHERE created_at < now() - $1::interval", .{cutoff_interval}, ); defer result.deinit(); while (result.nextUnsafe() catch null) |r| { try expired.append(self.allocator, .{ .id = r.get(i64, 0), .path = try self.allocator.dupe(u8, r.get([]const u8, 1)), }); } }
for (expired.items) |ref| { // skip current file if (self.current_file_path) |cur| { if (std.mem.eql(u8, ref.path, cur)) continue; }
// delete db record first (prevents playback from finding it) _ = self.db.exec("DELETE FROM log_file_refs WHERE id = $1", .{ref.id}) catch |err| { log.warn("gc: failed to delete db record {d}: {s}", .{ ref.id, @errorName(err) }); continue; };
// delete file self.dir.deleteFile(self.io, ref.path) catch |err| { log.warn("gc: failed to delete {s}: {s}", .{ ref.path, @errorName(err) }); }; }
if (expired.items.len > 0) { log.info("gc: removed {d} expired log files", .{expired.items.len}); }
// size-based GC: delete oldest files until total size is under max_dir_bytes if (self.max_dir_bytes > 0) { try self.gcBySize(); }
self.markDbSuccess(); }
/// delete oldest event files until total directory size is under max_dir_bytes fn gcBySize(self: *DiskPersist) !void { // query all refs ordered by creation time (oldest first) var refs: std.ArrayListUnmanaged(GcSizeRef) = .empty; defer { for (refs.items) |e| self.allocator.free(e.path); refs.deinit(self.allocator); }
var total_bytes: u64 = 0; { var result = try self.db.query( "SELECT id, path FROM log_file_refs ORDER BY id ASC", .{}, ); defer result.deinit(); while (result.nextUnsafe() catch null) |r| { const path = r.get([]const u8, 1); const stat = self.dir.statFile(self.io, path, .{}) catch continue; total_bytes += stat.size; try refs.append(self.allocator, .{ .id = r.get(i64, 0), .path = try self.allocator.dupe(u8, path), .size = stat.size, }); } }
if (total_bytes <= self.max_dir_bytes) return;
log.info("gc: event dir size {d} MB exceeds limit {d} MB, pruning oldest files", .{ total_bytes / (1024 * 1024), self.max_dir_bytes / (1024 * 1024), });
var removed: u64 = 0; for (refs.items) |ref| { if (total_bytes <= self.max_dir_bytes) break;
// never delete the current file if (self.current_file_path) |cur| { if (std.mem.eql(u8, ref.path, cur)) continue; }
_ = self.db.exec("DELETE FROM log_file_refs WHERE id = $1", .{ref.id}) catch |err| { log.warn("gc: failed to delete db record {d}: {s}", .{ ref.id, @errorName(err) }); continue; };
self.dir.deleteFile(self.io, ref.path) catch |err| { log.warn("gc: failed to delete {s}: {s}", .{ ref.path, @errorName(err) }); continue; // don't subtract size if file delete failed };
total_bytes -= @min(total_bytes, ref.size); removed += 1; }
if (removed > 0) { log.info("gc: size-based pruning removed {d} files, dir now {d} MB", .{ removed, total_bytes / (1024 * 1024), }); } }
/// take down all events for a user (set flag + zero payload) pub fn takeDownUser(self: *DiskPersist, uid: u64) !void { self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io);
// iterate all log files var refs: std.ArrayListUnmanaged([]const u8) = .empty; defer { for (refs.items) |p| self.allocator.free(p); refs.deinit(self.allocator); }
{ var result = try self.db.query("SELECT path FROM log_file_refs ORDER BY seq_start DESC", .{}); defer result.deinit(); while (result.nextUnsafe() catch null) |r| { try refs.append(self.allocator, try self.allocator.dupe(u8, r.get([]const u8, 0))); } }
for (refs.items) |path| { var file = self.dir.openFile(self.io, path, .{ .mode = .read_write }) catch continue; defer file.close(self.io); mutateUserEventsInFile(file, self.io, uid) catch |err| { log.warn("takedown: failed to process {s}: {s}", .{ path, @errorName(err) }); }; } }
// --- internals ---
fn resumeLog(self: *DiskPersist) !void { // find most recent log file if (try self.db.rowUnsafe( "SELECT id, path, seq_start FROM log_file_refs ORDER BY seq_start DESC LIMIT 1", .{}, )) |row| { var r = row; defer r.deinit() catch {}; const path = r.get([]const u8, 1); const seq_start: u64 = @intCast(r.get(i64, 2));
var file = self.dir.openFile(self.io, path, .{ .mode = .read_write }) catch { // file missing, start fresh try self.initLogFile(); return; };
// scan for last seq const last_seq = scanForLastSeq(file, self.io) catch { file.close(self.io); try self.initLogFile(); return; };
if (last_seq) |ls| { self.cur_seq = ls + 1; log.info("recovered seq from disk: last_seq={d}", .{ls}); } else { self.cur_seq = if (seq_start > 0) seq_start else 1; }
// track position for appending (at end of file) const file_stat = try file.stat(self.io); self.current_file_pos = file_stat.size;
self.current_file = file; self.current_file_path = try self.allocator.dupe(u8, path); } else { try self.initLogFile(); } }
fn initLogFile(self: *DiskPersist) !void { try self.createLogFile(self.cur_seq); }
fn createLogFile(self: *DiskPersist, start_seq: u64) !void { var name_buf: [64]u8 = undefined; const name = std.fmt.bufPrint(&name_buf, "evts-{d}", .{start_seq}) catch unreachable;
if (self.current_file) |f| f.close(self.io); if (self.current_file_path) |p| self.allocator.free(p);
self.current_file = try self.dir.createFile(self.io, name, .{ .truncate = false, .read = true }); self.current_file_pos = 0; self.current_file_path = try self.allocator.dupe(u8, name);
// register in Postgres _ = try self.db.exec( "INSERT INTO log_file_refs (path, seq_start) VALUES ($1, $2)", .{ name, @as(i64, @intCast(start_seq)) }, );
self.event_counter = 0; }
fn flushLocked(self: *DiskPersist) !void { if (self.evtbuf.items.len == 0) return;
// write buffered bytes to current file const file = self.current_file orelse return; file.writePositionalAll(self.io, self.outbuf.items, self.current_file_pos) catch |err| { log.err("flush: write failed: {s}", .{@errorName(err)}); return err; }; self.current_file_pos += self.outbuf.items.len;
// clear buffers self.outbuf.clearRetainingCapacity();
// free job data for (self.evtbuf.items) |job| { self.allocator.free(job.data); } self.evtbuf.clearRetainingCapacity();
// check if we need to rotate if (self.event_counter >= self.events_per_file) { self.createLogFile(self.cur_seq) catch |err| { log.err("flush: log rotation failed: {s}", .{@errorName(err)}); }; } }
fn flushLoop(self: *DiskPersist) void { while (self.alive.load(.acquire)) { // sleep for flush interval (replaces timedWait — Io.Condition has no timed variant) self.io.sleep(Io.Duration.fromMilliseconds(@intCast(default_flush_interval_ms)), .awake) catch {}; self.mutex.lockUncancelable(self.io); self.flushLocked() catch {}; self.mutex.unlock(self.io); } }};
// --- playback types ---
pub const PlaybackEntry = struct { seq: u64, kind: u32, uid: u64, data: []const u8, // payload only (owned by caller)};
const LogFileRef = struct { path: []const u8, seq_start: u64,};
const GcRef = struct { id: i64, path: []const u8,};
const GcSizeRef = struct { id: i64, path: []const u8, size: u64,};
// --- file-level operations ---
fn readEventsFrom(allocator: Allocator, file: Io.File, io: Io, since: u64, result: *std.ArrayListUnmanaged(PlaybackEntry)) !void { const file_size = (try file.stat(io)).size;
// if since > 0, scan to the right position var pos: u64 = 0; if (since > 0) { pos = seekToSeq(file, io, since, file_size) catch return; }
// read events while (pos + header_size <= file_size) { var hdr_buf: [header_size]u8 = undefined; const n = file.readPositionalAll(io, &hdr_buf, pos) catch break; if (n < header_size) break; pos += header_size;
const hdr = EvtHeader.decode(&hdr_buf);
// skip taken down / rebased events if (hdr.flags & (EvtFlags.takedown | EvtFlags.rebased) != 0) { pos += hdr.len; continue; }
if (hdr.seq <= since) { pos += hdr.len; continue; }
// read payload const data = allocator.alloc(u8, hdr.len) catch break; const read_n = file.readPositionalAll(io, data, pos) catch { allocator.free(data); break; }; if (read_n < hdr.len) { allocator.free(data); break; } pos += hdr.len;
result.append(allocator, .{ .seq = hdr.seq, .kind = hdr.kind, .uid = hdr.uid, .data = data, }) catch { allocator.free(data); break; }; }}
/// scan file headers to find position of first event with seq > targetfn seekToSeq(file: Io.File, io: Io, target: u64, file_size: u64) !u64 { var pos: u64 = 0; while (pos + header_size <= file_size) { var hdr_buf: [header_size]u8 = undefined; const n = try file.readPositionalAll(io, &hdr_buf, pos); if (n < header_size) break;
const hdr = EvtHeader.decode(&hdr_buf); if (hdr.seq > target) { return pos; } pos += header_size + hdr.len; } return pos;}
/// scan a file for the last sequence numberfn scanForLastSeq(file: Io.File, io: Io) !?u64 { const file_size = (try file.stat(io)).size;
var last_seq: ?u64 = null; var pos: u64 = 0; while (pos + header_size <= file_size) { var hdr_buf: [header_size]u8 = undefined; const n = try file.readPositionalAll(io, &hdr_buf, pos); if (n < header_size) break;
const hdr = EvtHeader.decode(&hdr_buf); last_seq = hdr.seq; pos += header_size + hdr.len; } return last_seq;}
/// set takedown flag + zero payload for all events belonging to uidfn mutateUserEventsInFile(file: Io.File, io: Io, uid: u64) !void { const file_size = (try file.stat(io)).size; var pos: u64 = 0;
while (pos + header_size <= file_size) { var hdr_buf: [header_size]u8 = undefined; const n = try file.readPositionalAll(io, &hdr_buf, pos); if (n < header_size) break;
const hdr = EvtHeader.decode(&hdr_buf);
if (hdr.uid == uid and (hdr.flags & EvtFlags.takedown) == 0) { // set takedown flag const new_flags = hdr.flags | EvtFlags.takedown; var flags_buf: [4]u8 = undefined; std.mem.writeInt(u32, &flags_buf, new_flags, .little); try file.writePositionalAll(io, &flags_buf, pos);
// zero the payload const payload_start = pos + header_size; var zeros: [4096]u8 = [_]u8{0} ** 4096; var remaining: u64 = hdr.len; var write_pos: u64 = payload_start; while (remaining > 0) { const chunk: usize = @min(remaining, zeros.len); try file.writePositionalAll(io, zeros[0..chunk], write_pos); write_pos += chunk; remaining -= chunk; } }
pos += header_size + hdr.len; }}
// === tests ===
test "header encode/decode roundtrip" { const hdr = EvtHeader{ .flags = 0, .kind = @intFromEnum(EvtKind.commit), .len = 1234, .uid = 42, .seq = 99999, }; var buf: [header_size]u8 = undefined; hdr.encode(&buf); const decoded = EvtHeader.decode(&buf); try std.testing.expectEqual(hdr.flags, decoded.flags); try std.testing.expectEqual(hdr.kind, decoded.kind); try std.testing.expectEqual(hdr.len, decoded.len); try std.testing.expectEqual(hdr.uid, decoded.uid); try std.testing.expectEqual(hdr.seq, decoded.seq);}
test "header is little-endian" { const hdr = EvtHeader{ .flags = 0, .kind = 1, .len = 256, .uid = 0, .seq = 1 }; var buf: [header_size]u8 = undefined; hdr.encode(&buf); // len=256 in LE is 0x00 0x01 0x00 0x00 try std.testing.expectEqual(@as(u8, 0x00), buf[8]); try std.testing.expectEqual(@as(u8, 0x01), buf[9]);}
fn getenv(key: [*:0]const u8) ?[]const u8 { const ptr = std.c.getenv(key) orelse return null; return std.mem.sliceTo(ptr, 0);}
fn requireDatabaseUrl() ![]const u8 { return getenv("DATABASE_URL") orelse return error.SkipZigTest;}
test "persist and playback" { const database_url = try requireDatabaseUrl();
var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup();
const dir_path = try tmpDirRealPath(std.testing.allocator, tmp); defer std.testing.allocator.free(dir_path);
var dp = try DiskPersist.init(std.testing.allocator, dir_path, database_url, 5, std.testing.io); defer dp.deinit();
// persist some events (sync flush, no background thread) const seq1 = try dp.persist(.commit, 1, "payload-one"); const seq2 = try dp.persist(.identity, 2, "payload-two"); const seq3 = try dp.persist(.commit, 1, "payload-three");
try std.testing.expectEqual(@as(u64, 1), seq1); try std.testing.expectEqual(@as(u64, 2), seq2); try std.testing.expectEqual(@as(u64, 3), seq3);
// flush manually { dp.mutex.lockUncancelable(dp.io); defer dp.mutex.unlock(dp.io); try dp.flushLocked(); }
// playback from cursor=0 → all events var entries: std.ArrayListUnmanaged(PlaybackEntry) = .empty; defer { for (entries.items) |e| std.testing.allocator.free(e.data); entries.deinit(std.testing.allocator); } try dp.playback(0, std.testing.allocator, &entries);
try std.testing.expectEqual(@as(usize, 3), entries.items.len); try std.testing.expectEqualStrings("payload-one", entries.items[0].data); try std.testing.expectEqual(@as(u64, 1), entries.items[0].seq); try std.testing.expectEqualStrings("payload-two", entries.items[1].data); try std.testing.expectEqualStrings("payload-three", entries.items[2].data);}
test "playback with cursor" { const database_url = try requireDatabaseUrl();
var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup();
const dir_path = try tmpDirRealPath(std.testing.allocator, tmp); defer std.testing.allocator.free(dir_path);
var dp = try DiskPersist.init(std.testing.allocator, dir_path, database_url, 5, std.testing.io); defer dp.deinit();
_ = try dp.persist(.commit, 1, "a"); _ = try dp.persist(.commit, 2, "b"); _ = try dp.persist(.commit, 3, "c"); { dp.mutex.lockUncancelable(dp.io); defer dp.mutex.unlock(dp.io); try dp.flushLocked(); }
// playback from cursor=2 → only seq 3 var entries: std.ArrayListUnmanaged(PlaybackEntry) = .empty; defer { for (entries.items) |e| std.testing.allocator.free(e.data); entries.deinit(std.testing.allocator); } try dp.playback(2, std.testing.allocator, &entries);
try std.testing.expectEqual(@as(usize, 1), entries.items.len); try std.testing.expectEqual(@as(u64, 3), entries.items[0].seq); try std.testing.expectEqualStrings("c", entries.items[0].data);}
test "seq recovery after reinit" { const database_url = try requireDatabaseUrl();
var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup();
const dir_path = try tmpDirRealPath(std.testing.allocator, tmp); defer std.testing.allocator.free(dir_path);
// write some events { var dp = try DiskPersist.init(std.testing.allocator, dir_path, database_url, 5, std.testing.io); defer dp.deinit(); _ = try dp.persist(.commit, 1, "x"); _ = try dp.persist(.commit, 2, "y"); _ = try dp.persist(.account, 3, "z"); dp.mutex.lockUncancelable(dp.io); defer dp.mutex.unlock(dp.io); try dp.flushLocked(); }
// reinit — should recover seq { var dp = try DiskPersist.init(std.testing.allocator, dir_path, database_url, 5, std.testing.io); defer dp.deinit(); try std.testing.expectEqual(@as(u64, 3), dp.lastSeq().?); const seq4 = try dp.persist(.commit, 1, "w"); try std.testing.expectEqual(@as(u64, 4), seq4); }}
test "takedown zeros payload" { const database_url = try requireDatabaseUrl();
var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup();
const dir_path = try tmpDirRealPath(std.testing.allocator, tmp); defer std.testing.allocator.free(dir_path);
var dp = try DiskPersist.init(std.testing.allocator, dir_path, database_url, 5, std.testing.io); defer dp.deinit();
_ = try dp.persist(.commit, 42, "secret-data"); _ = try dp.persist(.commit, 99, "other-data"); { dp.mutex.lockUncancelable(dp.io); defer dp.mutex.unlock(dp.io); try dp.flushLocked(); }
// take down user 42 try dp.takeDownUser(42);
// playback should skip user 42's events var entries: std.ArrayListUnmanaged(PlaybackEntry) = .empty; defer { for (entries.items) |e| std.testing.allocator.free(e.data); entries.deinit(std.testing.allocator); } try dp.playback(0, std.testing.allocator, &entries);
try std.testing.expectEqual(@as(usize, 1), entries.items.len); try std.testing.expectEqualStrings("other-data", entries.items[0].data);}
test "uidForDid assigns and caches UIDs" { const database_url = try requireDatabaseUrl();
var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup();
const dir_path = try tmpDirRealPath(std.testing.allocator, tmp); defer std.testing.allocator.free(dir_path);
var dp = try DiskPersist.init(std.testing.allocator, dir_path, database_url, 5, std.testing.io); defer dp.deinit();
// first call creates the account const uid1 = try dp.uidForDid("did:plc:alice"); try std.testing.expect(uid1 > 0);
// same DID returns same UID (from cache) const uid1_again = try dp.uidForDid("did:plc:alice"); try std.testing.expectEqual(uid1, uid1_again);
// different DID gets a different UID const uid2 = try dp.uidForDid("did:plc:bob"); try std.testing.expect(uid2 > 0); try std.testing.expect(uid1 != uid2);}
test "uidForDid survives reinit" { const database_url = try requireDatabaseUrl();
var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup();
const dir_path = try tmpDirRealPath(std.testing.allocator, tmp); defer std.testing.allocator.free(dir_path);
var uid1: u64 = undefined; { var dp = try DiskPersist.init(std.testing.allocator, dir_path, database_url, 5, std.testing.io); defer dp.deinit(); uid1 = try dp.uidForDid("did:plc:carol"); }
// reinit — UID should be the same from database { var dp = try DiskPersist.init(std.testing.allocator, dir_path, database_url, 5, std.testing.io); defer dp.deinit(); const uid1_again = try dp.uidForDid("did:plc:carol"); try std.testing.expectEqual(uid1, uid1_again); }}
test "takedown with real UIDs" { const database_url = try requireDatabaseUrl();
var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup();
const dir_path = try tmpDirRealPath(std.testing.allocator, tmp); defer std.testing.allocator.free(dir_path);
var dp = try DiskPersist.init(std.testing.allocator, dir_path, database_url, 5, std.testing.io); defer dp.deinit();
const alice_uid = try dp.uidForDid("did:plc:alice"); const bob_uid = try dp.uidForDid("did:plc:bob");
_ = try dp.persist(.commit, alice_uid, "alice-post"); _ = try dp.persist(.commit, bob_uid, "bob-post"); _ = try dp.persist(.commit, alice_uid, "alice-post-2"); { dp.mutex.lockUncancelable(dp.io); defer dp.mutex.unlock(dp.io); try dp.flushLocked(); }
// take down alice try dp.takeDownUser(alice_uid);
// playback should only have bob's event var entries: std.ArrayListUnmanaged(PlaybackEntry) = .empty; defer { for (entries.items) |e| std.testing.allocator.free(e.data); entries.deinit(std.testing.allocator); } try dp.playback(0, std.testing.allocator, &entries);
try std.testing.expectEqual(@as(usize, 1), entries.items.len); try std.testing.expectEqualStrings("bob-post", entries.items[0].data); try std.testing.expectEqual(bob_uid, entries.items[0].uid);}
fn tmpDirRealPath(allocator: Allocator, tmp: std.testing.TmpDir) ![]const u8 { var buf: [Io.Dir.max_path_bytes]u8 = undefined; const n = try tmp.dir.realPath(std.testing.io, &buf); return try allocator.dupe(u8, buf[0..n]);}