Something went wrong. Try again.
atproto relay in zig zlay.waow.tech
relay zig atproto
Something went wrong. Try again.
Zig
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434//! collection index backfill — discovers collections and imports DIDs from a source relay//!//! two collection discovery sources (unioned)://! 1. lexicon garden llms.txt — ~700 known NSIDs//! 2. RBC column family scan — firehose-observed collections already in the index//!//! progress tracked in postgres (backfill_progress table) for resumability.//! triggered via POST /admin/backfill-collections, status via GET.
const std = @import("std");const Io = std.Io;const http = std.http;const pg = @import("pg");const collection_index_mod = @import("collection_index.zig");const event_log_mod = @import("event_log.zig");
const Allocator = std.mem.Allocator;const log = std.log.scoped(.backfill);
const FetchResult = struct { dids: [][]const u8, next_cursor: ?[]const u8,};
pub const Backfiller = struct { allocator: Allocator, collection_index: *collection_index_mod.CollectionIndex, persist: *event_log_mod.DiskPersist, running: std.atomic.Value(bool), thread: ?std.Thread, source: []const u8, io: Io, shutdown: *std.atomic.Value(bool),
fn db(self: *Backfiller) *pg.Pool { return self.persist.db; }
pub fn init( allocator: Allocator, collection_index: *collection_index_mod.CollectionIndex, persist: *event_log_mod.DiskPersist, io: Io, shutdown: *std.atomic.Value(bool), ) Backfiller { return .{ .allocator = allocator, .collection_index = collection_index, .persist = persist, .running = .{ .raw = false }, .thread = null, .source = "", .io = io, .shutdown = shutdown, }; }
pub fn isRunning(self: *Backfiller) bool { return self.running.load(.acquire); }
/// block until any in-progress backfill thread completes. /// must be called before tearing down DiskPersist or CollectionIndex. pub fn waitForCompletion(self: *Backfiller) void { if (self.thread) |t| { t.join(); self.thread = null; } }
/// start a backfill from the given source relay. returns error if already running. pub fn start(self: *Backfiller, source: []const u8) !void { if (self.running.cmpxchgStrong(false, true, .acq_rel, .acquire) != null) { return error.AlreadyRunning; } errdefer self.running.store(false, .release);
self.source = try self.allocator.dupe(u8, source); errdefer { self.allocator.free(self.source); self.source = ""; } self.thread = std.Thread.spawn(.{}, run, .{self}) catch return error.SpawnFailed; }
fn run(self: *Backfiller) void { defer { self.allocator.free(self.source); self.source = ""; // note: do NOT clear self.thread here — waitForCompletion() needs // the handle to join this thread before dp/ci teardown. the running // flag gates start(), so a stale thread handle is harmless. self.running.store(false, .release); }
const pool = self.db();
// discover collections const collections = self.discoverCollections() catch |err| { log.err("collection discovery failed: {s}", .{@errorName(err)}); return; }; defer { for (collections) |c| self.allocator.free(c); self.allocator.free(collections); }
log.info("discovered {d} collections to backfill from {s}", .{ collections.len, self.source });
// insert progress rows (skip existing) for (collections) |collection| { _ = pool.exec( "INSERT INTO backfill_progress (collection, source) VALUES ($1, $2) ON CONFLICT (collection, source) DO NOTHING", .{ collection, self.source }, ) catch |err| { log.warn("failed to insert progress for {s}: {s}", .{ collection, @errorName(err) }); }; }
// backfill each collection for (collections) |collection| { if (self.shutdown.load(.acquire)) { log.info("backfill interrupted by shutdown", .{}); return; } self.backfillCollection(collection) catch |err| { log.warn("backfill failed for {s}: {s}", .{ collection, @errorName(err) }); }; }
log.info("backfill complete", .{}); }
fn discoverCollections(self: *Backfiller) ![][]const u8 { var seen: std.StringHashMapUnmanaged(void) = .empty; defer seen.deinit(self.allocator);
// source 1: lexicon garden const garden = self.fetchLexiconGarden() catch |err| blk: { log.warn("lexicon garden fetch failed: {s}", .{@errorName(err)}); break :blk &[_][]const u8{}; }; defer { for (garden) |c| self.allocator.free(c); self.allocator.free(garden); } for (garden) |c| { if (!seen.contains(c)) { const duped = try self.allocator.dupe(u8, c); errdefer self.allocator.free(duped); try seen.put(self.allocator, duped, {}); } }
// source 2: observed collections from RBC scan const observed = self.collection_index.listKnownCollections(self.allocator) catch |err| blk: { log.warn("RBC scan failed: {s}", .{@errorName(err)}); break :blk &[_][]const u8{}; }; defer { for (observed) |c| self.allocator.free(c); self.allocator.free(observed); } for (observed) |c| { if (!seen.contains(c)) { const duped = try self.allocator.dupe(u8, c); errdefer self.allocator.free(duped); try seen.put(self.allocator, duped, {}); } }
// collect into result const result = try self.allocator.alloc([]const u8, seen.count()); var i: usize = 0; var key_iter = seen.keyIterator(); while (key_iter.next()) |key| { result[i] = key.*; i += 1; }
return result; }
/// fetch NSIDs from https://lexicon.garden/llms.txt /// parses lines matching `- [`<nsid>`](` fn fetchLexiconGarden(self: *Backfiller) ![][]const u8 { var client: http.Client = .{ .allocator = self.allocator, .io = self.io }; defer client.deinit();
var aw: std.Io.Writer.Allocating = .init(self.allocator); defer aw.deinit();
const result = client.fetch(.{ .location = .{ .url = "https://lexicon.garden/llms.txt" }, .response_writer = &aw.writer, .method = .GET, }) catch return error.FetchFailed;
if (result.status != .ok) return error.FetchFailed;
const body = aw.toArrayList().items;
var nsids: std.ArrayListUnmanaged([]const u8) = .empty; defer nsids.deinit(self.allocator);
// parse lines like: - [`app.bsky.feed.post`]( var lines = std.mem.splitScalar(u8, body, '\n'); while (lines.next()) |line| { // look for pattern: - [`<nsid>`]( const backtick_start = std.mem.indexOf(u8, line, "- [`") orelse continue; const nsid_start = backtick_start + 4; // skip "- [`" const rest = line[nsid_start..]; const backtick_end = std.mem.indexOf(u8, rest, "`](") orelse continue; const nsid = rest[0..backtick_end];
// basic NSID validation: must contain at least one dot if (!std.mem.containsAtLeast(u8, nsid, 1, ".")) continue; // skip obviously non-record types (but include them anyway — they'll return empty)
const duped = try self.allocator.dupe(u8, nsid); errdefer self.allocator.free(duped); try nsids.append(self.allocator, duped); }
log.info("lexicon garden: found {d} NSIDs", .{nsids.items.len}); return try nsids.toOwnedSlice(self.allocator); }
fn backfillCollection(self: *Backfiller, collection: []const u8) !void { const pool = self.db();
// single query: check completion, get cursor + count for resume var cursor: ?[]const u8 = null; defer if (cursor) |c| self.allocator.free(c); var imported: i64 = 0; { var row = (pool.rowUnsafe( "SELECT completed_at IS NOT NULL, cursor, imported_count FROM backfill_progress WHERE collection = $1 AND source = $2", .{ collection, self.source }, ) catch return error.DatabaseError) orelse return; defer row.deinit() catch {};
if (row.get(bool, 0)) return; // already done
const saved_cursor = row.get([]const u8, 1); imported = row.get(i64, 2);
if (saved_cursor.len > 0) { cursor = try self.allocator.dupe(u8, saved_cursor); log.info("{s}: resuming from cursor (imported {d} so far)", .{ collection, imported }); } }
// reuse one HTTP client across all pages for this collection var client: http.Client = .{ .allocator = self.allocator, .io = self.io }; defer client.deinit();
var page_count: usize = 0; while (!self.shutdown.load(.acquire)) { const fetch_result = self.fetchPage(&client, collection, cursor) catch |err| { log.warn("{s}: fetch page failed: {s}", .{ collection, @errorName(err) }); break; }; defer { for (fetch_result.dids) |d| self.allocator.free(d); self.allocator.free(fetch_result.dids); if (fetch_result.next_cursor) |nc| self.allocator.free(nc); }
// add each DID to collection index for (fetch_result.dids) |did| { self.collection_index.addCollection(did, collection) catch {}; imported += 1; }
page_count += 1;
// update cursor in progress table const new_cursor = fetch_result.next_cursor orelse ""; _ = pool.exec( "UPDATE backfill_progress SET cursor = $1, imported_count = $2 WHERE collection = $3 AND source = $4", .{ new_cursor, imported, collection, self.source }, ) catch {};
if (fetch_result.next_cursor) |nc| { // advance cursor if (cursor) |old| self.allocator.free(old); cursor = self.allocator.dupe(u8, nc) catch break;
// brief pause between pages self.io.sleep(Io.Duration.fromMilliseconds(100), .awake) catch {}; } else { // no more pages — mark complete _ = pool.exec( "UPDATE backfill_progress SET completed_at = now(), cursor = '', imported_count = $1 WHERE collection = $2 AND source = $3", .{ imported, collection, self.source }, ) catch {}; log.info("{s}: complete ({d} DIDs, {d} pages)", .{ collection, imported, page_count }); break; } } }
fn fetchPage(self: *Backfiller, client: *http.Client, collection: []const u8, cursor: ?[]const u8) !FetchResult { // build URL var url_buf: [1024]u8 = undefined; const url = if (cursor) |c| std.fmt.bufPrint(&url_buf, "https://{s}/xrpc/com.atproto.sync.listReposByCollection?collection={s}&limit=1000&cursor={s}", .{ self.source, collection, c }) catch return error.UrlTooLong else std.fmt.bufPrint(&url_buf, "https://{s}/xrpc/com.atproto.sync.listReposByCollection?collection={s}&limit=1000", .{ self.source, collection }) catch return error.UrlTooLong;
var aw: std.Io.Writer.Allocating = .init(self.allocator); defer aw.deinit();
const result = client.fetch(.{ .location = .{ .url = url }, .response_writer = &aw.writer, .method = .GET, }) catch return error.FetchFailed;
if (result.status != .ok) return error.FetchFailed;
const body = aw.toArrayList().items;
// parse: {"repos": [{"did": "..."}, ...], "cursor": "..."} const parsed = std.json.parseFromSlice(ListReposResponse, self.allocator, body, .{ .ignore_unknown_fields = true }) catch return error.ParseFailed; defer parsed.deinit();
const repos = parsed.value.repos orelse return .{ .dids = try self.allocator.alloc([]const u8, 0), .next_cursor = null, };
var dids: std.ArrayListUnmanaged([]const u8) = .empty; defer dids.deinit(self.allocator);
for (repos) |repo| { const duped = self.allocator.dupe(u8, repo.did) catch continue; dids.append(self.allocator, duped) catch { self.allocator.free(duped); continue; }; }
const next_cursor = if (parsed.value.cursor) |c| self.allocator.dupe(u8, c) catch null else null;
return .{ .dids = dids.toOwnedSlice(self.allocator) catch return error.OutOfMemory, .next_cursor = next_cursor, }; }
const ListReposResponse = struct { repos: ?[]const RepoEntry = null, cursor: ?[]const u8 = null, };
const RepoEntry = struct { did: []const u8, };
/// return status summary for the admin endpoint pub fn getStatus(self: *Backfiller, allocator: Allocator) ![]u8 { const pool = self.db();
var aw: Io.Writer.Allocating = .init(allocator); defer aw.deinit(); const w = &aw.writer;
// query aggregate stats var total: i64 = 0; var completed: i64 = 0; var total_imported: i64 = 0; { var row = (pool.rowUnsafe( "SELECT COUNT(*)::bigint, COUNT(completed_at)::bigint, COALESCE(SUM(imported_count), 0)::bigint FROM backfill_progress", .{}, ) catch null) orelse null; if (row) |*r| { total = r.get(i64, 0); completed = r.get(i64, 1); total_imported = r.get(i64, 2); r.deinit() catch {}; } }
w.print("{{\"running\":{},\"total\":{d},\"completed\":{d},\"in_progress\":{d},\"total_imported\":{d},\"collections\":[", .{ self.isRunning(), total, completed, total - completed, total_imported, }) catch return error.FormatError;
// per-collection detail var result = pool.query( "SELECT collection, source, cursor, imported_count, completed_at IS NOT NULL FROM backfill_progress ORDER BY collection, source", .{}, ) catch return error.DatabaseError; defer result.deinit();
var first = true; while (result.nextUnsafe() catch null) |dbrow| { if (!first) w.writeByte(',') catch {}; first = false;
const collection = dbrow.get([]const u8, 0); const source = dbrow.get([]const u8, 1); const cursor_val = dbrow.get([]const u8, 2); const count = dbrow.get(i64, 3); const is_completed = dbrow.get(bool, 4);
w.print("{{\"collection\":\"{s}\",\"source\":\"{s}\",\"imported\":{d},\"completed\":{}", .{ collection, source, count, is_completed, }) catch {};
if (cursor_val.len > 0 and !is_completed) { w.print(",\"cursor\":\"{s}\"", .{cursor_val}) catch {}; }
w.writeByte('}') catch {}; }
w.writeAll("]}") catch {}; return try aw.toOwnedSlice(); }};