//! collection index backfill — discovers collections and imports DIDs from a source relay //! //! two collection discovery sources (unioned): //! 1. lexicon garden llms.txt — ~700 known NSIDs //! 2. RBC column family scan — firehose-observed collections already in the index //! //! progress tracked in postgres (backfill_progress table) for resumability. //! triggered via POST /admin/backfill-collections, status via GET. const std = @import("std"); const Io = std.Io; const http = std.http; const pg = @import("pg"); const collection_index_mod = @import("collection_index.zig"); const event_log_mod = @import("event_log.zig"); const Allocator = std.mem.Allocator; const log = std.log.scoped(.backfill); const FetchResult = struct { dids: [][]const u8, next_cursor: ?[]const u8, }; pub const Backfiller = struct { allocator: Allocator, collection_index: *collection_index_mod.CollectionIndex, persist: *event_log_mod.DiskPersist, running: std.atomic.Value(bool), thread: ?std.Thread, source: []const u8, io: Io, shutdown: *std.atomic.Value(bool), fn db(self: *Backfiller) *pg.Pool { return self.persist.db; } pub fn init( allocator: Allocator, collection_index: *collection_index_mod.CollectionIndex, persist: *event_log_mod.DiskPersist, io: Io, shutdown: *std.atomic.Value(bool), ) Backfiller { return .{ .allocator = allocator, .collection_index = collection_index, .persist = persist, .running = .{ .raw = false }, .thread = null, .source = "", .io = io, .shutdown = shutdown, }; } pub fn isRunning(self: *Backfiller) bool { return self.running.load(.acquire); } /// block until any in-progress backfill thread completes. /// must be called before tearing down DiskPersist or CollectionIndex. pub fn waitForCompletion(self: *Backfiller) void { if (self.thread) |t| { t.join(); self.thread = null; } } /// start a backfill from the given source relay. returns error if already running. pub fn start(self: *Backfiller, source: []const u8) !void { if (self.running.cmpxchgStrong(false, true, .acq_rel, .acquire) != null) { return error.AlreadyRunning; } errdefer self.running.store(false, .release); self.source = try self.allocator.dupe(u8, source); errdefer { self.allocator.free(self.source); self.source = ""; } self.thread = std.Thread.spawn(.{}, run, .{self}) catch return error.SpawnFailed; } fn run(self: *Backfiller) void { defer { self.allocator.free(self.source); self.source = ""; // note: do NOT clear self.thread here — waitForCompletion() needs // the handle to join this thread before dp/ci teardown. the running // flag gates start(), so a stale thread handle is harmless. self.running.store(false, .release); } const pool = self.db(); // discover collections const collections = self.discoverCollections() catch |err| { log.err("collection discovery failed: {s}", .{@errorName(err)}); return; }; defer { for (collections) |c| self.allocator.free(c); self.allocator.free(collections); } log.info("discovered {d} collections to backfill from {s}", .{ collections.len, self.source }); // insert progress rows (skip existing) for (collections) |collection| { _ = pool.exec( "INSERT INTO backfill_progress (collection, source) VALUES ($1, $2) ON CONFLICT (collection, source) DO NOTHING", .{ collection, self.source }, ) catch |err| { log.warn("failed to insert progress for {s}: {s}", .{ collection, @errorName(err) }); }; } // backfill each collection for (collections) |collection| { if (self.shutdown.load(.acquire)) { log.info("backfill interrupted by shutdown", .{}); return; } self.backfillCollection(collection) catch |err| { log.warn("backfill failed for {s}: {s}", .{ collection, @errorName(err) }); }; } log.info("backfill complete", .{}); } fn discoverCollections(self: *Backfiller) ![][]const u8 { var seen: std.StringHashMapUnmanaged(void) = .empty; defer seen.deinit(self.allocator); // source 1: lexicon garden const garden = self.fetchLexiconGarden() catch |err| blk: { log.warn("lexicon garden fetch failed: {s}", .{@errorName(err)}); break :blk &[_][]const u8{}; }; defer { for (garden) |c| self.allocator.free(c); self.allocator.free(garden); } for (garden) |c| { if (!seen.contains(c)) { const duped = try self.allocator.dupe(u8, c); errdefer self.allocator.free(duped); try seen.put(self.allocator, duped, {}); } } // source 2: observed collections from RBC scan const observed = self.collection_index.listKnownCollections(self.allocator) catch |err| blk: { log.warn("RBC scan failed: {s}", .{@errorName(err)}); break :blk &[_][]const u8{}; }; defer { for (observed) |c| self.allocator.free(c); self.allocator.free(observed); } for (observed) |c| { if (!seen.contains(c)) { const duped = try self.allocator.dupe(u8, c); errdefer self.allocator.free(duped); try seen.put(self.allocator, duped, {}); } } // collect into result const result = try self.allocator.alloc([]const u8, seen.count()); var i: usize = 0; var key_iter = seen.keyIterator(); while (key_iter.next()) |key| { result[i] = key.*; i += 1; } return result; } /// fetch NSIDs from https://lexicon.garden/llms.txt /// parses lines matching `- [``](` fn fetchLexiconGarden(self: *Backfiller) ![][]const u8 { var client: http.Client = .{ .allocator = self.allocator, .io = self.io }; defer client.deinit(); var aw: std.Io.Writer.Allocating = .init(self.allocator); defer aw.deinit(); const result = client.fetch(.{ .location = .{ .url = "https://lexicon.garden/llms.txt" }, .response_writer = &aw.writer, .method = .GET, }) catch return error.FetchFailed; if (result.status != .ok) return error.FetchFailed; const body = aw.toArrayList().items; var nsids: std.ArrayListUnmanaged([]const u8) = .empty; defer nsids.deinit(self.allocator); // parse lines like: - [`app.bsky.feed.post`]( var lines = std.mem.splitScalar(u8, body, '\n'); while (lines.next()) |line| { // look for pattern: - [``]( const backtick_start = std.mem.indexOf(u8, line, "- [`") orelse continue; const nsid_start = backtick_start + 4; // skip "- [`" const rest = line[nsid_start..]; const backtick_end = std.mem.indexOf(u8, rest, "`](") orelse continue; const nsid = rest[0..backtick_end]; // basic NSID validation: must contain at least one dot if (!std.mem.containsAtLeast(u8, nsid, 1, ".")) continue; // skip obviously non-record types (but include them anyway — they'll return empty) const duped = try self.allocator.dupe(u8, nsid); errdefer self.allocator.free(duped); try nsids.append(self.allocator, duped); } log.info("lexicon garden: found {d} NSIDs", .{nsids.items.len}); return try nsids.toOwnedSlice(self.allocator); } fn backfillCollection(self: *Backfiller, collection: []const u8) !void { const pool = self.db(); // single query: check completion, get cursor + count for resume var cursor: ?[]const u8 = null; defer if (cursor) |c| self.allocator.free(c); var imported: i64 = 0; { var row = (pool.rowUnsafe( "SELECT completed_at IS NOT NULL, cursor, imported_count FROM backfill_progress WHERE collection = $1 AND source = $2", .{ collection, self.source }, ) catch return error.DatabaseError) orelse return; defer row.deinit() catch {}; if (row.get(bool, 0)) return; // already done const saved_cursor = row.get([]const u8, 1); imported = row.get(i64, 2); if (saved_cursor.len > 0) { cursor = try self.allocator.dupe(u8, saved_cursor); log.info("{s}: resuming from cursor (imported {d} so far)", .{ collection, imported }); } } // reuse one HTTP client across all pages for this collection var client: http.Client = .{ .allocator = self.allocator, .io = self.io }; defer client.deinit(); var page_count: usize = 0; while (!self.shutdown.load(.acquire)) { const fetch_result = self.fetchPage(&client, collection, cursor) catch |err| { log.warn("{s}: fetch page failed: {s}", .{ collection, @errorName(err) }); break; }; defer { for (fetch_result.dids) |d| self.allocator.free(d); self.allocator.free(fetch_result.dids); if (fetch_result.next_cursor) |nc| self.allocator.free(nc); } // add each DID to collection index for (fetch_result.dids) |did| { self.collection_index.addCollection(did, collection) catch {}; imported += 1; } page_count += 1; // update cursor in progress table const new_cursor = fetch_result.next_cursor orelse ""; _ = pool.exec( "UPDATE backfill_progress SET cursor = $1, imported_count = $2 WHERE collection = $3 AND source = $4", .{ new_cursor, imported, collection, self.source }, ) catch {}; if (fetch_result.next_cursor) |nc| { // advance cursor if (cursor) |old| self.allocator.free(old); cursor = self.allocator.dupe(u8, nc) catch break; // brief pause between pages self.io.sleep(Io.Duration.fromMilliseconds(100), .awake) catch {}; } else { // no more pages — mark complete _ = pool.exec( "UPDATE backfill_progress SET completed_at = now(), cursor = '', imported_count = $1 WHERE collection = $2 AND source = $3", .{ imported, collection, self.source }, ) catch {}; log.info("{s}: complete ({d} DIDs, {d} pages)", .{ collection, imported, page_count }); break; } } } fn fetchPage(self: *Backfiller, client: *http.Client, collection: []const u8, cursor: ?[]const u8) !FetchResult { // build URL var url_buf: [1024]u8 = undefined; const url = if (cursor) |c| std.fmt.bufPrint(&url_buf, "https://{s}/xrpc/com.atproto.sync.listReposByCollection?collection={s}&limit=1000&cursor={s}", .{ self.source, collection, c }) catch return error.UrlTooLong else std.fmt.bufPrint(&url_buf, "https://{s}/xrpc/com.atproto.sync.listReposByCollection?collection={s}&limit=1000", .{ self.source, collection }) catch return error.UrlTooLong; var aw: std.Io.Writer.Allocating = .init(self.allocator); defer aw.deinit(); const result = client.fetch(.{ .location = .{ .url = url }, .response_writer = &aw.writer, .method = .GET, }) catch return error.FetchFailed; if (result.status != .ok) return error.FetchFailed; const body = aw.toArrayList().items; // parse: {"repos": [{"did": "..."}, ...], "cursor": "..."} const parsed = std.json.parseFromSlice(ListReposResponse, self.allocator, body, .{ .ignore_unknown_fields = true }) catch return error.ParseFailed; defer parsed.deinit(); const repos = parsed.value.repos orelse return .{ .dids = try self.allocator.alloc([]const u8, 0), .next_cursor = null, }; var dids: std.ArrayListUnmanaged([]const u8) = .empty; defer dids.deinit(self.allocator); for (repos) |repo| { const duped = self.allocator.dupe(u8, repo.did) catch continue; dids.append(self.allocator, duped) catch { self.allocator.free(duped); continue; }; } const next_cursor = if (parsed.value.cursor) |c| self.allocator.dupe(u8, c) catch null else null; return .{ .dids = dids.toOwnedSlice(self.allocator) catch return error.OutOfMemory, .next_cursor = next_cursor, }; } const ListReposResponse = struct { repos: ?[]const RepoEntry = null, cursor: ?[]const u8 = null, }; const RepoEntry = struct { did: []const u8, }; /// return status summary for the admin endpoint pub fn getStatus(self: *Backfiller, allocator: Allocator) ![]u8 { const pool = self.db(); var aw: Io.Writer.Allocating = .init(allocator); defer aw.deinit(); const w = &aw.writer; // query aggregate stats var total: i64 = 0; var completed: i64 = 0; var total_imported: i64 = 0; { var row = (pool.rowUnsafe( "SELECT COUNT(*)::bigint, COUNT(completed_at)::bigint, COALESCE(SUM(imported_count), 0)::bigint FROM backfill_progress", .{}, ) catch null) orelse null; if (row) |*r| { total = r.get(i64, 0); completed = r.get(i64, 1); total_imported = r.get(i64, 2); r.deinit() catch {}; } } w.print("{{\"running\":{},\"total\":{d},\"completed\":{d},\"in_progress\":{d},\"total_imported\":{d},\"collections\":[", .{ self.isRunning(), total, completed, total - completed, total_imported, }) catch return error.FormatError; // per-collection detail var result = pool.query( "SELECT collection, source, cursor, imported_count, completed_at IS NOT NULL FROM backfill_progress ORDER BY collection, source", .{}, ) catch return error.DatabaseError; defer result.deinit(); var first = true; while (result.nextUnsafe() catch null) |dbrow| { if (!first) w.writeByte(',') catch {}; first = false; const collection = dbrow.get([]const u8, 0); const source = dbrow.get([]const u8, 1); const cursor_val = dbrow.get([]const u8, 2); const count = dbrow.get(i64, 3); const is_completed = dbrow.get(bool, 4); w.print("{{\"collection\":\"{s}\",\"source\":\"{s}\",\"imported\":{d},\"completed\":{}", .{ collection, source, count, is_completed, }) catch {}; if (cursor_val.len > 0 and !is_completed) { w.print(",\"cursor\":\"{s}\"", .{cursor_val}) catch {}; } w.writeByte('}') catch {}; } w.writeAll("]}") catch {}; return try aw.toOwnedSlice(); } };