Something went wrong. Try again.
🌿 Collaborative wiki on ATProto forked from juprodh.me/lichen.wiki
Something went wrong. Try again.
2.3 kB · 80 lines
TypeScript
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081#!/usr/bin/env bun
// PDS backfill: crawl wiki.lichen.* records from one or more repos and feed them// through the firehose handler. The firehose is live-only; this is how we pull in// records we missed (joined late, another client wrote them, targeted recovery).//// bun run scripts/backfill.ts <did> [<did>...] [--dry-run] [--collection <nsid>]//// Reads/writes the DB at DB_PATH (default lichen.db) — point it at the prod DB on// the VPS. listRecords is public, so no auth is needed.
import { runBackfill } from "../src/lib/backfill/crawler.ts";import { formatReport, hasFatal } from "../src/lib/backfill/report.ts";import { COLLECTIONS } from "../src/lib/collections.ts";
interface ParsedArgs { dids: string[]; dryRun: boolean; collection?: string;}
function parseArgs(argv: string[]): ParsedArgs { const dids: string[] = []; let dryRun = false; let collection: string | undefined;
for (let i = 0; i < argv.length; i++) { const arg = argv[i]; if (arg === "--dry-run") { dryRun = true; } else if (arg === "--collection") { collection = argv[++i]; if (!collection) throw new Error("--collection requires an NSID argument"); } else if (arg?.startsWith("did:")) { dids.push(arg); } else { throw new Error(`unexpected argument: ${arg}`); } }
return { dids, dryRun, ...(collection ? { collection } : {}) };}
const USAGE = "Usage: bun run scripts/backfill.ts <did> [<did>...] [--dry-run] [--collection <nsid>]";
async function main(): Promise<number> { let parsed: ParsedArgs; try { parsed = parseArgs(process.argv.slice(2)); } catch (err) { console.error(`${String(err)}\n${USAGE}`); return 2; }
if (parsed.dids.length === 0) { console.error(`No DIDs given.\n${USAGE}`); return 2; }
if ( parsed.collection && !Object.values(COLLECTIONS).includes(parsed.collection as never) ) { console.warn( `Warning: --collection ${parsed.collection} is not a wiki.lichen collection; it will match nothing.`, ); }
const report = await runBackfill(parsed.dids, { dryRun: parsed.dryRun, ...(parsed.collection ? { collections: [parsed.collection] } : {}), });
console.log(`\n${formatReport(report, { dryRun: parsed.dryRun })}`); return hasFatal(report) ? 1 : 0;}
process.exit(await main());