Something went wrong. Try again.
search for standard sites pub-search.waow.tech
search zig blog atproto
Something went wrong. Try again.
5.4 kB · 146 lines
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147#!/usr/bin/env -S uv run --script --quiet# /// script# requires-python = ">=3.12"# dependencies = ["httpx"]# ///"""Ad-hoc backfill: pull repos straight from their PDS into the index.
Wraps POST /admin/backfill (backend resolves the PDS, lists records, runs thenormal extract+index path — banned DIDs are dropped by the indexer, bridgy-hosted repos are rejected at the door). Accepts DIDs or handles, one or many.Writes land in turso; serving picks them up at the next replica catchup + swap(offline-replica-catchup) until the snapshot builder automates that.
The endpoint is async: it responds 202 and runs the backfill in the background(completion signal is the logfire line `backfill: <did> done`). The backendruns at most ONE backfill at a time and returns 409 while busy — this scriptwaits and retries, so a list of targets is naturally paced by server-sidecompletion.
Usage: scripts/backfill did:plc:xyz... scripts/backfill someone.bsky.social pfrazee.com scripts/backfill --collection com.whtwnd.blog.entry did:plc:xyz scripts/backfill --from-file dids.txt scripts/backfill --sync did:plc:xyz # block and report counts
BACKFILL_TOKEN is read from the environment if the endpoint is guarded."""
import argparseimport osimport sysimport time
import httpx
API = os.environ.get("BACKFILL_API", "https://leaflet-search-backend.fly.dev")BUSY_RETRY_SECONDS = 15.0BUSY_MAX_WAIT_SECONDS = 1800.0
def log(msg: str) -> None: print(msg, flush=True)
def resolve_handle(client: httpx.Client, handle: str) -> str | None: r = client.get( "https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle", params={"handle": handle.lstrip("@")}, timeout=10, ) if r.status_code != 200: return None return r.json().get("did")
def submit(client: httpx.Client, did: str, params: dict, sync: bool) -> tuple[str, str]: """returns (status, detail) where status is ok|accepted|pend|fail.""" timeout = 900 if sync else 30 waited = 0.0 while True: try: r = client.post(f"{API}/admin/backfill", params=params, timeout=timeout) except (httpx.TimeoutException, httpx.TransportError): if sync: # fly's proxy drops long-held connections even though the # backfill keeps running server-side — pending, not failure return "pend", f"connection dropped; confirm via logfire 'backfill: {did} done'" return "fail", "request failed before the backend accepted it"
if r.status_code == 409: if waited >= BUSY_MAX_WAIT_SECONDS: return "fail", "backend busy with another backfill; gave up waiting" time.sleep(BUSY_RETRY_SECONDS) waited += BUSY_RETRY_SECONDS continue
try: body = r.json() except ValueError: return "fail", f"non-JSON response (HTTP {r.status_code})"
if r.status_code == 202: return "accepted", f"running server-side; confirm via logfire 'backfill: {did} done'" if r.status_code != 200 or "error" in body: return "fail", str(body) return "ok", ( f"docs={body.get('documents')} pubs={body.get('publications')} " f"recs={body.get('recommends')} skipped={body.get('skipped')}" )
def main() -> None: parser = argparse.ArgumentParser(description="ad-hoc repo backfill") parser.add_argument("targets", nargs="*", help="DIDs or handles") parser.add_argument("--collection", help="restrict to one collection NSID") parser.add_argument("--from-file", help="file with one DID/handle per line") parser.add_argument("--sync", action="store_true", help="block and report counts") args = parser.parse_args()
targets = list(args.targets) if args.from_file: with open(args.from_file) as f: targets += [ln.strip() for ln in f if ln.strip() and not ln.startswith("#")] if not targets: parser.error("no targets given")
token = os.environ.get("BACKFILL_TOKEN", "")
ok = failed = 0 with httpx.Client() as client: for target in targets: did = target if not target.startswith("did:"): resolved = resolve_handle(client, target) if not resolved: log(f"FAIL {target}: handle did not resolve") failed += 1 continue did = resolved log(f"{target} -> {did}")
params: dict = {"did": did} if args.collection: params["collection"] = args.collection if token: params["token"] = token if args.sync: params["sync"] = "1"
status, detail = submit(client, did, params, args.sync) if status == "fail": log(f"FAIL {did}: {detail}") failed += 1 else: log(f"{status:8s} {did}: {detail}") ok += 1
log(f"\n{ok} submitted, {failed} failed out of {len(targets)}") if ok: log("note: serving replica is frozen (SYNC_DISABLE) — run the catchup+swap to surface these in search") sys.exit(1 if failed else 0)
if __name__ == "__main__": main()