Something went wrong. Try again.
Something went wrong. Try again.
3.2 kB · 96 lines
Python
at fixes-stack
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697"""Stage A (Tangled-API-free): enumerate every PR on the network via Constellation.
For each repo DID known to the knot, ask Constellation which sh.tangled.repo.pullrecords point at it. Gives {author did, rkey} for every PR globally, with nodependency on api.tangled.org."""import json, os, queue, threading, time, urllib.parse, urllib.request
SD = "/tmp/claude-1000/-home-jmm-Code-blogposts/7ba1f72e-807b-4749-bcd3-a065da793e6d/scratchpad"C = "https://constellation.microcosm.blue"UA = "tangled-stats-research/0.2 (permadeath.com; blog post stats)"
def get(url, tries=4): for i in range(tries): try: req = urllib.request.Request(url, headers={"User-Agent": UA}) with urllib.request.urlopen(req, timeout=60) as f: return json.load(f) except Exception: if i == tries - 1: raise time.sleep(2 * (i + 1))
def links(target, collection, path): out, cursor = [], None while True: u = (f"{C}/links?target={urllib.parse.quote(target, safe='')}" f"&collection={collection}&path={path}&limit=1000") if cursor: u += "&cursor=" + urllib.parse.quote(cursor) d = get(u) out += d.get("linking_records", []) cursor = d.get("cursor") if not cursor or not d.get("linking_records"): break return out
def main(): dids = [l.strip() for l in open(f"{SD}/repo_dids.txt") if l.strip()] done = set() if os.path.exists(f"{SD}/alt_links2.jsonl"): for l in open(f"{SD}/alt_links2.jsonl"): try: done.add(json.loads(l)["repoDid"]) except Exception: pass todo = [d for d in dids if d not in done] print(f"todo {len(todo)} / {len(dids)}", flush=True)
q = queue.Queue() for d in todo: q.put(d) out = open(f"{SD}/alt_links2.jsonl", "a") lock = threading.Lock() c = [0, 0]
def worker(): while True: try: did = q.get_nowait() except queue.Empty: return try: seen, recs = set(), [] for path in (".target.repoDid", ".target.repo"): for r in links(did, "sh.tangled.repo.pull", path): k = (r["did"], r["rkey"]) if k not in seen: seen.add(k) recs.append(r) with lock: out.write(json.dumps({"repoDid": did, "n": len(recs), "pulls": recs}) + "\n") c[0] += 1 c[1] += len(recs) if c[0] % 500 == 0: out.flush() print(f" {c[0]}/{len(todo)} repos, {c[1]} pulls", flush=True) except Exception as e: with lock: print(f"ERR {did}: {str(e)[:70]}", flush=True) finally: q.task_done()
ts = [threading.Thread(target=worker, daemon=True) for _ in range(8)] [t.start() for t in ts] [t.join() for t in ts] out.flush() out.close() print("DONE", c, flush=True)
if __name__ == "__main__": main()