Something went wrong. Try again.
Something went wrong. Try again.
2.8 kB · 68 lines
Python
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869import json, threading, queue, time, urllib.parse, urllib.request, osA="https://api.tangled.org/xrpc"; UA="tangled-stats-research/0.1 (permadeath.com)"SD="/tmp/claude-1000/-home-jmm-Code-blogposts/7ba1f72e-807b-4749-bcd3-a065da793e6d/scratchpad"def get(url,tries=4): for i in range(tries): try: r=urllib.request.Request(url,headers={"User-Agent":UA}) with urllib.request.urlopen(r,timeout=60) as f: return json.load(f) except Exception: if i==tries-1: raise time.sleep(1.5*(i+1))def par(items, fn, n=12): q=queue.Queue(); [q.put(i) for i in items]; lock=threading.Lock(); res=[]; c=[0] def w(): while True: try: it=q.get_nowait() except queue.Empty: return try: v=fn(it) with lock: if v is not None: res.append(v) except Exception as e: with lock: print("ERR",it,e,flush=True) finally: with lock: c[0]+=1 if c[0]%500==0: print(f" {c[0]}/{len(items)}",flush=True) q.task_done() ts=[threading.Thread(target=w,daemon=True) for _ in range(n)] [t.start() for t in ts]; [t.join() for t in ts]; return res
# 1. repos that actually have pulls -> owner recordsactive=[]; authors=set()for l in open(f"{SD}/pulls.jsonl"): d=json.loads(l) if d["n"]: active.append(d["repoDid"]) for p in d["pulls"]: authors.add(p["uri"].split("/")[2])print("active repos",len(active),"pull authors",len(authors),flush=True)
recs=par(active, lambda did: get(f"{A}/sh.tangled.repo.getRepoByRepoDid?repoDid={did}"))json.dump(recs, open(f"{SD}/repo_records.json","w"))owners=set()for r in recs: u=(r.get("uri") or "") if u.startswith("at://"): owners.add(u.split("/")[2])print("owners",len(owners),flush=True)
# 2. every known actor -> their repo records (catches non-knot1 knots)actors=sorted(authors|owners)print("actors to expand",len(actors),flush=True)def allrepos(did): out=[];cur=None while True: u=f"{A}/sh.tangled.repo.listRepos?subject={did}&limit=1000"+(f"&cursor={urllib.parse.quote(cur)}" if cur else "") d=get(u); out+=d.get("items",[]); cur=d.get("cursor") if not cur or not d.get("items"): break return {"did":did,"items":out}ar=par(actors, allrepos)json.dump(ar, open(f"{SD}/actor_repos.json","w"))known=set(l.strip() for l in open(f"{SD}/repo_dids.txt"))new=set()for a in ar: for i in a["items"]: rd=(i.get("value") or {}).get("repoDid") if rd and rd not in known: new.add(rd)print("NEW repo dids",len(new),flush=True)open(f"{SD}/repo_dids_extra.txt","w").write("\n".join(sorted(new)))