Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
1.6 kB · 38 lines
Python
at fixes-stack
123456789101112131415161718192021222324252627282930313233343536373839import json,threading,queue,time,urllib.request,osUA="tangled-stats-research/0.1 (permadeath.com)"SD="/tmp/claude-1000/-home-jmm-Code-blogposts/7ba1f72e-807b-4749-bcd3-a065da793e6d/scratchpad"def get(url,tries=3): for i in range(tries): try: r=urllib.request.Request(url,headers={"User-Agent":UA}) with urllib.request.urlopen(r,timeout=45) as f: return json.load(f) except Exception: if i==tries-1: raise time.sleep(1+i)dids=set()for l in open(f"{SD}/pulls.jsonl"): d=json.loads(l) for p in d["pulls"]: dids.add(p["uri"].split("/")[2])dids=sorted(dids); print(len(dids),flush=True)out={}if os.path.exists(f"{SD}/handles.json"): out=json.load(open(f"{SD}/handles.json"))q=queue.Queue(); [q.put(d) for d in dids if d not in out]; lock=threading.Lock(); c=[0]def w(): while True: try: did=q.get_nowait() except queue.Empty: return h=None try: doc=get("https://"+did[8:]+"/.well-known/did.json") if did.startswith("did:web:") else get(f"https://plc.directory/{did}") aka=[a for a in doc.get("alsoKnownAs",[]) if a.startswith("at://")] if aka: h=aka[0][5:] except Exception as e: with lock: print("ERR",did,e,flush=True) with lock: out[did]=h; c[0]+=1 if c[0]%100==0: print(c[0],flush=True) q.task_done()ts=[threading.Thread(target=w,daemon=True) for _ in range(6)][t.start() for t in ts];[t.join() for t in ts]json.dump(out,open(f"{SD}/handles.json","w"))print("DONE",len(out),flush=True)