Something went wrong. Try again.
Something went wrong. Try again.
2.8 kB · 59 lines
Python
at fixes-stack
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960import json,threading,queue,time,gzip,re,os,urllib.requestUA="tangled-stats-research/0.1 (permadeath.com)"SD="/tmp/claude-1000/-home-jmm-Code-blogposts/7ba1f72e-807b-4749-bcd3-a065da793e6d/scratchpad"def get(url,tries=3,timeout=90): for i in range(tries): try: r=urllib.request.Request(url,headers={"User-Agent":UA}) with urllib.request.urlopen(r,timeout=timeout) as f: return f.read() except Exception: if i==tries-1: raise time.sleep(1.5*(i+1))mp=json.load(open(f"{SD}/merged_pulls.json")); PM=json.load(open(f"{SD}/pds_map.json"))seen=set()if os.path.exists(f"{SD}/commits.jsonl"): for l in open(f"{SD}/commits.jsonl"): try: seen.add(json.loads(l)["uri"]) except: passtodo=[p for p in mp if (p["cid"] or p.get("inlinePatch")) and p["uri"] not in seen]print("todo",len(todo),"of",len(mp),flush=True)FROM=re.compile(rb"(?m)^From ([0-9a-f]{40}) ")DATE=re.compile(rb"(?m)^Date: (.+?)\r?$")AUTH=re.compile(rb"(?m)^From: (.+?)\r?$")out=open(f"{SD}/commits.jsonl","a"); lock=threading.Lock(); c=[0,0]q=queue.Queue(); [q.put(p) for p in todo]def w(): while True: try: p=q.get_nowait() except queue.Empty: return try: did=p["uri"].split("/")[2]; host=PM.get(did) inline=p.get("inlinePatch") if inline: txt=inline.encode("utf8","replace"); raw=txt elif host and p["cid"]: raw=get(f"{host}/xrpc/com.atproto.sync.getBlob?did={did}&cid={p['cid']}") try: txt=gzip.decompress(raw) except Exception: txt=raw else: txt=None if txt is not None: shas=[s.decode() for s in FROM.findall(txt)] n=len(shas) rec={"uri":p["uri"],"repoDid":p["repoDid"],"author":did,"commits":n, "shas":shas, "createdAt":p["createdAt"],"stateUpdatedAt":p["stateUpdatedAt"], "rounds":p["rounds"],"patchBytes":len(raw), "commitDates":[d.decode("utf8","replace") for d in DATE.findall(txt)][:n], "commitAuthors":[a.decode("utf8","replace") for a in AUTH.findall(txt)][:n]} with lock: out.write(json.dumps(rec)+"\n"); c[0]+=1; c[1]+=n except Exception as e: with lock: print("ERR",p["uri"][:70],str(e)[:60],flush=True) finally: with lock: if c[0] and c[0]%250==0: out.flush(); print(f" {c[0]}/{len(todo)} commits={c[1]}",flush=True) q.task_done()ts=[threading.Thread(target=w,daemon=True) for _ in range(10)][t.start() for t in ts];[t.join() for t in ts]out.flush();out.close();print("DONE",c,flush=True)