Something went wrong. Try again.
Something went wrong. Try again.
3.1 kB · 79 lines
Python
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980import json, sys, threading, queue, time, urllib.parseimport urllib.request
A = "https://api.tangled.org/xrpc"UA = "tangled-stats-research/0.1 (permadeath.com; blog post stats)"SD = "/tmp/claude-1000/-home-jmm-Code-blogposts/7ba1f72e-807b-4749-bcd3-a065da793e6d/scratchpad"
dids = [l.strip() for l in open(f"{SD}/repo_dids.txt") if l.strip()]done = set()try: for l in open(f"{SD}/pulls.jsonl"): try: done.add(json.loads(l)["repoDid"]) except Exception: passexcept FileNotFoundError: passtodo = [d for d in dids if d not in done]print(f"todo {len(todo)} / {len(dids)}", flush=True)
q = queue.Queue()for d in todo: q.put(d)out = open(f"{SD}/pulls.jsonl", "a")lock = threading.Lock()counter = [0, 0]
def get(url, tries=4): for i in range(tries): try: req = urllib.request.Request(url, headers={"User-Agent": UA}) with urllib.request.urlopen(req, timeout=60) as r: return json.load(r) except Exception as e: if i == tries - 1: raise time.sleep(1.5 * (i + 1))
def worker(): while True: try: did = q.get_nowait() except queue.Empty: return try: items, cursor = [], None while True: u = f"{A}/sh.tangled.repo.listPulls?subject={urllib.parse.quote(did)}&limit=1000" if cursor: u += "&cursor=" + urllib.parse.quote(cursor) d = get(u) for it in d.get("items", []): v = it.get("value") or {} rounds = v.get("rounds") or [] tgt = v.get("target") or {} items.append({ "uri": it["uri"], "state": it.get("state"), "createdAt": v.get("createdAt"), "stateUpdatedAt": it.get("stateUpdatedAt"), "title": (v.get("title") or "")[:200], "comments": it.get("commentCount", 0), "rounds": len(rounds) if isinstance(rounds, list) else 0, "targetBranch": tgt.get("branch"), "targetRepo": tgt.get("repoDid") or tgt.get("repo"), "bodyLen": len(v.get("body") or ""), }) cursor = d.get("cursor") if not cursor or not d.get("items"): break rec = {"repoDid": did, "n": len(items), "pulls": items} with lock: out.write(json.dumps(rec) + "\n") counter[0] += 1; counter[1] += len(items) if counter[0] % 250 == 0: out.flush() print(f"{counter[0]} repos, {counter[1]} pulls", flush=True) except Exception as e: with lock: print(f"ERR {did}: {e}", flush=True) finally: q.task_done()
ts = [threading.Thread(target=worker, daemon=True) for _ in range(12)][t.start() for t in ts][t.join() for t in ts]out.flush(); out.close()print("DONE", counter, flush=True)