import json, sys, threading, queue, time, urllib.parse
import urllib.request

A = "https://api.tangled.org/xrpc"
UA = "tangled-stats-research/0.1 (permadeath.com; blog post stats)"
SD = "/tmp/claude-1000/-home-jmm-Code-blogposts/7ba1f72e-807b-4749-bcd3-a065da793e6d/scratchpad"

dids = [l.strip() for l in open(f"{SD}/repo_dids.txt") if l.strip()]
done = set()
try:
    for l in open(f"{SD}/pulls.jsonl"):
        try: done.add(json.loads(l)["repoDid"])
        except Exception: pass
except FileNotFoundError:
    pass
todo = [d for d in dids if d not in done]
print(f"todo {len(todo)} / {len(dids)}", flush=True)

q = queue.Queue()
for d in todo: q.put(d)
out = open(f"{SD}/pulls.jsonl", "a")
lock = threading.Lock()
counter = [0, 0]

def get(url, tries=4):
    for i in range(tries):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA})
            with urllib.request.urlopen(req, timeout=60) as r:
                return json.load(r)
        except Exception as e:
            if i == tries - 1: raise
            time.sleep(1.5 * (i + 1))

def worker():
    while True:
        try: did = q.get_nowait()
        except queue.Empty: return
        try:
            items, cursor = [], None
            while True:
                u = f"{A}/sh.tangled.repo.listPulls?subject={urllib.parse.quote(did)}&limit=1000"
                if cursor: u += "&cursor=" + urllib.parse.quote(cursor)
                d = get(u)
                for it in d.get("items", []):
                    v = it.get("value") or {}
                    rounds = v.get("rounds") or []
                    tgt = v.get("target") or {}
                    items.append({
                        "uri": it["uri"],
                        "state": it.get("state"),
                        "createdAt": v.get("createdAt"),
                        "stateUpdatedAt": it.get("stateUpdatedAt"),
                        "title": (v.get("title") or "")[:200],
                        "comments": it.get("commentCount", 0),
                        "rounds": len(rounds) if isinstance(rounds, list) else 0,
                        "targetBranch": tgt.get("branch"),
                        "targetRepo": tgt.get("repoDid") or tgt.get("repo"),
                        "bodyLen": len(v.get("body") or ""),
                    })
                cursor = d.get("cursor")
                if not cursor or not d.get("items"): break
            rec = {"repoDid": did, "n": len(items), "pulls": items}
            with lock:
                out.write(json.dumps(rec) + "\n")
                counter[0] += 1; counter[1] += len(items)
                if counter[0] % 250 == 0:
                    out.flush()
                    print(f"{counter[0]} repos, {counter[1]} pulls", flush=True)
        except Exception as e:
            with lock: print(f"ERR {did}: {e}", flush=True)
        finally:
            q.task_done()

ts = [threading.Thread(target=worker, daemon=True) for _ in range(12)]
[t.start() for t in ts]
[t.join() for t in ts]
out.flush(); out.close()
print("DONE", counter, flush=True)