Something went wrong. Try again.
Something went wrong. Try again.
3.6 kB · 101 lines
Python
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102"""Real commit counts from the knots, not from PR patches.
sh.tangled.repo.log returns a `total` alongside the page of commits, so onerequest per repository gives its true commit count on the default branch -including everything pushed straight to a branch, which the patch-derivedfigures miss entirely.
Pass 1 (this file): totals per repo, one cheap request each, to size the problemand find where the commits actually are."""import json, os, queue, threading, time, urllib.parse, urllib.request
SD = "/tmp/claude-1000/-home-jmm-Code-blogposts/7ba1f72e-807b-4749-bcd3-a065da793e6d/scratchpad"UA = "tangled-stats-research/0.6 (permadeath.com; blog post stats)"
def get(url, tries=5, timeout=20): for i in range(tries): try: req = urllib.request.Request(url, headers={"User-Agent": UA}) with urllib.request.urlopen(req, timeout=timeout) as f: return json.load(f) except urllib.error.HTTPError as e: if e.code == 429: # the knot asks us to slow down time.sleep(2 + 3 * i) continue if e.code in (400, 404): return None if i == tries - 1: return None time.sleep(1 + i) except Exception: if i == tries - 1: return None time.sleep(1 + i) return None
def main(): # default branch per repo, straight from the knot's own listing branch = {} for l in open(f"{SD}/knot1_repos.jsonl"): for r in json.loads(l)["repos"]: db = (r.get("defaultBranch") or {}).get("ref") or "" branch[r["repo"]] = db.replace("refs/heads/", "") or "main"
repos = json.load(open(f"{SD}/closure_repos.json")) done = {} if os.path.exists(f"{SD}/commit_totals.jsonl"): for l in open(f"{SD}/commit_totals.jsonl"): try: r = json.loads(l) done[r["repo"]] = r except Exception: pass todo = [r for r in repos if r not in done] print(f"todo {len(todo)} / {len(repos)}", flush=True)
q = queue.Queue() for r in todo: q.put(r) out = open(f"{SD}/commit_totals.jsonl", "a") lock = threading.Lock() c = [0, 0, 0]
def worker(): while True: try: rd = q.get_nowait() except queue.Empty: return try: ref = branch.get(rd, "main") d = get(f"https://knot1.tangled.sh/xrpc/sh.tangled.repo.log" f"?repo={urllib.parse.quote(rd)}&ref={urllib.parse.quote(ref)}&limit=1") total = (d or {}).get("total") time.sleep(0.05) with lock: out.write(json.dumps({"repo": rd, "ref": ref, "total": total}) + "\n") c[0] += 1 if isinstance(total, int): c[1] += 1 c[2] += total if c[0] % 1000 == 0: out.flush() print(f" {c[0]}/{len(todo)} with counts {c[1]} commits {c[2]}", flush=True) except Exception: pass finally: q.task_done()
ts = [threading.Thread(target=worker, daemon=True) for _ in range(4)] [t.start() for t in ts] [t.join() for t in ts] out.flush() out.close() print(f"DONE repos {c[0]} answered {c[1]} total commits {c[2]}", flush=True)
if __name__ == "__main__": main()