Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
5.7 kB · 150 lines
Python
at fixes-stack
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151"""Pass 4: per-month commit counts, so any window can be sliced afterwards.
Pass 3 stored a date *range* per identity, which is enough to exclude importedhistory but not to answer "how many in August" - 42k commits sat in groups thatmerely overlap the month. This records identity -> {month: count} instead, soall-time and any single month come from the same data with no re-crawl.
Runs only over the repos pass 3 classified as native (history starting when therepo appeared on Tangled), and stops walking once a page predates Tangled itself."""import json, os, queue, sys, threading, time, urllib.parse, urllib.request, collections
SD = "/tmp/claude-1000/-home-jmm-Code-blogposts/7ba1f72e-807b-4749-bcd3-a065da793e6d/scratchpad"UA = "tangled-stats-research/0.8 (permadeath.com; blog post stats)"BUDGET = int(sys.argv[1]) if len(sys.argv) > 1 else 200000CUTOFF = "2025-01-01"MAX_PAGES = 400
def get(url, tries=5, timeout=25): for i in range(tries): try: req = urllib.request.Request(url, headers={"User-Agent": UA}) with urllib.request.urlopen(req, timeout=timeout) as f: return json.load(f) except urllib.error.HTTPError as e: if e.code == 429: time.sleep(2 + 3 * i) continue if e.code in (400, 404): return None if i == tries - 1: return None time.sleep(1 + i) except Exception: if i == tries - 1: return None time.sleep(1 + i) return None
def sha_of(c): h = c.get("hash") return "".join(f"{x:02x}" for x in h) if isinstance(h, list) else str(h)
def main(): meta = json.load(open(f"{SD}/repo_meta.json")) totals = {} for l in open(f"{SD}/commit_totals.jsonl"): r = json.loads(l) if isinstance(r["total"], int) and r["total"] > 0: totals[r["repo"]] = (r["total"], r["ref"])
# native = history starts when the repo appeared on Tangled (pass 3's rule) native = [] for l in open(f"{SD}/commit_identities.jsonl"): r = json.loads(l) m = meta.get(r["repo"]) or {} fs = [v["first"] for v in r["ids"].values() if v["first"] != "9999"] if not fs: continue cre = (m.get("createdAt") or "")[:10] if (min(fs) >= cre) if cre else (min(fs) >= CUTOFF): native.append(r["repo"])
done = set() if os.path.exists(f"{SD}/commit_monthly.jsonl"): for l in open(f"{SD}/commit_monthly.jsonl"): try: done.add(json.loads(l)["repo"]) except Exception: pass cands = [(rd, *totals[rd]) for rd in native if rd in totals and rd not in done] print(f"native repos to walk: {len(cands)}", flush=True)
q = queue.Queue() for c in cands: q.put(c) out = open(f"{SD}/commit_monthly.jsonl", "a") lock = threading.Lock() spent = [0] n = [0, 0]
def worker(): while True: with lock: if spent[0] >= BUDGET: return try: rd, total, ref = q.get_nowait() except queue.Empty: return try: knot = (meta.get(rd) or {}).get("knot") or "knot1.tangled.sh" seen = {} cursor, pages = ref, 0 while pages < MAX_PAGES: d = get(f"https://{knot}/xrpc/sh.tangled.repo.log" f"?repo={urllib.parse.quote(rd)}&ref={urllib.parse.quote(cursor)}&limit=100") with lock: spent[0] += 1 pages += 1 if not d: break commits = d.get("commits") or [] if not commits: break for c in commits: a = c.get("author") or {} seen[sha_of(c)] = (a.get("Email") or "?", (a.get("When") or "")[:7]) if len(commits) < 100 or (d.get("total") or 0) <= 100: break newest = max(((c.get("author") or {}).get("When") or "")[:10] for c in commits) if newest and newest < CUTOFF: break nxt = sha_of(commits[-1]) if nxt == cursor: break cursor = nxt with lock: if spent[0] >= BUDGET: break time.sleep(0.05) ids = collections.defaultdict(collections.Counter) for email, month in seen.values(): ids[email][month or "?"] += 1 with lock: out.write(json.dumps({"repo": rd, "total": total, "ids": {e: dict(m) for e, m in ids.items()}}) + "\n") n[0] += 1 n[1] += len(seen) if n[0] % 400 == 0: out.flush() print(f" {n[0]}/{len(cands)} repos reqs {spent[0]} commits {n[1]:,}", flush=True) except Exception: pass finally: q.task_done()
ts = [threading.Thread(target=worker, daemon=True) for _ in range(4)] [t.start() for t in ts] [t.join() for t in ts] out.flush() out.close() print(f"DONE repos {n[0]} requests {spent[0]} commits {n[1]:,}", flush=True)
if __name__ == "__main__": main()