Something went wrong. Try again.
Something went wrong. Try again.
3.1 kB · 84 lines
Python
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485"""Discovery pass: every Tangled account that owns a repo.
The repo-centric enumeration was the broken part - it could only see pull recordsthat carry a target.* link, and 826 records network-wide carry no target at all.The fix is to stop enumerating pulls and start enumerating *people*: askconstellation who owns each of the knot's repo DIDs, then read each person'scomplete sh.tangled.repo.pull set straight from their PDS, where every record ispresent regardless of its generation or shape."""import json, os, queue, threading, time, urllib.parse, urllib.request
SD = "/tmp/claude-1000/-home-jmm-Code-blogposts/7ba1f72e-807b-4749-bcd3-a065da793e6d/scratchpad"C = "https://constellation.microcosm.blue"UA = "tangled-stats-research/0.3 (permadeath.com; blog post stats)"
def get(url, tries=4): for i in range(tries): try: req = urllib.request.Request(url, headers={"User-Agent": UA}) with urllib.request.urlopen(req, timeout=60) as f: return json.load(f) except Exception: if i == tries - 1: raise time.sleep(2 * (i + 1))
def main(): dids = [l.strip() for l in open(f"{SD}/repo_dids.txt") if l.strip()] done = {} if os.path.exists(f"{SD}/alt_all_owners.jsonl"): for l in open(f"{SD}/alt_all_owners.jsonl"): try: r = json.loads(l) done[r["repoDid"]] = r["owner"] except Exception: pass todo = [d for d in dids if d not in done] print(f"todo {len(todo)} / {len(dids)}", flush=True)
q = queue.Queue() for d in todo: q.put(d) out = open(f"{SD}/alt_all_owners.jsonl", "a") lock = threading.Lock() c = [0, 0]
def worker(): while True: try: did = q.get_nowait() except queue.Empty: return try: d = get(f"{C}/links?target={urllib.parse.quote(did, safe='')}" f"&collection=sh.tangled.repo&path=.repoDid&limit=10") recs = (d or {}).get("linking_records", []) owner = recs[0]["did"] if recs else None with lock: out.write(json.dumps({"repoDid": did, "owner": owner}) + "\n") c[0] += 1 c[1] += 1 if owner else 0 if c[0] % 1000 == 0: out.flush() print(f" {c[0]}/{len(todo)} owners found {c[1]}", flush=True) except Exception as e: with lock: print(f"ERR {did}: {str(e)[:60]}", flush=True) finally: q.task_done()
ts = [threading.Thread(target=worker, daemon=True) for _ in range(8)] [t.start() for t in ts] [t.join() for t in ts] out.flush() out.close() owners = {json.loads(l)["owner"] for l in open(f"{SD}/alt_all_owners.jsonl")} owners.discard(None) print(f"DONE distinct repo owners: {len(owners)}", flush=True)
if __name__ == "__main__": main()