Something went wrong. Try again.
mirror your GitHub repos to tangled.org automatically synchub.to
mirror sync tangled github git
Something went wrong. Try again.
17 kB · 462 lines
TypeScript
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463import { Agent } from '@atproto/api'import type { OAuthSession } from '@atproto/oauth-client-node'import { now as tidNow } from '@atcute/tid'import { and, eq, sql } from 'drizzle-orm'import { repoMapping } from '../db/schema'import { useDb } from './db'import { installationOctokit } from './github-app'
const REPO_LEXICON = 'sh.tangled.repo'const REPO_CREATE_NSID = 'sh.tangled.repo.create'const LIST_RECORDS_PAGE_SIZE = 100
/** * GitHub repo fields we mirror into the `sh.tangled.repo` record. Kept narrow * so the merge helper is easy to reason about and to test without pulling in * the full Octokit type. */export interface GithubRepoMetadata { full_name: string description: string | null homepage: string | null topics?: string[]}
/** * Strip our `[READ-ONLY] Mirror of ...` prefix from a description, if present. * Idempotent: returns the original string when there's no prefix to remove. * Guards against accumulating prefixes if a GitHub description ever round-trips * back through our marker (e.g. a user copy-pasted the tangled description * into GitHub). */export function stripReadOnlyMarker(value: string | null | undefined): string { if (!value) return '' let s = value // Strip repeatedly so any accidental doubling is collapsed. for (;;) { const next = s.replace(/^\[READ-ONLY\]\s*Mirror of https:\/\/github\.com\/[^\s.]+\/[^\s.]+\.\s*/, '') if (next === s) return s s = next }}
/** * Build the `description` we want on the tangled-side record from GitHub's * current state. Always rebuilt from scratch so we never compound the marker. */export function buildReadOnlyDescription(githubFullName: string, githubDescription: string | null | undefined): string { const stripped = stripReadOnlyMarker(githubDescription).trim() const prefix = `[READ-ONLY] Mirror of https://github.com/${githubFullName}.` return stripped ? `${prefix} ${stripped}` : prefix}
/** * Merge GitHub metadata into an existing PDS record value, preserving fields * we don't manage (`$type`, `name`, `knot`, `repoDid`, `createdAt`, plus any * future additions). Pass `existing = undefined` for the initial enrolment * write. */export function mergeRepoRecord( existing: Record<string, unknown> | undefined, base: { name: string, knot: string, repoDid: string, createdAt: string }, gh: GithubRepoMetadata,): Record<string, unknown> { const description = buildReadOnlyDescription(gh.full_name, gh.description) const website = gh.homepage && gh.homepage.length > 0 ? gh.homepage : undefined const topics = Array.isArray(gh.topics) ? gh.topics : undefined
// Start from existing so unknown fields survive a round-trip. Then overlay // the immutable base (in case the existing record is malformed) and the // managed metadata. const merged: Record<string, unknown> = { ...existing } merged.$type = REPO_LEXICON merged.name = base.name merged.knot = base.knot merged.repoDid = base.repoDid merged.createdAt = (typeof existing?.createdAt === 'string' && existing.createdAt) || base.createdAt merged.description = description if (topics !== undefined) merged.topics = topics if (website !== undefined) merged.website = website else delete merged.website return merged}
/** * Default knot for users with no `sh.tangled.knot` records. PLAN.md "Open * questions" #1: confirm with the tangled team that this is the right * appview-hosted default. */const DEFAULT_KNOT = 'knot1.tangled.sh'
export interface EnrolResult { status: 'enrolled' | 'already' | 'skipped' reason?: 'private' | 'fork' | 'no-identity' | 'name-conflict' | 'invalid-name'}
/** * User-facing message stored in `repo_mapping.lastError` when enrolment is * blocked by a name that already exists on the user's tangled account. Kept as * a constant so `repo-health` and tests can match it exactly. Issue #4. */export const NAME_CONFLICT_ERROR = 'a repo with this name already exists on your tangled account; rename or remove it, then resync'
/** * True when the knot's `repo.create` rejection is a name collision: the user * already has a `sh.tangled.repo` of that name, so the knot refuses to mint a * second. The knot surfaces this as an `AccessControl` 400 ("DID does not have * sufficient access permissions"). */function isNameConflictResponse(status: number, body: string): boolean { if (status !== 400) return false const lc = body.toLowerCase() return lc.includes('accesscontrol') || lc.includes('sufficient access permissions')}
/** * Stored `repo_mapping.lastError` when the GitHub repo name can't be a tangled * repo name (e.g. `.github`, whose leading dot the knot rejects as an invalid * path sequence). Terminal: the name won't become valid on retry. */export const INVALID_NAME_ERROR = 'this repository name can\'t be used as a tangled repo name, so it can\'t be mirrored'
/** * True when the knot rejects `repo.create` because the name isn't a valid repo * path (surfaced as a `Generic` 400 mentioning an invalid path sequence). */function isInvalidNameResponse(status: number, body: string): boolean { if (status !== 400) return false return body.toLowerCase().includes('invalid path sequence')}
/** * List the names of the user's existing `sh.tangled.repo` records. Used to * detect a collision before calling the knot so we fail fast with a clear * status instead of a knot 400 (issue #4). */export async function listExistingRepoNames(agent: Agent, did: string): Promise<Set<string>> { const names = new Set<string>() let cursor: string | undefined do { // eslint-disable-next-line no-await-in-loop -- sequential pagination const page = await agent.com.atproto.repo.listRecords({ repo: did, collection: REPO_LEXICON, limit: LIST_RECORDS_PAGE_SIZE, cursor, }) for (const rec of page.data.records) { const value = rec.value as Record<string, unknown> if (typeof value.name === 'string') names.add(value.name) } cursor = page.data.cursor } while (cursor) return names}
/** * Enroll a single GitHub repo on tangled. * * Flow: * 1. Skip if a `repo_mapping` row already exists. * 2. Fetch GitHub repo metadata via the install token. Skip private/fork. * 3. Pick a knot (user default → `DEFAULT_KNOT`). * 4. Get a service-auth JWT for `(aud=did:web:<knot>, lxm=sh.tangled.repo.create)`. * 5. POST to `https://<knot>/xrpc/sh.tangled.repo.create` with * `{ rkey, name, source, defaultBranch }`. The knot clones the repo from * `source` and mints a `repoDid`. * 6. Write a `sh.tangled.repo` record on the user's PDS. * 7. Insert the `repo_mapping` row. */export async function enrollRepo(opts: { oauthSession: OAuthSession installationId: number githubRepoId: number /** * Used by the dashboard "Resync now" action. When true, ignore an existing * `repo_mapping` row in `active` state and re-run the enrolment flow. Note * this still performs the knot procedure call, which mints a *new* * `repoDid`; v1 then overwrites the mapping with the new identity. A * more surgical "poke the knot to re-sync from source" path is a future * improvement. */ force?: boolean}): Promise<EnrolResult> { const db = useDb()
const existing = await db.select({ id: repoMapping.id, status: repoMapping.status, tangledFullName: repoMapping.tangledFullName }) .from(repoMapping) .where(sql`${repoMapping.installationId} = ${opts.installationId} AND ${repoMapping.githubRepoId} = ${opts.githubRepoId}`) if (existing.length > 0 && !opts.force) { return { status: 'already' } }
// 1. GitHub repo metadata. const octokit = await installationOctokit(opts.installationId) const { data: repo } = await octokit.request('GET /repositories/{repository_id}', { repository_id: opts.githubRepoId, })
if (repo.private) return { status: 'skipped', reason: 'private' } if (repo.fork) return { status: 'skipped', reason: 'fork' }
const [owner, name] = repo.full_name.split('/') if (!owner || !name) { throw new Error(`unexpected github full_name shape: ${repo.full_name}`) }
// 2. Pick a knot. Users *can* configure additional knots; v1 always uses // the default. Wiring user choice through is dashboard work. const knot = DEFAULT_KNOT
// 3. Service-auth JWT for the knot procedure. const agent = new Agent(opts.oauthSession)
// Fail fast on a name collision (issue #4): if the user already has a // `sh.tangled.repo` of this name that isn't the one this mapping owns, the // knot would reject `repo.create` with an opaque `AccessControl` 400 and the // job would retry to exhaustion. Record a clear error status instead and // stop, so the dashboard can explain it and the user can act. const ownName = `${opts.oauthSession.did}/${name}` const alreadyOurs = existing[0]?.tangledFullName === ownName if (!alreadyOurs) { const existingNames = await listExistingRepoNames(agent, opts.oauthSession.did) if (existingNames.has(name)) { await recordEnrolError(db, opts, repo.full_name, NAME_CONFLICT_ERROR) return { status: 'skipped', reason: 'name-conflict' } } }
const aud = `did:web:${knot}` const exp = Math.floor(Date.now() / 1000) + 60 const { data: { token } } = await agent.com.atproto.server.getServiceAuth({ aud, lxm: REPO_CREATE_NSID, exp, })
// 4. Knot procedure call. Tangled mints a repoDid here and starts cloning // from `source`. const rkey = tidNow() const sourceUrl = `https://github.com/${owner}/${name}` const knotResponse = await fetch(`https://${knot}/xrpc/${REPO_CREATE_NSID}`, { method: 'POST', headers: { 'authorization': `Bearer ${token}`, 'content-type': 'application/json', }, body: JSON.stringify({ rkey, name, source: sourceUrl, defaultBranch: repo.default_branch, }), }) if (!knotResponse.ok) { const body = await knotResponse.text() // A name collision can still race in between the pre-check and this call // (or the pre-check's listRecords lagged the firehose). Treat it as the // same terminal, user-actionable condition rather than a retryable throw. if (isNameConflictResponse(knotResponse.status, body)) { await recordEnrolError(db, opts, repo.full_name, NAME_CONFLICT_ERROR) return { status: 'skipped', reason: 'name-conflict' } } if (isInvalidNameResponse(knotResponse.status, body)) { await recordEnrolError(db, opts, repo.full_name, INVALID_NAME_ERROR) return { status: 'skipped', reason: 'invalid-name' } } throw new Error(`knot ${knot} returned ${knotResponse.status}: ${body}`) } const knotJson: { repoDid?: string } = await knotResponse.json() const { repoDid } = knotJson if (!repoDid) { throw new Error(`knot ${knot} returned no repoDid`) }
// 5. PDS record so the appview firehose discovers the repo. Includes the // read-only marker and current GitHub metadata from the off — no follow-up // metadata sync needed at enrolment time. const record = mergeRepoRecord(undefined, { name, knot, repoDid, createdAt: new Date().toISOString() }, { full_name: repo.full_name, description: repo.description, homepage: repo.homepage, topics: repo.topics, }, ) await agent.com.atproto.repo.putRecord({ repo: opts.oauthSession.did, collection: REPO_LEXICON, rkey, record, })
// 6. Persist mapping. On a forced resync the row already exists; update // in place so we retain `lastSyncedRefs` (the worker uses it for ref-tip // dedupe) but refresh the tangled-side identifiers and clear any prior // error. if (existing.length > 0) { await db.update(repoMapping) .set({ githubFullName: repo.full_name, tangledRepoDid: repoDid, tangledFullName: `${opts.oauthSession.did}/${name}`, knot, status: 'active', lastError: null, updatedAt: new Date(), }) .where(sql`${repoMapping.id} = ${existing[0]!.id}`) } else { await db.insert(repoMapping).values({ installationId: opts.installationId, githubRepoId: opts.githubRepoId, githubFullName: repo.full_name, tangledRepoDid: repoDid, tangledFullName: `${opts.oauthSession.did}/${name}`, knot, status: 'active', }) }
return { status: 'enrolled' }}
/** * Upsert a `repo_mapping` row in `error` state with a user-facing `lastError`. * Used when enrolment can't complete but we still want the repo to show on the * dashboard with an explanation rather than vanish silently. */async function recordEnrolError( db: ReturnType<typeof useDb>, opts: { installationId: number, githubRepoId: number }, githubFullName: string, message: string,): Promise<void> { const existing = await db.select({ id: repoMapping.id }) .from(repoMapping) .where(sql`${repoMapping.installationId} = ${opts.installationId} AND ${repoMapping.githubRepoId} = ${opts.githubRepoId}`) if (existing.length > 0) { await db.update(repoMapping) .set({ status: 'error', lastError: message, updatedAt: new Date() }) .where(sql`${repoMapping.id} = ${existing[0]!.id}`) return } await db.insert(repoMapping).values({ installationId: opts.installationId, githubRepoId: opts.githubRepoId, githubFullName, status: 'error', lastError: message, })}
export interface SyncMetadataResult { status: 'synced' | 'skipped' reason?: 'no-mapping' | 'disabled' | 'private' | 'fork' | 'no-pds-record'}
/** * Refresh the `sh.tangled.repo` record on the user's PDS to match GitHub's * current description, topics, and homepage. Triggered on `repository.edited`. * * We don't store the rkey locally, so we discover it by listing the user's * `sh.tangled.repo` records and matching on `repoDid` (which we do store). * `swapRecord` is passed for optimistic concurrency in case two webhook * deliveries race. */export async function syncRepoMetadata(opts: { oauthSession: OAuthSession installationId: number githubRepoId: number}): Promise<SyncMetadataResult> { const db = useDb()
const rows = await db.select().from(repoMapping).where( and( eq(repoMapping.installationId, opts.installationId), eq(repoMapping.githubRepoId, opts.githubRepoId), ), ).limit(1) if (rows.length === 0) return { status: 'skipped', reason: 'no-mapping' } const row = rows[0]!
if (row.disabledAt) return { status: 'skipped', reason: 'disabled' } if (!row.tangledRepoDid || !row.knot) return { status: 'skipped', reason: 'no-mapping' }
// Refetch GitHub state rather than trusting the webhook body. const octokit = await installationOctokit(opts.installationId) const { data: repo } = await octokit.request('GET /repositories/{repository_id}', { repository_id: opts.githubRepoId, }) if (repo.private) return { status: 'skipped', reason: 'private' } if (repo.fork) return { status: 'skipped', reason: 'fork' }
const [, name] = repo.full_name.split('/') if (!name) throw new Error(`unexpected github full_name shape: ${repo.full_name}`)
const agent = new Agent(opts.oauthSession)
// Discover the rkey by walking the collection until we find the record // matching this repo's `repoDid`. Typical installs have <100 records so // pagination is mostly defensive. let cursor: string | undefined let found: { uri: string, cid: string, value: Record<string, unknown> } | undefined do { // eslint-disable-next-line no-await-in-loop -- sequential pagination const page = await agent.com.atproto.repo.listRecords({ repo: opts.oauthSession.did, collection: REPO_LEXICON, limit: LIST_RECORDS_PAGE_SIZE, cursor, }) for (const rec of page.data.records) { const value = rec.value as Record<string, unknown> if (value.repoDid === row.tangledRepoDid) { found = { uri: rec.uri, cid: rec.cid, value } break } } cursor = found ? undefined : page.data.cursor } while (cursor)
if (!found) return { status: 'skipped', reason: 'no-pds-record' }
const rkey = found.uri.split('/').pop() if (!rkey) throw new Error(`could not parse rkey from at-uri: ${found.uri}`)
const createdAt = typeof found.value.createdAt === 'string' ? found.value.createdAt : new Date().toISOString()
const record = mergeRepoRecord(found.value, { name, knot: row.knot, repoDid: row.tangledRepoDid, createdAt }, { full_name: repo.full_name, description: repo.description, homepage: repo.homepage, topics: repo.topics, }, )
await agent.com.atproto.repo.putRecord({ repo: opts.oauthSession.did, collection: REPO_LEXICON, rkey, record, swapRecord: found.cid, })
// Refresh the cached display name. githubFullName is display-only (joins go // through githubRepoId), but the dashboard reads it. if (row.githubFullName !== repo.full_name) { await db.update(repoMapping) .set({ githubFullName: repo.full_name, updatedAt: new Date() }) .where(eq(repoMapping.id, row.id)) }
return { status: 'synced' }}