Something went wrong. Try again.
A CLI for Semble link crawling and connection making
Something went wrong. Try again.
7.7 kB · 244 lines
TypeScript
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245#!/usr/bin/env nodeimport { resolve } from 'node:path';import { config as dotenvConfig } from 'dotenv';import { Command } from 'commander';import { Client } from '@atcute/client';import type {} from '@atcute/atproto';import { intro, outro, spinner, log } from '@clack/prompts';import { ensureSession, clearOAuthSession } from './auth.js';import { findExistingCardUri } from './semble.js';import { createCard, createConnection, type UrlMetadata } from './pds.js';import { getPageData, getLinkMetadata } from './crawler.js';import { loadCache, saveCache, getSourceRun, setLinkEntry, isLinked } from './cache.js';import type { AssemblinkerCache } from './types.js';import { promptForLink, confirmContinue, type LinkView } from './interactive.js';import type { LinkEntry, LinkAction as CacheLinkAction } from './types.js';
dotenvConfig({ path: resolve(process.cwd(), '.env') });
const program = new Command();
program .name('assemblinker') .description('Link outbound URLs from a page into Semble') .argument('<url>', 'URL to crawl') .option('--api-key <key>', 'Semble API key (falls back to SEMBLE_API_KEY env)') .option('--max <n>', 'number of links to review per batch', '10') .parse();
const inputUrlArg = program.args[0];const options = program.opts<{ apiKey?: string; max: string }>();const maxLinks = Math.max(1, Math.min(50, Number.parseInt(options.max, 10) || 10));const apiKey = options.apiKey || process.env.SEMBLE_API_KEY;
async function main() { intro('assemblinker');
const urlInput = normalizeUrl(inputUrlArg); if (!urlInput) { log.error('Please provide a valid URL'); process.exit(1); }
const s = spinner(); s.start('Authenticating…'); const session = await ensureSession(); const client = new Client({ handler: session.handler }); s.stop(`Authenticated as ${session.handle ?? session.did}`);
const cache = await loadCache();
s.start('Ensuring source card exists…'); const sourceCardUri = await ensureCardUri(client, session.did, urlInput, cache); s.stop(`Source card: ${sourceCardUri}`);
s.start('Crawling source page…'); let page: Awaited<ReturnType<typeof getPageData>>; try { page = await getPageData(urlInput); } catch (err) { s.stop('Failed to crawl source page'); log.error(err instanceof Error ? err.message : String(err)); process.exit(1); } s.stop(`Found ${page.links.length} external link(s)`);
const externalLinks = page.links.slice(0, 500); if (externalLinks.length === 0) { await finish(0); }
const run = getSourceRun(cache, urlInput); run.cardUri = sourceCardUri;
let quit = false; for (let i = 0; i < externalLinks.length && !quit; i += maxLinks) { const batch = externalLinks.slice(i, i + maxLinks);
s.start(`Fetching metadata for links ${i + 1}–${i + batch.length}…`); for (const linkUrl of batch) { let entry = cache.links[linkUrl]; if (!entry || !entry.title || !entry.description) { try { const meta = await getLinkMetadata(linkUrl); entry = { url: linkUrl, title: meta.title, description: meta.description, fetchedAt: new Date().toISOString(), }; setLinkEntry(cache, linkUrl, entry); } catch (err) { setLinkEntry(cache, linkUrl, { url: linkUrl, fetchedAt: new Date().toISOString(), error: err instanceof Error ? err.message : String(err), }); } }
// If Semble already has this card, remember its AT URI so we can connect by URI later. if (!cache.cards[linkUrl] && apiKey) { try { const existing = await findExistingCardUri(apiKey, linkUrl); if (existing) cache.cards[linkUrl] = existing; } catch { // ignore lookup failures } } } await saveCache(cache); s.stop(`Batch ${i / maxLinks + 1} ready`);
for (const linkUrl of batch) { if (quit) break; if (isLinked(cache, urlInput, linkUrl)) continue;
const actionHistory = run.links[linkUrl] ?? {}; const entry = cache.links[linkUrl]; const view: LinkView = { url: linkUrl, title: entry?.title, description: entry?.description, connected: actionHistory.connected, saved: actionHistory.saved, };
const action = await promptForLink(view);
switch (action) { case 'connect': { const targetUri = cache.cards[linkUrl]; const target = targetUri ?? linkUrl; try { const connectionUri = await createConnection(client, session.did, sourceCardUri, target); log.success(`Connected → ${connectionUri}`); run.links[linkUrl] = { ...actionHistory, connected: true, connectionUri, connectionType: 'RELATED', }; } catch (err) { log.error(err instanceof Error ? err.message : String(err)); } break; } case 'save': { let targetUri = cache.cards[linkUrl]; if (!targetUri) { try { targetUri = await createCard(client, session.did, linkUrl, metadataFromEntry(entry)); cache.cards[linkUrl] = targetUri; log.success(`Saved card → ${targetUri}`); } catch (err) { log.error(err instanceof Error ? err.message : String(err)); break; } } try { const connectionUri = await createConnection(client, session.did, sourceCardUri, targetUri); log.success(`Connected → ${connectionUri}`); run.links[linkUrl] = { ...actionHistory, saved: true, connected: true, cardUri: targetUri, connectionUri, connectionType: 'RELATED', }; } catch (err) { log.error(err instanceof Error ? err.message : String(err)); } break; } case 'skip': break; case 'quit': quit = true; break; }
run.updatedAt = new Date().toISOString(); await saveCache(cache); }
if (!quit && i + maxLinks < externalLinks.length) { const shouldContinue = await confirmContinue(); if (!shouldContinue) break; } }
const connectionCount = Object.values(run.links).filter((a) => a.connected).length; await finish(connectionCount);}
async function finish(connectionCount: number): Promise<never> { outro(`Done. ${connectionCount} connection${connectionCount === 1 ? '' : 's'} assembled`); await clearOAuthSession(); process.exit(0);}
function normalizeUrl(input: string): string | undefined { try { const url = new URL(input); if (url.protocol !== 'http:' && url.protocol !== 'https:') return; return url.href; } catch { return undefined; }}
async function ensureCardUri( client: Client, did: string, url: string, cache: AssemblinkerCache,): Promise<string> { if (cache.cards[url]) return cache.cards[url];
if (apiKey) { const existing = await findExistingCardUri(apiKey, url); if (existing) { cache.cards[url] = existing; return existing; } }
const { title, description } = await getLinkMetadata(url); const uri = await createCard(client, did, url, { title, description }); cache.cards[url] = uri; return uri;}
function metadataFromEntry(entry?: LinkEntry): UrlMetadata { return { title: entry?.title, description: entry?.description, };}
main().catch((err) => { console.error(err); process.exit(1);});