Something went wrong. Try again.
A CLI for Semble link crawling and connection making
Something went wrong. Try again.
3.0 kB · 103 lines
TypeScript
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104import { parse } from 'node-html-parser';
const UA = 'assemblinker/0.0.1 (https://tangled.org/bmann.ca/assemblinker)';
export interface PageData { url: string; title?: string; description?: string; links: string[];}
export async function fetchPage(url: string): Promise<{ html: string; finalUrl: string }> { const response = await fetch(url, { headers: { 'user-agent': UA }, signal: AbortSignal.timeout(15000), redirect: 'follow', }); if (!response.ok) { throw new Error(`HTTP ${response.status} for ${url}`); } const finalUrl = response.url || url; const html = await response.text(); return { html, finalUrl };}
export function parsePage(sourceUrl: string, html: string, finalUrl?: string): PageData { const root = parse(html, { comment: false }); const baseUrl = finalUrl ?? sourceUrl;
const baseHref = root.querySelector('base[href]')?.getAttribute('href'); const resolveBase = baseHref ? new URL(baseHref, baseUrl).href : baseUrl;
const title = root.querySelector('meta[property="og:title"]')?.getAttribute('content') || root.querySelector('title')?.text || undefined;
const description = root.querySelector('meta[property="og:description"]')?.getAttribute('content') || root.querySelector('meta[name="description"]')?.getAttribute('content') || bodyExcerpt(root) || undefined;
const sourceHostname = new URL(sourceUrl).hostname; const seen = new Set<string>(); const links: string[] = [];
for (const anchor of root.querySelectorAll('a[href]')) { const href = anchor.getAttribute('href'); if (!href) continue;
try { const resolved = new URL(href, resolveBase); if (resolved.protocol !== 'http:' && resolved.protocol !== 'https:') continue; resolved.hash = ''; const normalized = resolved.href; if (seen.has(normalized)) continue; if (resolved.hostname === sourceHostname) continue; seen.add(normalized); links.push(normalized); } catch { // ignore malformed URLs } }
return { url: baseUrl, title: title?.trim() || undefined, description: description?.trim() || undefined, links, };}
function bodyExcerpt(root: ReturnType<typeof parse>): string | undefined { const body = root.querySelector('body'); if (!body) return;
for (const tag of body.querySelectorAll('script, style, noscript, iframe, nav, footer, header')) { tag.remove(); }
const text = body.text .replace(/\s+/g, ' ') .trim() .slice(0, 300) .trim();
return text || undefined;}
export async function getPageData(url: string): Promise<PageData> { const { html, finalUrl } = await fetchPage(url); return parsePage(url, html, finalUrl);}
export async function getLinkMetadata(url: string): Promise<Pick<PageData, 'title' | 'description'>> { try { const { title, description } = await getPageData(url); return { title, description }; } catch { return {}; }}