Something went wrong. Try again.
An experimental, prototype Atmospheric (atproto) virtual (2.5d) community workspace for @spacecadets.love fieldwork.spacecadets.love
Something went wrong. Try again.
37 kB · 684 lines
JavaScript
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685// parse.js — read what a web page says about itself, and decide which web// addresses are safe to ask about.//// When someone pastes a link into Fieldwork we want the page's title, picture,// icon and, above all, what it says about the open social web: is it an atproto// record, who wrote it (atproto, fediverse, IndieWeb), where is its feed. Pages// publish this as small labelled tags in their <head> ("meta" and "link" tags),// as JSON-LD, and as microformats in the body.//// This file does only that reading. It is plain JavaScript with no libraries and// no browser features, so the very same code runs in the Worker (worker/index.js)// and in Node (the smoke test). It never fetches anything.//// Everything in a page is written by a stranger, so treat it as hostile:// - it only looks at the first 384 KB of the page, in one pass (no DOM, no// backtracking), so a huge or twisted page can't make it slow;// - web addresses must be http(s) (javascript:, data: and friends are dropped);// - atproto addresses (at://...) must pass a strict check or they are dropped;// - text is cleaned (entities decoded once, control characters removed) and cut// to a fixed length, and lists are cut to 20 entries.// The result is plain data. Whoever shows it must still show it as text, never as// markup.//// Exports: parsePage, normalizeTarget, cleanAtUri, cleanDid, kindFromContentType,// iconPixels, applyManifest.
const MAX_HTML = 384 * 1024; // how much of a page is readconst MAX_TAGS = 15000; // stop after this many tags (real pages put what we want near the top)const LIST_CAP = 20; // longest list we returnconst TITLE_CAP = 300, DESC_CAP = 1000, STR_CAP = 500, URL_CAP = 1000;
// ───────────────────────── cleaning text ─────────────────────────
const NAMED = new Map(Object.entries({ amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', ndash: '–', mdash: '—', hellip: '…', lsquo: '‘', rsquo: '’', ldquo: '“', rdquo: '”', copy: '©', reg: '®', trade: '™', middot: '·', bull: '•', laquo: '«', raquo: '»', times: '×', deg: '°', euro: '€', pound: '£', hearts: '♥',}));
/** Turn & ' ' < and a few friends into the characters they stand for (once, not repeatedly). */function decode(s) { if (s.indexOf('&') < 0) return s; return s.replace(/&(?:#[xX]([0-9a-fA-F]{1,6})|#([0-9]{1,7})|([A-Za-z][A-Za-z0-9]{1,9}));/g, (m, hex, dec, name) => { if (name) return NAMED.get(name) ?? NAMED.get(name.toLowerCase()) ?? m; const cp = hex ? parseInt(hex, 16) : parseInt(dec, 10); if (!cp || cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff)) return '�'; return String.fromCodePoint(cp); });}
/** Text from a page, made safe to show: decoded, control and direction-flipping characters removed, one line, trimmed, cut to `max`. */function clean(s, max = STR_CAP) { if (s == null) return ''; let t = decode(String(s)).replace(/[\u0000-\u001f\u007f-\u009f---]/g, ' ').replace(/\s+/g, ' ').trim(); if (t.length > max) { t = t.slice(0, max); const last = t.charCodeAt(t.length - 1); if (last >= 0xd800 && last <= 0xdbff) t = t.slice(0, -1); // don't leave half a character t = t.trim(); } return t;}
/** A web address as an absolute http(s) address with no password in it, or null. */function toUrl(raw, base) { if (raw == null) return null; const s = decode(String(raw)).replace(/[\u0000-\u001f\u007f-\u009f]/g, '').trim(); if (!s || s.length > 3000) return null; let u; try { u = new URL(s, base || undefined); } catch { return null; } if (u.protocol !== 'http:' && u.protocol !== 'https:') return null; if (!u.hostname) return null; u.username = ''; u.password = ''; const href = u.href; return href.length <= URL_CAP ? href : null;}
// ───────────────────────── atproto and friends ─────────────────────────
const AT_URI = /^at:\/\/(did:[a-z]+:[A-Za-z0-9._:%-]+|[a-z0-9.-]+)(\/[a-zA-Z0-9.-]+(\/[A-Za-z0-9._~:-]+)?)?$/;const HANDLE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$/;const NSID = /^[a-zA-Z]([a-zA-Z0-9-]*[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?)+$/;const DID = /^did:[a-z]+:[A-Za-z0-9._:%-]{1,200}$/;
/** * An at:// address that is shaped like a real one, or null. The pattern is strict * on purpose; on top of it we refuse "." and ".." parts and names that are not * handles, DIDs or record-type names, so junk like at://x/../y never gets through. */export function cleanAtUri(raw) { if (raw == null) return null; const s = decode(String(raw)).trim(); if (s.length > STR_CAP || !AT_URI.test(s)) return null; const [authority, collection, rkey] = s.slice(5).split('/'); if (!authority.startsWith('did:') && !HANDLE.test(authority)) return null; if (collection !== undefined && !NSID.test(collection)) return null; if (rkey !== undefined && (rkey === '.' || rkey === '..')) return null; return s;}
/** A DID (an atproto account's permanent id) or null. */export function cleanDid(raw) { const s = String(raw ?? '').trim(); return DID.test(s) ? s : null;}
/** An at:// address, also accepting a bare did:... (some pages leave off the at://). */function atFromContent(raw) { const s = decode(String(raw ?? '')).trim(); return cleanAtUri(s.startsWith('did:') ? 'at://' + s : s);}
const authorityOf = (uri) => uri.slice(5).split('/')[0];
/** Who an at:// address belongs to: { did } or { handle }. */function whoIs(uri) { const a = authorityOf(uri); return a.startsWith('did:') ? { did: a } : { handle: a };}
/** "@user@host" (or "user@host") for the fediverse, normalised, or null. */function cleanFediverse(raw) { const m = /^@?([A-Za-z0-9._-]{1,64})@([A-Za-z0-9.-]{3,253})$/.exec(clean(raw, 400)); if (!m || !m[2].includes('.') || /^[.-]|[.-]$|\.\./.test(m[2])) return null; return `@${m[1]}@${m[2].toLowerCase()}`;}
/** If a web address is a Bluesky profile page, say whose: { did } or { handle }. */function bskyProfile(url) { try { const u = new URL(url); if (u.hostname !== 'bsky.app') return null; const m = /^\/profile\/([^/]+)\/?$/.exec(u.pathname); if (!m) return null; const who = decodeURIComponent(m[1]); return cleanDid(who) ? { did: who } : HANDLE.test(who.toLowerCase()) ? { handle: who.toLowerCase() } : null; } catch { return null; }}
// ───────────────────────── which addresses we may look up ─────────────────────────
/** * Check a web address someone asked us to look up, and return it in one standard * form (no #fragment). Throws an Error with a plain-language message if it isn't * allowed: only http and https, no passwords, only the normal ports, and only * public-looking site names (no IP addresses, no localhost, no names that only * exist inside a private network). The same check is applied to every redirect. */export function normalizeTarget(input) { if (typeof input !== 'string' || !input.trim()) throw new Error('Give me a web address to look up.'); let s = input.trim(); if (s.length > 2000) throw new Error('That web address is too long.'); if (!/^[a-z][a-z0-9+.-]*:/i.test(s)) s = 'https://' + s; // "example.com/page" works too let u; try { u = new URL(s); } catch { throw new Error('That does not look like a web address.'); } if (u.protocol !== 'http:' && u.protocol !== 'https:') throw new Error('Only http and https web addresses can be looked up.'); if (u.username || u.password) throw new Error('Web addresses with a name or password in them are not looked up.'); if (u.port && u.port !== '80' && u.port !== '443') throw new Error('Only the normal web ports (80 and 443) are allowed.'); const host = u.hostname.toLowerCase().replace(/\.$/, ''); if (!host) throw new Error('That web address has no site name.'); if (host.startsWith('[') || host.includes(':')) throw new Error('Addresses made of numbers (IP addresses) are not looked up.'); if (/^[0-9.]+$/.test(host) || /^0x[0-9a-f]+$/i.test(host)) throw new Error('Addresses made of numbers (IP addresses) are not looked up.'); if (host.length > 253 || !/^[a-z0-9.-]+$/.test(host) || host.includes('..') || host.startsWith('.')) throw new Error('That site name is not valid.'); if (!host.includes('.')) throw new Error('That site name has no dot in it, so it is not a public site.'); if (host === 'localhost' || /\.(localhost|local|internal|localdomain|lan|home\.arpa)$/.test(host)) throw new Error('Sites that only exist on a private network are not looked up.'); u.hash = ''; return u.href;}
/** What kind of thing a non-page file is, from its content type. */export function kindFromContentType(ct) { const t = String(ct || '').split(';')[0].trim().toLowerCase(); if (t.startsWith('image/')) return 'image'; if (t.startsWith('video/')) return 'video'; if (t.startsWith('audio/')) return 'audio'; if (t === 'application/pdf') return 'pdf'; if (t.startsWith('model/')) return 'model'; return 'file';}
// ───────────────────────── reading the page (one pass) ─────────────────────────
// Only these attributes are ever kept; everything else on a tag is skipped.const WANTED = new Set(['class', 'rel', 'href', 'src', 'content', 'name', 'property', 'type', 'sizes', 'title', 'lang', 'media', 'color', 'datetime', 'value', 'alt', 'data']);// Tags that never have contents or a closing tag.const VOID = new Set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr']);// Tags whose insides are not markup. We skip them, except <title> and JSON-LD scripts.const RAW_END = { script: /<\/script(?=[\s/>])/gi, style: /<\/style(?=[\s/>])/gi, textarea: /<\/textarea(?=[\s/>])/gi, title: /<\/title(?=[\s/>])/gi,};// The <meta> names worth keeping (anything else is ignored so it can't crowd these out).const META_KEY = /^(?:(?:og|twitter|at|fediverse|atprotocol|article|video|music|book|profile):|(?:description|author|generator|theme-color|application-name|date|dc\.date|dcterms\.(?:created|modified|date)|msapplication-tilecolor)$)/;const isSpace = (c) => c === 32 || c === 9 || c === 10 || c === 12 || c === 13;
function scan(src) { const out = { title: null, lang: null, base: null, metas: [], links: [], ld: [], entry: null, cards: [] }; const n = src.length; const stack = []; // open tags inside a microformat; empty when we're outside one const ctxs = []; // microformat objects being filled in (innermost last) let collectors = 0; // how many open tags are collecting their text let i = 0, tags = 0, svg = 0;
const addText = (chunk) => { for (const f of stack) if (f.collect && f.text.length < 600) f.text += chunk.slice(0, 600 - f.text.length); }; const setProp = (ctx, token, value) => { const d = ctx.d; if (value == null || value === '') return; if (ctx.type === 'entry') { switch (token) { case 'p-name': d.name ??= value; break; case 'dt-published': d.published ??= value; break; case 'u-photo': d.photo ??= value; break; case 'u-featured': d.featured ??= value; break; case 'u-url': d.url ??= value; break; case 'p-summary': d.summary ??= value; break; case 'u-syndication': if (d.syndication.length < LIST_CAP) d.syndication.push(value); break; case 'u-in-reply-to': if (d.inReplyTo.length < LIST_CAP) d.inReplyTo.push(value); break; } } else { switch (token) { case 'p-name': d.name ??= value; break; case 'u-url': d.url ??= value; break; case 'u-photo': d.photo ??= value; break; } } }; const finish = (f) => { if (f.collect) collectors--; const text = f.collect ? f.text : ''; for (const [ctx, token] of f.props) setProp(ctx, token, token.startsWith('u-') ? text.trim() : text.replace(/\s+/g, ' ').trim().slice(0, 300)); if (f.ctx) { ctxs.pop(); const t = text.replace(/\s+/g, ' ').trim(); if (f.ctx.type === 'card' && !f.ctx.d.name && t && t.length <= 100) f.ctx.d.name = t; // a card with no name uses its own words } }; const close = (name) => { if (name === 'svg') svg = Math.max(0, svg - 1); if (!stack.length) return; // Find the matching open tag a few levels up; if there isn't one, ignore the stray closing tag. let k = stack.length - 1; const stop = Math.max(0, stack.length - 32); while (k >= stop && stack[k].name !== name) k--; if (k < stop) return; while (stack.length > k) finish(stack.pop()); };
const open = (name, a) => { // Microformats first (they need to know about every tag while we're inside one). const inside = ctxs.length > 0; const classes = a.class; let tokens = null; if (classes && (inside || classes.includes('h-'))) tokens = classes.split(/\s+/); if (inside || tokens) { let own = null; const parent = ctxs[ctxs.length - 1]; const isEntry = tokens ? tokens.includes('h-entry') : false; const isCard = tokens ? tokens.includes('h-card') : false; const isAuthor = tokens ? (tokens.includes('p-author') || tokens.includes('u-author')) : false; if (!inside && isEntry && !out.entry) { own = { type: 'entry', d: { name: null, published: null, photo: null, featured: null, url: null, summary: null, author: null, syndication: [], inReplyTo: [] } }; out.entry = own.d; } else if (!inside && isCard && out.cards.length < 10) { own = { type: 'card', d: { name: null, url: null, photo: null } }; out.cards.push(own.d); } else if (inside && parent.type === 'entry' && !parent.d.author && isAuthor) { own = { type: 'card', d: { name: null, url: null, photo: null } }; parent.d.author = own.d; } if (own) { // a card can read its link and picture from its own tag if (own.type === 'card') { if (name === 'a' && a.href) own.d.url = a.href; if (name === 'img') { if (a.src) own.d.photo = a.src; if (a.alt) own.d.name = a.alt; } } } const props = []; let collect = false; if (inside && tokens) { for (const t of tokens) { if (!(t.startsWith('p-') || t.startsWith('u-') || t.startsWith('dt-'))) continue; if (t === 'p-author' || t === 'u-author') continue; let v; // value from an attribute, or undefined if the text is needed if (t.startsWith('u-')) v = (name === 'a' || name === 'area' || name === 'link') ? a.href : (name === 'img' || name === 'audio' || name === 'video' || name === 'source') ? a.src : name === 'object' ? a.data : undefined; else if (t.startsWith('dt-')) v = name === 'time' ? a.datetime : name === 'abbr' ? a.title : name === 'data' ? a.value : undefined; else v = name === 'img' ? a.alt : name === 'abbr' ? a.title : name === 'data' ? a.value : undefined; if (v !== undefined) setProp(parent, t, v.slice(0, 2000)); else { props.push([parent, t]); collect = true; } } } if (!VOID.has(name) && stack.length < 512 && (inside || own)) { if (own) collect = true; stack.push({ name, ctx: own, props, collect, text: '' }); if (collect) collectors++; if (own) ctxs.push(own); } else if (name === 'img' && collectors && a.alt) addText(a.alt); } };
outer: while (i < n) { const lt = src.indexOf('<', i); if (lt < 0) { if (collectors) addText(src.slice(i)); break; } if (lt > i && collectors) addText(src.slice(i, lt)); i = lt; const c = src.charCodeAt(i + 1); if (c === 33) { // <!-- comment --> or <!doctype> if (src.startsWith('<!--', i)) { const e = src.indexOf('-->', i + 4); i = e < 0 ? n : e + 3; } else { const e = src.indexOf('>', i + 2); i = e < 0 ? n : e + 1; } continue; } if (c === 63) { const e = src.indexOf('>', i + 2); i = e < 0 ? n : e + 1; continue; } if (c === 47) { // closing tag let j = i + 2; while (j < n) { const d = src.charCodeAt(j); if (isSpace(d) || d === 47 || d === 62) break; j++; } const name = src.slice(i + 2, j).toLowerCase(); const e = src.indexOf('>', j); i = e < 0 ? n : e + 1; if (name) close(name); continue; } if (!((c >= 65 && c <= 90) || (c >= 97 && c <= 122))) { i++; if (collectors) addText('<'); continue; }
// An opening tag: read its name, then its attributes. if (++tags > MAX_TAGS) break; let j = i + 1; while (j < n) { const d = src.charCodeAt(j); if (isSpace(d) || d === 47 || d === 62) break; j++; } const name = src.slice(i + 1, j).toLowerCase(); const a = Object.create(null); let count = 0, selfClose = false; for (;;) { let d = 0; while (j < n) { d = src.charCodeAt(j); if (isSpace(d) || d === 47) j++; else break; } if (j >= n) { i = n; break outer; } if (d === 62) { selfClose = src.charCodeAt(j - 1) === 47; j++; break; } const s = j; while (j < n) { d = src.charCodeAt(j); if (isSpace(d) || d === 61 || d === 62 || d === 47) break; j++; } if (j === s) { j++; continue; } // a stray "=" or similar const an = src.slice(s, j); while (j < n && isSpace(src.charCodeAt(j))) j++; let value = ''; if (src.charCodeAt(j) === 61) { j++; while (j < n && isSpace(src.charCodeAt(j))) j++; const q = src.charCodeAt(j); if (q === 34 || q === 39) { const e = src.indexOf(q === 34 ? '"' : "'", j + 1); if (e < 0) { i = n; break outer; } // a quote that never ends: nothing after it can be trusted value = src.slice(j + 1, e); j = e + 1; } else { const s2 = j; while (j < n) { d = src.charCodeAt(j); if (isSpace(d) || d === 62) break; j++; } value = src.slice(s2, j); } } if (++count <= 40) { const ln = an.toLowerCase(); if (WANTED.has(ln) && a[ln] === undefined) a[ln] = value; } } i = j;
// Tags with raw insides. const re = RAW_END[name]; if (re) { re.lastIndex = i; const m = re.exec(src); const body = m ? src.slice(i, m.index) : src.slice(i); if (m) { const e = src.indexOf('>', m.index); i = e < 0 ? n : e + 1; } else i = n; if (name === 'title') { if (out.title === null && svg === 0) out.title = body.slice(0, 2000); } else if (name === 'script' && out.ld.length < 4 && body.length <= 120000 && /ld\+json/i.test(a.type || '')) out.ld.push(body); continue; }
switch (name) { case 'html': if (out.lang === null && a.lang) out.lang = a.lang; break; case 'svg': svg++; break; case 'base': if (out.base === null && a.href) out.base = a.href; break; case 'meta': { if (a.content === undefined) break; for (const key of [a.property, a.name]) { if (key == null) continue; const k = key.trim().toLowerCase(); if (META_KEY.test(k) && out.metas.length < 400) out.metas.push({ key: k, content: a.content.slice(0, 2500), media: a.media || '' }); } break; } case 'link': case 'a': { if (a.rel && a.href !== undefined && out.links.length < 500) { out.links.push({ rel: a.rel.toLowerCase().split(/\s+/).filter(Boolean), href: a.href.slice(0, 3200), type: (a.type || '').toLowerCase().trim(), sizes: (a.sizes || '').slice(0, 100), title: (a.title || '').slice(0, 600), tag: name }); } break; } } open(name, a); } while (stack.length) finish(stack.pop()); return out;}
// ───────────────────────── JSON-LD ─────────────────────────
const LD_MAIN = /^(Article|NewsArticle|BlogPosting|SocialMediaPosting|TechArticle|ScholarlyArticle|Report|WebPage|ItemPage|CollectionPage|AboutPage|ProfilePage|Product|Recipe|Event|Book|Movie|MusicRecording|PodcastEpisode|VideoObject|AudioObject|ImageObject|CreativeWork|Course|SoftwareApplication|Dataset)$/;
function ldFlatten(v, out, depth) { if (out.length > 400 || depth > 6) return; if (Array.isArray(v)) { for (const x of v.slice(0, 100)) ldFlatten(x, out, depth + 1); return; } if (v && typeof v === 'object') { out.push(v); if (v['@graph']) ldFlatten(v['@graph'], out, depth + 1); }}const ldTypes = (v) => (Array.isArray(v['@type']) ? v['@type'] : [v['@type']]).filter((t) => typeof t === 'string').map((t) => t.replace(/^.*[/#]/, ''));function ldStr(v) { if (typeof v === 'string') return v; if (v && typeof v === 'object' && !Array.isArray(v)) { if (typeof v['@value'] === 'string') return v['@value']; } return '';}function ldFirstUrl(v) { if (Array.isArray(v)) v = v[0]; if (typeof v === 'string') return v; if (v && typeof v === 'object') return ldStr(v.url) || ldStr(v.contentUrl) || ldStr(v['@id']); return '';}
function readLd(texts) { const nodes = []; for (const t of texts) { try { ldFlatten(JSON.parse(t), nodes, 0); } catch { /* not JSON: ignore */ } } const ld = { main: null, siteName: '', authors: [], video: null }; if (!nodes.length) return ld; const named = (x) => ldStr(x.headline) || ldStr(x.name); ld.main = nodes.find((x) => ldTypes(x).some((t) => LD_MAIN.test(t)) && named(x)) || null; const site = nodes.find((x) => ldTypes(x).includes('WebSite') && ldStr(x.name)); const pub = ld.main && (ld.main.publisher || ld.main.isPartOf); ld.siteName = (site && ldStr(site.name)) || (pub && typeof pub === 'object' && ldStr(pub.name)) || ''; if (ld.main) { const list = Array.isArray(ld.main.author) ? ld.main.author : ld.main.author ? [ld.main.author] : []; for (const au of list.slice(0, LIST_CAP)) { if (typeof au === 'string') ld.authors.push({ name: au }); else if (au && typeof au === 'object') { const same = (Array.isArray(au.sameAs) ? au.sameAs : [au.sameAs]).filter((x) => typeof x === 'string'); ld.authors.push({ name: ldStr(au.name), url: ldStr(au.url) || same[0] || '', photo: ldFirstUrl(au.image), same }); } } } const vid = nodes.find((x) => ldTypes(x).includes('VideoObject') && (x.embedUrl || x.contentUrl)); if (vid) ld.video = { embedUrl: ldStr(vid.embedUrl), contentUrl: ldStr(vid.contentUrl), width: vid.width, height: vid.height }; return ld;}
// ───────────────────────── putting it together ─────────────────────────
const pushUnique = (arr, v) => { if (v && !arr.includes(v) && arr.length < LIST_CAP) arr.push(v); };const posInt = (v) => { const n = Number.parseInt(String(v ?? '').replace(/px$/i, ''), 10); return Number.isFinite(n) && n > 0 && n <= 20000 ? n : null; };
/** A date as it was written if it looks like one, else null. */function cleanDate(raw) { const s = clean(raw, 60); if (!s) return null; if (/^\d{4}-\d{2}-\d{2}([T ][0-9:.+\-Zz ]{0,40})?$/.test(s)) return s; const t = Date.parse(s); return Number.isNaN(t) ? null : new Date(t).toISOString();}
function cleanColor(raw) { const s = clean(raw, 60); return /^(#[0-9a-fA-F]{3,8}|[a-zA-Z]{3,24}|(?:rgb|hsl)a?\([0-9 ,.%/-]+\))$/.test(s) ? s : null;}
/** The biggest side in a sizes="32x32 64x64" list; 9999 for "any" (a scalable icon); 0 if unknown. */function sizesPixels(sizes) { let best = 0; for (const part of String(sizes || '').toLowerCase().split(/\s+/)) { if (part === 'any') return 9999; const m = /^(\d{1,5})x(\d{1,5})$/.exec(part); if (m) best = Math.max(best, +m[1], +m[2]); } return best;}
/** How big (in pixels on a side) an icon found by parsePage is. 9999 means scalable, 0 unknown. */export function iconPixels(icon) { if (!icon) return 0; if (Number.isFinite(icon.px)) return icon.px; return /\.svg(\?|$)/i.test(icon.url || '') ? 9999 : sizesPixels(icon.sizes);}
/** The page's icon: biggest apple-touch-icon, else biggest link icon (a scalable one beats any size), else a pinned-tab mask icon. */function pickIcon(links, base) { let apple = null, plain = null, mask = null; for (const l of links) { const isApple = l.rel.some((r) => r === 'apple-touch-icon' || r === 'apple-touch-icon-precomposed'); const isIcon = l.rel.includes('icon'); const isMask = l.rel.includes('mask-icon'); if (!isApple && !isIcon && !isMask) continue; const url = toUrl(l.href, base); if (!url) continue; const svg = l.type.includes('svg') || /\.svg(\?|$)/i.test(url); const px = svg ? 9999 : sizesPixels(l.sizes) || (isApple ? 180 : 0); const cand = { url, sizes: clean(l.sizes, 100) || (svg ? 'any' : null), px }; if (isApple) { if (!apple || px > apple.px) apple = cand; } else if (isIcon) { if (!plain || px > plain.px) plain = cand; } else if (!mask) mask = { ...cand, px: 0 }; } return apple || plain || mask || null;}
/** * Read a page's own description of itself. `html` is the page text (only the * first 384 KB is read); `baseUrl` is the address it came from. Returns the * normalised object described in worker/README.md. Never throws on bad input. */export function parsePage(html, baseUrl) { const st = scan(String(html ?? '').slice(0, MAX_HTML)); const L = st.links; const base = toUrl(st.base, baseUrl) || baseUrl; const U = (raw) => toUrl(raw, base); const metaAll = (key) => st.metas.filter((m) => m.key === key).map((m) => m.content); const meta1 = (key) => metaAll(key).find((c) => c.trim()) || ''; // the first non-empty value, as written (each use cleans it once) const rel = (r) => L.filter((l) => l.rel.includes(r)); const ld = readLd(st.ld); const ldMain = ld.main || {}; const entry = st.entry;
// atproto const atproto = { canonical: [], alternate: [], author: [], me: [], custom: {}, publication: null, siteDid: null }; for (const c of metaAll('at:canonical')) pushUnique(atproto.canonical, atFromContent(c)); for (const c of metaAll('at:alternate')) pushUnique(atproto.alternate, atFromContent(c)); for (const c of metaAll('at:author')) pushUnique(atproto.author, atFromContent(c)); for (const c of metaAll('atprotocol:creator')) pushUnique(atproto.author, atFromContent(c)); for (const c of metaAll('at:me')) pushUnique(atproto.me, atFromContent(c)); for (const m of st.metas) { if (!/^at:[a-z0-9-]+:[a-z0-9._-]+$/.test(m.key)) continue; if (Object.keys(atproto.custom).length >= 10 && !atproto.custom[m.key]) continue; const uri = atFromContent(m.content); if (uri) pushUnique(atproto.custom[m.key] ||= [], uri); } for (const l of rel('site.standard.document')) pushUnique(atproto.canonical, cleanAtUri(l.href)); for (const l of rel('site.standard.publication')) { const u = cleanAtUri(l.href); if (u && !atproto.publication) atproto.publication = u; } let oembed = null; for (const l of rel('alternate')) { if (/^at:\/\//i.test(l.href.trim())) pushUnique(atproto.alternate, cleanAtUri(l.href)); else if (/oembed/.test(l.type)) { const u = U(l.href); if (u && (!oembed || l.type.includes('json'))) oembed = u; try { pushUnique(atproto.alternate, cleanAtUri(new URL(u).searchParams.get('url'))); } catch { /* not a web address */ } } }
// fediverse const fediverse = { creator: [], activity: null }; for (const c of metaAll('fediverse:creator')) pushUnique(fediverse.creator, cleanFediverse(c)); for (const l of rel('alternate')) if (l.type === 'application/activity+json' && !fediverse.activity) fediverse.activity = U(l.href);
// IndieWeb const first = (r) => { for (const l of rel(r)) { const u = U(l.href); if (u) return u; } return null; }; const indieweb = { me: [], webmention: first('webmention'), micropub: first('micropub'), indieauth: first('indieauth-metadata') || first('authorization_endpoint'), hub: first('hub'), syndication: [], inReplyTo: [], hEntry: null, hCard: null, endpoints: { pingback: first('pingback'), microsub: first('microsub'), authorization: first('authorization_endpoint'), token: first('token_endpoint'), self: first('self') } }; for (const l of rel('me')) pushUnique(indieweb.me, U(l.href)); for (const l of rel('syndication')) pushUnique(indieweb.syndication, U(l.href)); for (const l of rel('in-reply-to')) pushUnique(indieweb.inReplyTo, U(l.href)); if (entry) { const au = entry.author; indieweb.hEntry = { name: clean(entry.name, TITLE_CAP) || null, published: cleanDate(entry.published), photo: U(entry.photo), url: U(entry.url), author: au ? { name: clean(au.name, 200) || null, url: U(au.url), photo: U(au.photo) } : null, syndication: [], }; for (const s of entry.syndication) pushUnique(indieweb.hEntry.syndication, U(s)); for (const s of entry.syndication) pushUnique(indieweb.syndication, U(s)); for (const s of entry.inReplyTo) pushUnique(indieweb.inReplyTo, U(s)); } const pageHref = (u) => (u || '').replace(/#.*$/, '').replace(/\/$/, ''); const cards = st.cards.map((d) => ({ name: clean(d.name, 200) || null, url: U(d.url), photo: U(d.photo) })).filter((c) => c.name || c.url); const card = cards.find((c) => c.url && pageHref(c.url) === pageHref(baseUrl)) || (cards.length === 1 ? cards[0] : null); if (card) indieweb.hCard = card;
// feeds const feeds = []; for (const l of rel('alternate')) { if (!/^application\/(rss\+xml|atom\+xml|feed\+json)$/.test(l.type)) continue; const url = U(l.href); if (url && !feeds.some((f) => f.url === url) && feeds.length < LIST_CAP) feeds.push({ url, type: l.type, title: clean(l.title, 200) }); }
// authors: from the most trustworthy source down; the same person is merged const authors = []; const addAuthor = (raw) => { const a = { name: (raw.done ? raw.name : clean(raw.name, 200)) || null, url: raw.url || null, photo: raw.photo || null, did: raw.did || null, handle: raw.handle || null, fediverse: raw.fediverse || null }; if (a.url && !(raw.did || raw.handle)) Object.assign(a, bskyProfile(a.url) || {}); for (const s of raw.same || []) { const u = U(s); const b = u && bskyProfile(u); if (b) { a.did ||= b.did || null; a.handle ||= b.handle || null; } } if (!(a.name || a.url || a.did || a.handle || a.fediverse)) return; const same = (x) => (a.did && x.did === a.did) || (a.url && x.url === a.url) || (a.fediverse && x.fediverse === a.fediverse) || (a.handle && x.handle === a.handle) || (a.name && x.name && x.name.toLowerCase() === a.name.toLowerCase()); const hits = authors.filter(same); if (!hits.length) { if (authors.length < LIST_CAP) authors.push(a); return; } const hit = hits[0]; // this one entry may tie several earlier ones together: fold them all into the first for (const extra of hits.slice(1)) { for (const k of Object.keys(extra)) hit[k] ||= extra[k]; authors.splice(authors.indexOf(extra), 1); } for (const k of Object.keys(a)) hit[k] ||= a[k]; }; for (const uri of atproto.author) addAuthor(whoIs(uri)); for (const f of fediverse.creator) addAuthor({ fediverse: f }); for (const l of rel('author')) { const u = U(l.href); if (u) addAuthor({ url: u }); } for (const c of metaAll('author')) { const nm = clean(c, 200); if (nm) addAuthor({ name: nm, done: true }); } for (const au of ld.authors) addAuthor({ name: au.name, url: U(au.url), photo: U(au.photo), same: au.same }); if (indieweb.hEntry?.author) addAuthor({ ...indieweb.hEntry.author, done: true }); if (!authors.length && indieweb.hCard) addAuthor({ ...indieweb.hCard, done: true }); if (!authors.length) { const tw = clean(meta1('twitter:creator'), 100); if (/^@\w{1,30}$/.test(tw)) addAuthor({ name: tw }); }
// the basics const title = clean(meta1('og:title'), TITLE_CAP) || clean(meta1('twitter:title'), TITLE_CAP) || indieweb.hEntry?.name || clean(ldStr(ldMain.headline) || ldStr(ldMain.name), TITLE_CAP) || clean(st.title, TITLE_CAP) || null; const description = clean(meta1('og:description'), DESC_CAP) || clean(meta1('twitter:description'), DESC_CAP) || clean(meta1('description'), DESC_CAP) || clean(entry?.summary, DESC_CAP) || clean(ldStr(ldMain.description), DESC_CAP) || null; const siteName = clean(meta1('og:site_name'), 200) || clean(ld.siteName, 200) || clean(meta1('application-name'), 200) || null; const lang = [st.lang, meta1('og:locale')].map((v) => clean(v, 35)).find((v) => /^[A-Za-z]{2,3}([-_][A-Za-z0-9]{1,8})*$/.test(v)) || null; const canonical = U(rel('canonical')[0]?.href) || U(meta1('og:url')) || null;
// picture const imgUrl = U(meta1('og:image:secure_url')) || U(meta1('og:image')) || U(meta1('og:image:url')) || U(meta1('twitter:image')) || U(meta1('twitter:image:src')) || U(entry?.photo) || U(entry?.featured) || U(ldFirstUrl(ldMain.image)) || null; const image = imgUrl ? { url: imgUrl, width: posInt(meta1('og:image:width')), height: posInt(meta1('og:image:height')), alt: clean(meta1('og:image:alt') || meta1('twitter:image:alt'), 300) || null, } : null;
// video let video = null; { let url = U(meta1('og:video:secure_url')) || U(meta1('og:video:url')) || U(meta1('og:video')); let type = clean(meta1('og:video:type'), 100).toLowerCase() || null; let embedUrl = U(meta1('twitter:player')); if (type && /flash/.test(type)) { url = null; type = null; } else if (url && type === 'text/html') { embedUrl ||= url; url = null; } embedUrl ||= U(ld.video?.embedUrl); url ||= U(ld.video?.contentUrl); if (url || embedUrl) video = { url: url || null, type, width: posInt(meta1('og:video:width') || ld.video?.width), height: posInt(meta1('og:video:height') || ld.video?.height), embedUrl: embedUrl || null }; }
const ldLicense = U(ldStr(ldMain.license)); const result = { ok: true, url: baseUrl, canonical, title, description, siteName, type: clean(meta1('og:type'), 60) || null, lang, image, video, icon: pickIcon(L, base), themeColor: (() => { const all = st.metas.filter((m) => m.key === 'theme-color'); const m = all.find((x) => !x.media) || all.find((x) => !/dark/i.test(x.media)) || all[0]; return m ? cleanColor(m.content) : null; })(), published: cleanDate(meta1('article:published_time')) || cleanDate(indieweb.hEntry?.published) || cleanDate(ldStr(ldMain.datePublished)) || cleanDate(meta1('date') || meta1('dc.date') || meta1('dcterms.created') || meta1('dcterms.date')) || null, modified: cleanDate(meta1('article:modified_time')) || cleanDate(meta1('og:updated_time')) || cleanDate(ldStr(ldMain.dateModified)) || cleanDate(meta1('dcterms.modified')) || null, license: U(rel('license')[0]?.href) || ldLicense || null, generator: clean(meta1('generator'), 100) || null, authors, atproto, fediverse, indieweb, feeds, oembed, manifest: U(rel('manifest')[0]?.href) || null, kind: 'page', contentType: 'text/html', }; return result;}
// ───────────────────────── the web app manifest ─────────────────────────
/** * Some sites keep their best icon only in a "web app manifest" file. Give this the * parsed JSON of that file (and its address) and it fills in a better icon, theme * colour and site name on `page`, only where the page didn't already have * something at least as good. A bad or hostile file just changes nothing. */export function applyManifest(page, json, manifestUrl) { if (!json || typeof json !== 'object' || Array.isArray(json)) return page; let best = null; const icons = Array.isArray(json.icons) ? json.icons.slice(0, LIST_CAP) : []; for (const ic of icons) { if (!ic || typeof ic !== 'object' || typeof ic.src !== 'string') continue; if (typeof ic.purpose === 'string' && /^monochrome$/i.test(ic.purpose.trim())) continue; const url = toUrl(ic.src, manifestUrl); if (!url) continue; const svg = String(ic.type || '').includes('svg') || /\.svg(\?|$)/i.test(url); const px = svg ? 9999 : sizesPixels(ic.sizes); if (!best || px > best.px) best = { url, sizes: clean(ic.sizes, 100) || (svg ? 'any' : null), px }; } if (best && best.px >= 64 && iconPixels(page.icon) < 64) page.icon = best; if (!page.themeColor && typeof json.theme_color === 'string') page.themeColor = cleanColor(json.theme_color); if (!page.siteName) page.siteName = clean(typeof json.name === 'string' ? json.name : json.short_name, 200) || null; return page;}