Something went wrong. Try again.
Migrating content using the Ghost API to Offprint / Standard Site on atproto
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209#!/usr/bin/env nodeimport { mkdir } from 'node:fs/promises';import { loadConfig } from './config.js';import { createLogger } from './logger.js';import { loadState, saveState, type AppState, type PostState } from './state.js';import { fetchAllPosts } from './ghost.js';import { collectImageUrls, convertHtmlToOffprint } from './html-to-offprint.js';import { getOrUploadImage, resolveUrl, type AssetState } from './assets.js';import { getSession, validateStandardDocument, createRecord, putRecord, rkeyFromUri, type StandardDocumentRecord, type AtprotoSession,} from './atproto.js';import type { BlobRef } from './types.js';
async function main() { const config = loadConfig(); const log = createLogger(config.verbose); log.info('starting GhostOff');
await mkdir(config.exportDir, { recursive: true }); const state = await loadState(config.stateFile); state.publicationAtUri = config.publicationAtUri;
const session = config.dryRun ? undefined : await getSession(config, log);
if (session) { state.accountDid = session.did; // The state file (cached blob refs and record rkeys) is pinned to a specific // account/PDS. If it belongs to a different account than the one we logged in // with, reusing it would make new records reference blobs that don't exist on // this account's PDS (putRecord "Could not find blob"). Treat it as a fresh // import instead of silently failing. const stateDids = new Set<string>(); for (const p of Object.values(state.posts)) { const m = p.documentUri?.match(/^at:\/\/(did:[^/]+)\//); if (m) stateDids.add(m[1]); } const hasState = Object.keys(state.posts).length > 0 || (state.assets != null && Object.keys(state.assets).length > 0); if (hasState && stateDids.size > 0 && !stateDids.has(session.did)) { log.warn( `state ${config.stateFile} belongs to account ${[...stateDids].join(', ')}, not ${session.did}; ` + 'treating this run as a fresh import (re-uploading assets, creating new records).' ); state.posts = {}; state.assets = {}; } }
let processed = 0; let failed = 0; let skipped = 0;
try { const posts = await fetchAllPosts({ ghostUrl: config.ghostUrl, ghostApiKey: config.ghostApiKey }); log.info(`fetched ${posts.length} posts`);
for (const post of posts) { const existing = state.posts[post.uuid]; try { if (post.visibility !== 'public') { log.info(`skipping non-public post: ${post.title}`); skipped++; continue; }
log.info(`processing: ${post.title}`); const assetMap = new Map<string, BlobRef>();
const inlineUrls = collectImageUrls(post.html, config.ghostUrl); if (post.feature_image) { inlineUrls.push(post.feature_image); }
if (!config.dryRun && session) { state.assets ??= {}; for (const url of new Set(inlineUrls)) { const info = await getOrUploadImage( url, config.ghostUrl, config.exportDir, state.assets as Record<string, AssetState>, session, log ); if (info) { assetMap.set(info.url, info.blob); } } await saveState(config.stateFile, state); } else if (config.dryRun) { const DUMMY_CID = 'bafyreiaouvhh2kl5s3jo2uaqnn23hlpl2kut2n2qsgfvme54umlro3iari'; for (const url of new Set(inlineUrls)) { assetMap.set(url, { $type: 'blob', ref: { $link: DUMMY_CID }, mimeType: 'image/jpeg', size: 1000 }); } }
const contentWrapper = await convertHtmlToOffprint(post.html, config.ghostUrl, assetMap, log);
const tags = (post.tags ?? []).map((t) => t.name); const description = post.custom_excerpt ?? post.excerpt ?? undefined;
const documentRecord: StandardDocumentRecord = { $type: 'site.standard.document', site: config.publicationAtUri, path: `/${post.slug}`, title: post.title, description, publishedAt: post.published_at, updatedAt: post.updated_at, textContent: post.plaintext ?? '', tags, content: { $type: 'app.offprint.content', items: contentWrapper.items }, };
if (post.feature_image) { const featureBlob = assetMap.get(resolveUrl(config.ghostUrl, post.feature_image)); if (featureBlob) { documentRecord.coverImage = featureBlob; } }
validateStandardDocument(documentRecord, log);
let documentRef; if (config.dryRun || !session) { log.info(`[dry-run] would create/update site.standard.document for "${post.title}"`); documentRef = { uri: 'at://dry-run/site.standard.document/dry', cid: 'dry-run' }; } else if (existing?.documentUri?.startsWith('at://did:') && existing.documentCid && existing.documentCid !== 'dry-run') { documentRef = await putRecord( session, 'site.standard.document', rkeyFromUri(existing.documentUri), documentRecord, existing.documentCid, log ); } else { documentRef = await createRecord(session, 'site.standard.document', documentRecord, log); }
const articleRecord = { $type: 'app.offprint.document.article', document: { $type: 'com.atproto.repo.strongRef', uri: documentRef.uri, cid: documentRef.cid, }, };
let articleRef; if (config.dryRun || !session) { log.info(`[dry-run] would create/update app.offprint.document.article for "${post.title}"`); articleRef = { uri: 'at://dry-run/app.offprint.document.article/dry', cid: 'dry-run' }; } else if (existing?.articleUri?.startsWith('at://did:') && existing.articleCid && existing.articleCid !== 'dry-run') { articleRef = await putRecord( session, 'app.offprint.document.article', rkeyFromUri(existing.articleUri), articleRecord, existing.articleCid, log ); } else { articleRef = await createRecord(session, 'app.offprint.document.article', articleRecord, log); }
state.posts[post.uuid] = { ghostSlug: post.slug, ghostTitle: post.title, documentUri: documentRef.uri, documentCid: documentRef.cid, articleUri: articleRef.uri, articleCid: articleRef.cid, }; if (!config.dryRun) { await saveState(config.stateFile, state); }
processed++; } catch (err: any) { log.error(`failed to process "${post.title ?? post.slug}": ${err?.message ?? err}`); log.error(err?.stack ?? ''); failed++; } }
log.info(`done: ${processed} processed, ${failed} failed, ${skipped} skipped`); } finally { if (!config.dryRun && session) { log.info('logging out'); await session.logout().catch((err: any) => { log.error(`logout failed: ${err?.message ?? err}`); }); } }
process.exit(failed > 0 ? 1 : 0);}
main().catch((err) => { console.error('[ghostoff] FATAL', err); process.exit(1);});