Something went wrong. Try again.
Migrating content using the Ghost API to Offprint / Standard Site on atproto
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216import { createHash } from 'node:crypto';import { mkdir } from 'node:fs/promises';import { dirname, extname, join } from 'node:path';import sharp from 'sharp';import type { BlobRef } from './types.js';import type { Logger } from './logger.js';import { writeFile, readFile, stat } from 'node:fs/promises';import { sleep, backoffDelay } from './rate-limit.js';import type { AtprotoSession } from './atproto.js';
export interface AssetInfo { url: string; localPath: string; blob: BlobRef; width?: number; height?: number;}
export interface AssetState { url: string; blob: BlobRef; width?: number; height?: number;}
const BLOB_SIZE_LIMIT = 900_000;
export function resolveUrl(base: string, url: string): string { if (!url) return url; if (url.startsWith('http://') || url.startsWith('https://') || url.startsWith('//')) { return url.startsWith('//') ? `https:${url}` : url; } return new URL(url, base).toString();}
function assetFilePath(exportDir: string, url: string): { path: string; ext: string } { let ext = extname(new URL(url, 'https://example.com').pathname).toLowerCase(); if (!ext || ext.length > 5) ext = '.bin'; if (!['.jpg', '.jpeg', '.png', '.gif', '.webp', '.avif'].includes(ext)) { ext = '.bin'; } const hash = createHash('sha256').update(url).digest('hex'); return { path: join(exportDir, 'assets', `${hash}${ext}`), ext };}
function guessMimeType(path: string): string { const ext = extname(path).toLowerCase(); switch (ext) { case '.jpg': case '.jpeg': return 'image/jpeg'; case '.png': return 'image/png'; case '.gif': return 'image/gif'; case '.webp': return 'image/webp'; case '.avif': return 'image/avif'; default: return 'image/jpeg'; }}
export async function downloadImage(url: string, localPath: string, log: Logger): Promise<Buffer> { try { await stat(localPath); log.debug(`using cached image: ${url}`); return readFile(localPath); } catch { // not cached }
const response = await fetch(url, { headers: { 'User-Agent': 'GhostOff/0.1.0' }, }); if (!response.ok) { throw new Error(`download failed: ${response.status} ${response.statusText}`); } const buffer = Buffer.from(await response.arrayBuffer()); await mkdir(dirname(localPath), { recursive: true }); await writeFile(localPath, buffer); log.debug(`downloaded image: ${url} (${buffer.length} bytes)`); return buffer;}
async function processImageBuffer(buffer: Buffer, mimeType: string, log: Logger): Promise<{ buffer: Buffer; width: number; height: number; mimeType: string }> { let image = sharp(buffer); const metadata = await image.metadata(); let width = metadata.width ?? 1; let height = metadata.height ?? 1;
if (width > 2048 || height > 2048) { image = image.resize({ width: 2048, height: 2048, fit: 'inside', withoutEnlargement: true }); }
const targetFormat = mimeType === 'image/png' && buffer.length <= BLOB_SIZE_LIMIT ? 'png' : 'jpeg';
for (let attempt = 0; attempt < 10; attempt++) { let output = image.clone(); if (targetFormat === 'jpeg') { const quality = Math.max(40, 90 - attempt * 6); output = output.jpeg({ quality, progressive: true, force: true }); } else { output = output.png({ compressionLevel: 9, force: true }); }
const processed = await output.toBuffer({ resolveWithObject: true }); if (processed.data.length <= BLOB_SIZE_LIMIT || attempt === 9) { return { buffer: processed.data, width: processed.info.width, height: processed.info.height, mimeType: targetFormat === 'jpeg' ? 'image/jpeg' : 'image/png', }; }
image = image.resize({ width: Math.max(400, Math.round(width * 0.7)), height: Math.max(400, Math.round(height * 0.7)), fit: 'inside' }); const nextMeta = await image.metadata(); width = nextMeta.width ?? width; height = nextMeta.height ?? height; }
throw new Error('unable to compress image under blob limit');}
export async function getOrUploadImage( url: string, base: string, exportDir: string, stateAssets: Record<string, AssetState>, session: AtprotoSession, log: Logger): Promise<AssetInfo | undefined> { const resolved = resolveUrl(base, url);
// Skip formats that image processors can't handle (e.g. ICO favicons) const ext = extname(new URL(resolved, 'https://example.com').pathname).toLowerCase(); if (ext === '.ico') { log.debug(`skipping unsupported image format: ${resolved}`); return undefined; }
const cached = stateAssets[resolved]; if (cached) { log.debug(`reusing uploaded blob for ${resolved}`); return { url: resolved, localPath: '', blob: cached.blob, width: cached.width, height: cached.height, }; }
const { path: localPath } = assetFilePath(exportDir, resolved); let rawBuffer: Buffer; try { rawBuffer = await downloadImage(resolved, localPath, log); } catch (err: any) { log.error(`image download failed (${resolved}): ${err?.message ?? err}`); return undefined; }
if (rawBuffer.length === 0) { log.warn(`empty image: ${resolved}`); return undefined; }
let processed: { buffer: Buffer; width: number; height: number; mimeType: string }; try { const sourceMime = guessMimeType(localPath); processed = await processImageBuffer(rawBuffer, sourceMime, log); } catch (err: any) { log.error(`image processing failed (${resolved}): ${err?.message ?? err}`); return undefined; }
log.debug(`uploading blob for ${resolved} (${processed.buffer.length} bytes)`);
let blobRef: BlobRef | undefined; for (let attempt = 0; attempt < 3; attempt++) { try { blobRef = await session.uploadBlob(processed.buffer, processed.mimeType, log); break; } catch (err: any) { log.warn(`blob upload attempt ${attempt + 1} failed (${resolved}): ${err?.message ?? err}`); if (attempt < 2) { await sleep(backoffDelay(attempt)); continue; } log.error(`blob upload failed (${resolved}): ${err?.message ?? err}`); return undefined; } }
if (!blobRef) { return undefined; }
stateAssets[resolved] = { url: resolved, blob: blobRef, width: processed.width, height: processed.height, };
return { url: resolved, localPath, blob: blobRef, width: processed.width, height: processed.height, };}