Something went wrong. Try again.
🔥 🔥 🔥 A Free & Self-hostable Airtable Alternative nocodb.com
Something went wrong. Try again.
11 kB · 386 lines
JavaScript
at develop
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387import { spawn } from 'child_process';import { createHash } from 'crypto';import { existsSync, readdirSync, readFileSync, statSync, writeFileSync,} from 'fs';import { join, resolve } from 'path';import { cpus } from 'os';
const CHECKSUM_FILE = '.build-checksums.json';const ALL_PACKAGES = ['core', ...getPackages()];const BUILD_LEVELS = computeBuildLevels(ALL_PACKAGES);// Flatten levels → stable iteration order for checksum checksconst WORKSPACE_PACKAGES = BUILD_LEVELS.flat();
function getPackages() { try { const packagesDir = resolve('packages'); if (!existsSync(packagesDir)) { return []; } const packages = readdirSync(packagesDir).filter((name) => { const packagePath = join(packagesDir, name); return statSync(packagePath).isDirectory() && !name.startsWith('.'); }); return packages.map((name) => `packages/${name}`); } catch (error) { console.error('Error reading packages directory:', error.message); return []; }}
/** * Returns the @noco-integrations/* workspace dependency short-names for a package. * e.g. { "@noco-integrations/smtp-auth": "workspace:*" } → ["smtp-auth"] */function getWorkspaceDependencies(packagePath) { const pkgJsonPath = join(packagePath, 'package.json'); if (!existsSync(pkgJsonPath)) return []; try { const pkg = JSON.parse(readFileSync(pkgJsonPath, 'utf8')); const allDeps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) }; return Object.entries(allDeps) .filter(([name, version]) => name.startsWith('@noco-integrations/') && String(version).startsWith('workspace:'), ) .map(([name]) => name.replace('@noco-integrations/', '')); } catch { return []; }}
/** * Returns packages grouped into build levels via BFS. * All packages within a level have no unbuilt dependencies — they can be built * in parallel. Level N finishes completely before level N+1 starts. * * Example with 60 packages all depending only on core: * Level 0: [core] * Level 1: [all-60-packages] ← built in parallel */function computeBuildLevels(packages) { const nameToPath = new Map(); for (const pkg of packages) { const shortName = pkg === 'core' ? 'core' : pkg.replace('packages/', ''); nameToPath.set(shortName, pkg); }
const inDegree = new Map(); const dependents = new Map(); for (const pkg of packages) { inDegree.set(pkg, 0); dependents.set(pkg, []); } for (const pkg of packages) { for (const depName of getWorkspaceDependencies(pkg)) { const depPath = nameToPath.get(depName); if (depPath) { inDegree.set(pkg, inDegree.get(pkg) + 1); dependents.get(depPath).push(pkg); } } }
const levels = []; const placed = new Set(); let current = packages.filter((pkg) => inDegree.get(pkg) === 0); current.forEach((pkg) => placed.add(pkg));
while (current.length > 0) { levels.push([...current]); const next = []; for (const pkg of current) { for (const dep of dependents.get(pkg) || []) { inDegree.set(dep, inDegree.get(dep) - 1); if (inDegree.get(dep) === 0 && !placed.has(dep)) { next.push(dep); placed.add(dep); } } } current = next; }
// Any unplaced packages have a cycle — build them last with a warning const remaining = packages.filter((pkg) => !placed.has(pkg)); if (remaining.length > 0) { console.warn( `⚠️ Cycle detected, building last: ${remaining.join(', ')}`, ); levels.push(remaining); }
return levels;}
function getAllFiles(dir, files = []) { try { const items = readdirSync(dir); for (const item of items) { const fullPath = join(dir, item); const stat = statSync(fullPath); if (stat.isDirectory()) { if (item !== 'node_modules' && item !== 'dist' && !item.startsWith('.')) { getAllFiles(fullPath, files); } } else { files.push(fullPath); } } } catch { // Skip directories we can't read } return files;}
function calculateChecksum(packagePath) { try { if (!existsSync(packagePath)) { return null; }
const files = getAllFiles(packagePath); const hash = createHash('sha256'); files.sort();
for (const file of files) { try { const relativePath = file.replace(packagePath, '').replace(/\\/g, '/'); hash.update(relativePath); hash.update(readFileSync(file)); } catch { // Skip files we can't read } }
return hash.digest('hex'); } catch (error) { console.error(`Error calculating checksum for ${packagePath}:`, error.message); return null; }}
function loadChecksums() { if (!existsSync(CHECKSUM_FILE)) { return {}; } try { return JSON.parse(readFileSync(CHECKSUM_FILE, 'utf8')); } catch (error) { console.warn('Error reading checksum file, starting fresh:', error.message); return {}; }}
function saveChecksums(checksums) { try { writeFileSync(CHECKSUM_FILE, JSON.stringify(checksums, null, 2)); } catch (error) { console.error('Error saving checksums:', error.message); }}
function packageExists(packagePath) { return ( existsSync(packagePath) && existsSync(join(packagePath, 'package.json')) );}
function hasDistFolder(packagePath) { return existsSync(join(packagePath, 'dist'));}
/** * Builds a single package asynchronously. Output is buffered and printed * atomically when the process exits so parallel builds don't interleave. */function buildPackageAsync(packagePath) { return new Promise((resolve) => { const proc = spawn('pnpm', ['build'], { cwd: packagePath, shell: process.platform === 'win32', });
const chunks = []; proc.stdout?.on('data', (d) => chunks.push({ stream: 'out', d })); proc.stderr?.on('data', (d) => chunks.push({ stream: 'err', d }));
proc.on('close', (code) => { const success = code === 0; if (success) { console.log(` ✅ ${packagePath}`); } else { console.error(` ❌ ${packagePath} (exit ${code})`); if (chunks.length) { console.error(` --- output for ${packagePath} ---`); for (const { stream, d } of chunks) { (stream === 'out' ? process.stdout : process.stderr).write(d); } console.error(` --- end ${packagePath} ---`); } } resolve(success); });
proc.on('error', (err) => { console.error(` ❌ ${packagePath}: spawn error — ${err.message}`); resolve(false); }); });}
/** * Builds an array of packages in parallel, capped at maxConcurrent workers. * Returns { successCount, failCount }. */async function buildLevel(packages, maxConcurrent) { const queue = [...packages]; let successCount = 0; let failCount = 0;
async function worker() { while (queue.length > 0) { const pkg = queue.shift(); if (!pkg) break; if (await buildPackageAsync(pkg)) { successCount++; } else { failCount++; } } }
await Promise.all( Array.from({ length: Math.min(maxConcurrent, packages.length) }, worker), );
return { successCount, failCount };}
async function main() { const force = process.argv.includes('--force'); const verbose = process.argv.includes('--verbose'); const concurrencyArg = process.argv.find((a) => a.startsWith('--concurrency=')); const maxConcurrent = concurrencyArg ? Math.max(1, parseInt(concurrencyArg.split('=')[1], 10)) : cpus().length;
if (force) { console.log('🔄 Force rebuild requested, building all packages...'); }
const oldChecksums = loadChecksums(); const newChecksums = {}; const packagesNeedingBuild = [];
console.log('🔍 Checking package changes...');
for (const packagePath of WORKSPACE_PACKAGES) { if (!packageExists(packagePath)) { if (verbose) { console.log(`⚠️ Package ${packagePath} does not exist, skipping...`); } continue; }
const currentChecksum = calculateChecksum(packagePath); if (!currentChecksum) { console.log(`⚠️ Could not calculate checksum for ${packagePath}, skipping...`); continue; }
newChecksums[packagePath] = currentChecksum;
const hasChanged = oldChecksums[packagePath] !== currentChecksum; const missingDist = !hasDistFolder(packagePath);
if (force || hasChanged || missingDist) { const reason = force ? 'forced' : hasChanged ? 'changed' : 'missing dist'; if (verbose || hasChanged || missingDist) { console.log(`📋 ${packagePath} needs build (${reason})`); } packagesNeedingBuild.push(packagePath); } else if (verbose) { console.log(`✅ ${packagePath} unchanged, skipping build`); } }
if (packagesNeedingBuild.length === 0) { console.log('🎉 All packages are up to date, no builds needed!'); return; }
const packagesNeedingBuildSet = new Set(packagesNeedingBuild); console.log( `\n🚀 Building ${packagesNeedingBuild.length} package(s) (up to ${maxConcurrent} parallel)...`, );
let totalSuccess = 0; let totalFail = 0;
for (let i = 0; i < BUILD_LEVELS.length; i++) { const toBuild = BUILD_LEVELS[i].filter((pkg) => packagesNeedingBuildSet.has(pkg), ); if (toBuild.length === 0) continue;
console.log( `\n⚡ Level ${i} — ${toBuild.length} package(s) in parallel`, ); if (verbose) { console.log(` ${toBuild.join(', ')}`); }
const { successCount, failCount } = await buildLevel(toBuild, maxConcurrent); totalSuccess += successCount; totalFail += failCount;
if (failCount > 0) { console.error( `\n❌ ${failCount} package(s) failed in level ${i} — stopping build`, ); process.exit(1); } }
if (totalSuccess > 0) { saveChecksums(newChecksums); console.log(`\n✅ Successfully built ${totalSuccess} package(s)`); }
if (totalFail > 0) { console.log(`❌ Failed to build ${totalFail} package(s)`); process.exit(1); }
console.log('🎯 Build optimization complete!');}
if (process.argv.includes('--help')) { console.log(`Usage: node scripts/build-optimized.js [options]
Options: --force Force rebuild all packages regardless of changes --verbose Show detailed output for all packages --concurrency=N Max parallel builds per level (default: CPU count) --help Show this help message
Examples: node scripts/build-optimized.js # Build only changed packages node scripts/build-optimized.js --force # Force rebuild all node scripts/build-optimized.js --concurrency=4 # Limit to 4 parallel node scripts/build-optimized.js --verbose # Show per-package details`); process.exit(0);}
main().catch((error) => { console.error('❌ Build script failed:', error.message); process.exit(1);});