Something went wrong. Try again.
A nicer S3 CLI
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229import { after, before, test } from 'node:test';import * as assert from 'node:assert/strict';import * as FS from 'node:fs';import * as OS from 'node:os';import * as PATH from 'node:path';import { spawn } from 'node:child_process';import { createHash } from 'node:crypto';import { createZipArchive, ZipEntry, ZipFile } from 'node:zlib';import { startFakeS3, type FakeS3 } from './fake-s3.ts';
// s3cli as people and agents run it — a process, its stdout, stderr and exit// code — against a fake S3 in this process. (The process is spawned, not run// with spawnSync: that would block the fake from answering it.)
const MAIN = PATH.resolve(import.meta.dirname, '..', 'src', 'main.ts');let s3: FakeS3;let home: string;let work: string;
interface Run { code: number; stdout: string; stderr: string;}
function run(args: string[], options: { input?: string | Buffer; env?: Record<string, string> } = {}): Promise<Run> { return new Promise((resolve, reject) => { const child = spawn(process.execPath, ['--no-warnings', MAIN, ...args], { cwd: work, env: { ...process.env, S3CLI_CONFIG_DIR: PATH.join(home, 'config'), NO_COLOR: '1', ...options.env }, }); const out: Buffer[] = []; const err: Buffer[] = []; child.stdout.on('data', (chunk: Buffer) => out.push(chunk)); child.stderr.on('data', (chunk: Buffer) => err.push(chunk)); child.on('error', reject); child.on('close', (code) => resolve({ code: code ?? -1, stdout: Buffer.concat(out).toString(), stderr: Buffer.concat(err).toString() })); child.stdin.end(options.input ?? ''); });}
async function ok(args: string[], options?: Parameters<typeof run>[1]): Promise<Run> { const result = await run(args, options); assert.equal(result.code, 0, `s3cli ${args.join(' ')} exited ${result.code}:\n${result.stderr}`); return result;}
const stored = (bucket: string, key: string) => s3.buckets.get(bucket)?.get(key)?.body;
before(async () => { s3 = await startFakeS3({ pageSize: 3 }); home = FS.mkdtempSync(PATH.join(OS.tmpdir(), 's3cli-e2e-')); work = PATH.join(home, 'work'); FS.mkdirSync(work); FS.mkdirSync(PATH.join(home, 'config')); FS.writeFileSync(PATH.join(home, 'config', 'config.json'), JSON.stringify({ profiles: {}, buckets: {}, settings: { 'secret-store': 'file', 'part-size': '5' } })); s3.buckets.set('alpha', new Map()); s3.buckets.set('beta', new Map());});
after(async () => { await s3.close(); FS.rmSync(home, { recursive: true, force: true });});
test('without a profile, a command says how to get one, exit 3', async () => { const result = await run(['ls', 'alpha/']); assert.equal(result.code, 3); assert.match(result.stderr, /s3cli login/);});
test('login tries the keys, keeps them in the secret store, and remembers buckets', async () => { const env = { AWS_ACCESS_KEY_ID: 'AKIDEXAMPLE', AWS_SECRET_ACCESS_KEY: 'secret', AWS_ENDPOINT_URL: s3.endpoint, AWS_REGION: 'us-east-1' }; const result = await ok(['login', 'fake', '--from-env', '--all-buckets', '--json'], { env }); const data = JSON.parse(result.stdout); assert.deepEqual(data.buckets, ['alpha', 'beta']); assert.equal(data.default, true); const config = JSON.parse(FS.readFileSync(PATH.join(home, 'config', 'config.json'), 'utf-8')); assert.equal(config.profiles.fake.endpoint, s3.endpoint); assert.equal(config.profiles.fake.pathStyle, true); assert.ok(!FS.readFileSync(PATH.join(home, 'config', 'config.json'), 'utf-8').includes('"secret"'), 'the secret key is not in config.json'); const secrets = JSON.parse(FS.readFileSync(PATH.join(home, 'config', 'secrets.json'), 'utf-8')); assert.ok(secrets['profile:fake']); const listed = JSON.parse((await ok(['ls', '--json'])).stdout); assert.deepEqual(listed.map((entry: { name: string }) => entry.name), ['alpha', 'beta']);});
test('put, ls, stat, cat and get', async () => { FS.writeFileSync(PATH.join(work, 'hello.txt'), 'hello, world\n'); await ok(['put', 'hello.txt', 'alpha/docs/']); assert.equal(stored('alpha', 'docs/hello.txt')?.toString(), 'hello, world\n'); assert.equal(s3.buckets.get('alpha')?.get('docs/hello.txt')?.type, 'text/plain; charset=utf-8'); const ls = JSON.parse((await ok(['ls', 'alpha', '--json'])).stdout); assert.deepEqual(ls.map((entry: { name: string; type: string }) => `${entry.type}:${entry.name}`), ['folder:docs']); const stat = JSON.parse((await ok(['stat', 's3://alpha/docs/hello.txt', '--json'])).stdout); assert.equal(stat.size, 13); assert.equal((await ok(['cat', 'alpha/docs/hello.txt'])).stdout, 'hello, world\n'); await ok(['get', 'alpha/docs/hello.txt', 'copy.txt']); assert.equal(FS.readFileSync(PATH.join(work, 'copy.txt'), 'utf-8'), 'hello, world\n'); const missing = await run(['stat', 'alpha/docs/nope.txt']); assert.equal(missing.code, 4);});
test('stdin goes up, and a file larger than a part goes up in parts', async () => { await ok(['put', '-', 'alpha/stdin.txt'], { input: 'from stdin' }); assert.equal(stored('alpha', 'stdin.txt')?.toString(), 'from stdin'); const big = Buffer.alloc(12 * 1024 * 1024); for (let i = 0; i < big.length; i += 4096) big.writeUInt32LE(i, i); FS.writeFileSync(PATH.join(work, 'big.bin'), big); await ok(['put', 'big.bin', 'alpha/big.bin']); assert.match(s3.buckets.get('alpha')?.get('big.bin')?.etag ?? '', /-3$/); assert.ok(stored('alpha', 'big.bin')?.equals(big)); await ok(['get', 'alpha/big.bin', 'big-back.bin']); assert.equal(createHash('sha256').update(FS.readFileSync(PATH.join(work, 'big-back.bin'))).digest('hex'), createHash('sha256').update(big).digest('hex'));});
test('folders: put -r, ls through pages, get -r', async () => { const site = PATH.join(work, 'site'); FS.mkdirSync(PATH.join(site, 'css'), { recursive: true }); for (const name of ['a.html', 'b.html', 'c.html', 'd.html', 'css/x.css']) FS.writeFileSync(PATH.join(site, name), name); await ok(['put', '-r', 'site', 'beta/']); // Five entries at three a page: the listing must follow the continuation. const ls = JSON.parse((await ok(['ls', 'beta/site/', '--json'])).stdout); assert.deepEqual(ls.map((entry: { name: string }) => entry.name).sort(), ['a.html', 'b.html', 'c.html', 'css', 'd.html']); await ok(['get', '-r', 'beta/site/', 'restored']); assert.equal(FS.readFileSync(PATH.join(work, 'restored', 'css', 'x.css'), 'utf-8'), 'css/x.css'); const folder = await run(['get', 'beta/site/']); assert.equal(folder.code, 2); assert.match(folder.stderr, /-r/);});
test('get -r refuses a key that would land outside the folder, before writing anything', async () => { const now = { type: 'text/plain', modified: new Date(), etag: 'x' }; s3.buckets.get('beta')!.set('evil/fine.txt', { ...now, body: Buffer.from('fine') }); s3.buckets.get('beta')!.set('evil/../../escaped.txt', { ...now, body: Buffer.from('bad') }); const result = await run(['get', '-r', 'beta/evil/', 'evil-out']); assert.equal(result.code, 2); assert.match(result.stderr, /refusing the key/); assert.ok(!FS.existsSync(PATH.join(work, 'evil-out', 'fine.txt'))); assert.ok(!FS.existsSync(PATH.join(home, 'escaped.txt'))); s3.buckets.get('beta')!.delete('evil/../../escaped.txt'); s3.buckets.get('beta')!.delete('evil/fine.txt');});
test('get --zip streams a folder into an archive, put --unzip unpacks one', async () => { await ok(['get', '--zip', 'beta/site/', 'site.zip']); const zip = await ZipFile.open(PATH.join(work, 'site.zip')); const names = [...zip.entriesSync()].map(([name]) => name).sort(); await zip.close(); assert.deepEqual(names, ['a.html', 'b.html', 'c.html', 'css/x.css', 'd.html']); await ok(['put', '--unzip', 'site.zip', 'alpha/unpacked/']); assert.equal(stored('alpha', 'unpacked/css/x.css')?.toString(), 'css/x.css'); assert.equal(stored('alpha', 'unpacked/a.html')?.toString(), 'a.html');});
test('put --unzip refuses an archive that reaches outside the folder, uploading nothing', async () => { const chunks: Buffer[] = []; for await (const chunk of createZipArchive([ await ZipEntry.create('ok.txt', Buffer.from('ok')), await ZipEntry.create('../escape.txt', Buffer.from('x')), ])) chunks.push(chunk as Buffer); FS.writeFileSync(PATH.join(work, 'evil.zip'), Buffer.concat(chunks)); const result = await run(['put', '--unzip', 'evil.zip', 'alpha/evil/']); assert.equal(result.code, 2); assert.match(result.stderr, /nothing was uploaded/); assert.ok(![...s3.buckets.get('alpha')!.keys()].some((key) => key.includes('evil') || key.includes('escape')));});
test('cp and mv, in a bucket and between buckets', async () => { await ok(['cp', 'alpha/docs/hello.txt', 'alpha/docs/hello-copy.txt']); assert.equal(stored('alpha', 'docs/hello-copy.txt')?.toString(), 'hello, world\n'); await ok(['cp', 'alpha/docs/hello.txt', 'beta/']); assert.equal(stored('beta', 'hello.txt')?.toString(), 'hello, world\n'); await ok(['mv', 'alpha/docs/hello-copy.txt', 'alpha/moved/']); assert.equal(stored('alpha', 'docs/hello-copy.txt'), undefined); assert.equal(stored('alpha', 'moved/hello-copy.txt')?.toString(), 'hello, world\n'); await ok(['cp', '-r', 'beta/site/', 'alpha/site-copy/']); assert.equal(stored('alpha', 'site-copy/site/css/x.css')?.toString(), 'css/x.css');});
test('rm takes -r for folders, and asks (or wants --yes) before deleting many', async () => { const folder = await run(['rm', 'alpha/site-copy/']); assert.equal(folder.code, 2); assert.match(folder.stderr, /add -r/); const many = await run(['rm', '-r', 'alpha/site-copy/']); assert.equal(many.code, 2); assert.match(many.stderr, /--yes/); await ok(['rm', '-r', 'alpha/site-copy/', '--yes']); assert.ok(![...s3.buckets.get('alpha')!.keys()].some((key) => key.startsWith('site-copy/'))); await ok(['rm', 'alpha/moved/hello-copy.txt']); assert.equal(stored('alpha', 'moved/hello-copy.txt'), undefined); await ok(['rm', 'alpha/unpacked/*.html', '--yes']); assert.ok(![...s3.buckets.get('alpha')!.keys()].some((key) => /^unpacked\/[^/]+\.html$/.test(key)));});
test('mkdir and rmdir: folders, and at / buckets', async () => { await ok(['mkdir', 'alpha/empty']); assert.ok(s3.buckets.get('alpha')!.has('empty/')); await ok(['rmdir', 'alpha/empty']); assert.ok(!s3.buckets.get('alpha')!.has('empty/')); await ok(['mkdir', '/gamma']); assert.ok(s3.buckets.has('gamma')); assert.ok(JSON.parse((await ok(['bucket', 'list', '--json'])).stdout).some((entry: { name: string }) => entry.name === 'gamma')); await ok(['rmdir', '/gamma', '--yes']); assert.ok(!s3.buckets.has('gamma'));});
test('the shell reads a script from stdin, and stops at the first failure', async () => { const good = await ok(['shell'], { input: 'cd /alpha/docs\npwd\nls\n-cd nowhere\ncat hello.txt\n' }); assert.match(good.stdout, /\/alpha\/docs\//); assert.match(good.stdout, /hello, world/); const bad = await run(['shell'], { input: 'cd /alpha\nget nope.txt\npwd\n' }); assert.equal(bad.code, 4); assert.match(bad.stderr, /stopped at line 2/); assert.doesNotMatch(bad.stdout, /\/alpha\//);});
test('Tab: the shells get commands, and the folders and files in a bucket', async () => { const commands = await ok(['__complete', '--shell', 'fish', '--', 'mk']); assert.match(commands.stdout, /^mkdir\t/m); const remote = await ok(['__complete', '--shell', 'fish', '--', 'ls', 'alpha/d']); assert.deepEqual(remote.stdout.trim().split('\n'), ['alpha/docs/']); const bash = await ok(['__complete', '--shell', 'bash'], { env: { COMP_LINE: 's3cli get alpha/docs/he', COMP_POINT: '23' } }); assert.equal(bash.stdout, 'alpha/docs/hello.txt \n'); const files = await ok(['__complete', '--shell', 'fish', '--', 'put', '']); assert.equal(files.stdout.trim(), ':files'); for (const shell of ['bash', 'zsh', 'fish', 'powershell']) assert.match((await ok(['completion', shell])).stdout, /__complete/);});