Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285import { describe, expect, it } from "vitest";import { formatReport, runScript, type EngineDeps } from "../../runner/engine";import { parseScript } from "../../runner/parser";import type { Script } from "../../runner/parser";
const script = (text: string): Script => { const r = parseScript(text); if (!r.ok) throw new Error(JSON.stringify(r.errors)); return r.script;};
type Body = { cmd: string; [k: string]: unknown };type Answer = Record<string, unknown>;
/** A fake bridge: answers by command name from a table, records every request. */function fake(answers: Record<string, Answer | ((b: Body) => Answer)> = {}, extra: Partial<EngineDeps> = {}) { const sent: Array<{ body: Body; timeoutMs: number | undefined }> = []; let t = 0; const deps: EngineDeps = { request: async (body, opts) => { sent.push({ body, timeoutMs: opts?.timeoutMs }); t += 10; const a = answers[body.cmd] ?? { ok: true }; return { id: sent.length, ...(typeof a === "function" ? a(body) : a) } as never; }, now: () => t, sleep: async () => undefined, ...extra, }; return { deps, sent };}
const hello = `test "say hello" start http://127.0.0.1:4780/hello.html step "click the button" click role:button name:"Say hello" step "the page greets us" verify text:"Hello, world"`;
describe("runScript", () => { it("opens the start page, then resolves and clicks, then verifies, in order", async () => { const { deps, sent } = fake({ navigate: { ok: true, tab: 3 }, resolve: { ok: true, descriptor: { tag: "button" } } }); const r = await runScript(script(hello), deps); expect(r.ok).toBe(true); expect(sent.map((s) => s.body.cmd)).toEqual(["navigate", "resolve", "act", "verify"]); expect(sent[0]!.body).toMatchObject({ url: "http://127.0.0.1:4780/hello.html" }); expect(sent[1]!.body).toMatchObject({ target: { role: "button", name: "Say hello" } }); expect(sent[2]!.body).toMatchObject({ action: "click", descriptor: { tag: "button" } }); expect(sent[3]!.body).toMatchObject({ kind: "text", value: "Hello, world" }); }); it("sends the tab that navigate opened with every later command", async () => { const { deps, sent } = fake({ navigate: { ok: true, tab: 41 }, resolve: { ok: true, descriptor: {} } }); await runScript(script(hello), deps); expect(sent.slice(1).every((s) => s.body.tab === 41)).toBe(true); }); it("reports one result per step, with its duration", async () => { const { deps } = fake({ resolve: { ok: true, descriptor: {} } }); const r = await runScript(script(hello), deps); expect(r.steps.map((s) => [s.name, s.status])).toEqual([["open http://127.0.0.1:4780/hello.html", "passed"], ["click the button", "passed"], ["the page greets us", "passed"]]); expect(r.steps.every((s) => s.durationMs >= 0)).toBe(true); }); it("without a start line there is no opening step", async () => { const { deps, sent } = fake({ resolve: { ok: true, descriptor: {} } }); await runScript(script('test "t"\nstep "s"\n verify text:"x"'), deps); expect(sent.map((s) => s.body.cmd)).toEqual(["verify"]); }); it("a failed verify fails the step with the reason and what was found (mismatch, P1-3.b)", async () => { const { deps } = fake({ resolve: { ok: true, descriptor: {} }, verify: { ok: false, reason: "mismatch", detail: { expected: "Hello, world", found: "Hello, wolrd", nearby: [] } } }); const r = await runScript(script(hello), deps); expect(r.ok).toBe(false); const failed = r.steps.find((s) => s.status === "failed")!; expect(failed.name).toBe("the page greets us"); expect(failed.failure).toMatchObject({ line: 7, verb: "verify", reason: "mismatch", detail: { expected: "Hello, world", found: "Hello, wolrd" } }); }); it("stops at the first failing step and marks the rest skipped, without sending their commands", async () => { const { deps, sent } = fake({ resolve: { ok: false, reason: "not-found", detail: { target: { text: "x" } } } }); const r = await runScript(script(hello), deps); expect(r.steps.map((s) => s.status)).toEqual(["passed", "failed", "skipped"]); expect(sent.map((s) => s.body.cmd)).toEqual(["navigate", "resolve"]); }); it("a failed click (act) fails the step and does not go on to the next command of the step", async () => { const { deps, sent } = fake({ resolve: { ok: true, descriptor: {} }, act: { ok: false, reason: "not-enabled", detail: {} } }); const r = await runScript(script('test "t"\nstep "s"\n click text:"a"\n verify text:"b"'), deps); expect(r.steps[0]!.failure).toMatchObject({ reason: "not-enabled", verb: "click", line: 3 }); expect(sent.map((s) => s.body.cmd)).toEqual(["resolve", "act"]); }); it("a failed opening page fails the test before any step", async () => { const { deps, sent } = fake({ navigate: { ok: false, reason: "timeout", detail: { url: "u" } } }); const r = await runScript(script(hello), deps); expect(r.steps.map((s) => s.status)).toEqual(["failed", "skipped", "skipped"]); expect(sent).toHaveLength(1); }); it("uses the script's timeout, a command's own timeout when it has one, and 10 s otherwise", async () => { const a = fake({ resolve: { ok: true, descriptor: {} } }); await runScript(script('test "t"\ntimeout 3s\nstep "s"\n click text:"a"\n click text:"b" timeout:1s\n verify text:"c"'), a.deps); expect(a.sent.filter((s) => s.body.cmd === "resolve").map((s) => s.body.timeoutMs)).toEqual([3000, 1000]); expect(a.sent.find((s) => s.body.cmd === "verify")!.body.timeoutMs).toBe(3000); const b = fake({ resolve: { ok: true, descriptor: {} } }); await runScript(script('test "t"\nstep "s"\n click text:"a"'), b.deps); expect(b.sent[0]!.body.timeoutMs).toBe(10_000); }); it("a page load is given at least 10 s however short the script's timeout: that timeout is for elements, not for how fast a page loads", async () => { const short = fake({ navigate: { ok: true, tab: 1 }, resolve: { ok: true, descriptor: {} } }); await runScript(script('test "t"\nstart http://127.0.0.1:4780/x.html\ntimeout 700ms\nstep "s"\n goto http://127.0.0.1:4780/y.html\n click text:"a"'), short.deps); expect(short.sent.filter((s) => s.body.cmd === "navigate").map((s) => s.body.timeoutMs)).toEqual([10_000, 10_000]); expect(short.sent.find((s) => s.body.cmd === "resolve")!.body.timeoutMs).toBe(700); // elements still wait only as long as the script says }); it("a script that asks for longer gets longer for page loads too, and a timeout on the goto itself is exact", async () => { const long = fake({ navigate: { ok: true, tab: 1 } }); await runScript(script('test "t"\nstart http://127.0.0.1:4780/x.html\ntimeout 20s\nstep "s"\n goto http://127.0.0.1:4780/y.html'), long.deps); expect(long.sent.filter((s) => s.body.cmd === "navigate").map((s) => s.body.timeoutMs)).toEqual([20_000, 20_000]); const exact = fake({ navigate: { ok: true, tab: 1 } }); await runScript(script('test "t"\nstart http://127.0.0.1:4780/x.html\ntimeout 700ms\nstep "s"\n goto http://127.0.0.1:4780/y.html timeout:2s'), exact.deps); expect(exact.sent.filter((s) => s.body.cmd === "navigate").map((s) => s.body.timeoutMs)).toEqual([10_000, 2000]); }); it("gives the bridge a little longer than the page is told to wait, so the page's own answer wins", async () => { const { deps, sent } = fake({ resolve: { ok: true, descriptor: {} } }); await runScript(script('test "t"\nstep "s"\n click text:"a" timeout:2s'), deps); expect(sent[0]!.timeoutMs).toBeGreaterThan(2000); }); it("turns a bridge timeout or lost connection into a failed step", async () => { const { deps } = fake({ resolve: { ok: false, reason: "connection-lost", detail: { why: "the connection dropped" } } }); const r = await runScript(script('test "t"\nstep "s"\n click text:"a"'), deps); expect(r.ok).toBe(false); expect(r.steps[0]!.failure).toMatchObject({ reason: "connection-lost" }); }); it("an answer with no ok at all counts as a failure", async () => { const { deps } = fake({ resolve: { weird: true } as never }); const r = await runScript(script('test "t"\nstep "s"\n click text:"a"'), deps); expect(r.ok).toBe(false); expect(r.steps[0]!.failure).toMatchObject({ reason: "internal" }); }); it("a resolve answer without a descriptor is not clicked", async () => { const { deps, sent } = fake({ resolve: { ok: true } }); const r = await runScript(script('test "t"\nstep "s"\n click text:"a"'), deps); expect(r.ok).toBe(false); expect(sent.map((s) => s.body.cmd)).toEqual(["resolve"]); }); it("records the total duration and the test name", async () => { const { deps } = fake({ resolve: { ok: true, descriptor: {} } }); const r = await runScript(script(hello), deps); expect(r.name).toBe("say hello"); expect(r.durationMs).toBeGreaterThan(0); });});
describe("screenshots after each step", () => { const saved: Array<{ path: string; png: string }> = []; const save = async (path: string, png: string) => void saved.push({ path, png }); const withShots = (answers: Parameters<typeof fake>[0] = {}, extra: Partial<EngineDeps> = {}) => { saved.length = 0; return fake({ resolve: { ok: true, descriptor: {} }, screenshot: { ok: true, png: "UE5H", width: 10, height: 10 }, ...answers }, { saveScreenshot: save, ...extra }); };
it("takes one after the opening step and after every step that ran, and saves each under shots/", async () => { const f = withShots(); const r = await runScript(script(hello), f.deps); expect(saved.map((s) => s.path)).toEqual(["shots/000-01.png", "shots/001-01.png", "shots/002-01.png"]); expect(saved.every((s) => s.png === "UE5H")).toBe(true); expect(r.steps.map((s) => s.screenshot)).toEqual(["shots/000-01.png", "shots/001-01.png", "shots/002-01.png"]); }); it("asks for the screenshot after the step's commands, for the run's tab", async () => { const f = withShots({ navigate: { ok: true, tab: 9 } }); await runScript(script(hello), f.deps); expect(f.sent.map((s) => s.body.cmd)).toEqual(["navigate", "screenshot", "resolve", "act", "screenshot", "verify", "screenshot"]); expect(f.sent.filter((s) => s.body.cmd === "screenshot").every((s) => s.body.tab === 9)).toBe(true); }); it("names the file by step number and by how many commands the step ran", async () => { const f = withShots(); await runScript(script('test "t"\nstep "a"\n click text:"x"\n verify text:"y"\n click text:"z"'), f.deps); expect(saved.map((s) => s.path)).toEqual(["shots/000-03.png"]); }); it("takes one after a failed step too (what the page looked like when it failed), counting the failed command", async () => { const f = withShots({ verify: { ok: false, reason: "mismatch", detail: {} } }); const r = await runScript(script(hello), f.deps); expect(r.steps[2]).toMatchObject({ status: "failed", screenshot: "shots/002-01.png" }); }); it("takes none for a step that was skipped", async () => { const f = withShots({ resolve: { ok: false, reason: "not-found", detail: {} } }); const r = await runScript(script(hello), f.deps); expect(r.steps[2]).toMatchObject({ status: "skipped" }); expect(r.steps[2]!.screenshot).toBeUndefined(); expect(saved.map((s) => s.path)).toEqual(["shots/000-01.png", "shots/001-01.png"]); }); it("takes none at all when it was given nowhere to save them", async () => { const f = fake({ resolve: { ok: true, descriptor: {} } }); await runScript(script(hello), f.deps); expect(f.sent.some((s) => s.body.cmd === "screenshot")).toBe(false); }); it("numbers from 0 without an opening step too", async () => { const f = withShots(); await runScript(script('test "t"\nstep "s"\n verify text:"x"'), f.deps); expect(saved.map((s) => s.path)).toEqual(["shots/000-01.png"]); }); it.each([ ["Chrome refuses", { screenshot: { ok: false, reason: "internal", detail: { message: "permission is required" } } }, "permission is required"], ["the answer has no picture", { screenshot: { ok: true } }, "no picture"], ["the picture is not text", { screenshot: { ok: true, png: 5 } }, "no picture"], ])("a screenshot that fails because %s is a warning on the step, never a failure of the run", async (_n, answers, words) => { const f = withShots(answers as never); const r = await runScript(script(hello), f.deps); expect(r.ok).toBe(true); expect(r.steps.every((s) => s.status === "passed")).toBe(true); expect(r.steps[0]!.screenshot).toBeUndefined(); expect(r.steps[0]!.warnings[0]!.message).toContain(words); }); it("a save that throws is a warning too", async () => { const f = withShots({}, { saveScreenshot: async () => { throw new Error("disk full"); } }); const r = await runScript(script(hello), f.deps); expect(r.ok).toBe(true); expect(r.steps[0]!.warnings[0]!.message).toContain("disk full"); }); it("a request that throws is a warning too", async () => { let n = 0; const f = withShots({}, { request: async (b) => { if (b.cmd === "screenshot") throw new Error("bridge hiccup"); return { id: ++n, ok: true, descriptor: {}, tab: 1 } as never; } }); const r = await runScript(script(hello), f.deps); expect(r.ok).toBe(true); expect(r.steps[0]!.warnings[0]!.message).toContain("bridge hiccup"); }); it("gives the picture its own timeout, shorter than a command's", async () => { const f = withShots(); await runScript(script(hello), f.deps); const shot = f.sent.find((s) => s.body.cmd === "screenshot")!; expect(shot.body.timeoutMs).toBe(10_000); expect(shot.timeoutMs).toBeGreaterThan(10_000); });});
describe("formatReport", () => { it("lists a step's screenshot and warning under it", async () => { const r = await runScript(script(hello), fake({ resolve: { ok: true, descriptor: {} }, screenshot: { ok: true, png: "UE5H" } }, { saveScreenshot: async () => undefined }).deps); expect(formatReport(r, "hello.aite")).toContain(" screenshot: shots/001-01.png"); const w = await runScript(script(hello), fake({ resolve: { ok: true, descriptor: {} }, screenshot: { ok: false, reason: "internal", detail: { message: "nope" } } }, { saveScreenshot: async () => undefined }).deps); expect(formatReport(w, "hello.aite")).toContain(" warning: could not take a screenshot: nope"); });});
describe("formatReport (steps)", () => { const run = async (answers: Parameters<typeof fake>[0]) => runScript(script(hello), fake({ resolve: { ok: true, descriptor: {} }, ...answers }).deps);
it("lists each step and a summary for a passing test", async () => { const out = formatReport(await run({}), "hello.aite"); expect(out).toContain('test "say hello" (hello.aite)'); expect(out).toContain(" ok click the button"); expect(out).toContain(" ok the page greets us"); expect(out).toMatch(/3 passed, 0 failed, 0 skipped/); }); it("for a mismatch says what was expected and what was found, with the line", async () => { const out = formatReport(await run({ verify: { ok: false, reason: "mismatch", detail: { expected: "Hello, world", found: "Hello, wolrd", nearby: ["Waiting"] } } }), "hello.aite"); expect(out).toContain(" FAIL the page greets us"); expect(out).toContain('hello.aite:7: verify text:"Hello, world" failed (mismatch)'); expect(out).toContain('expected "Hello, world"'); expect(out).toContain('found "Hello, wolrd"'); expect(out).toContain('also on the page: "Waiting"'); expect(out).toMatch(/2 passed, 1 failed, 0 skipped/); }); it("for a click that found nothing names the target", async () => { const r = await runScript(script('test "t"\nstep "s"\n click role:button name:"Go"'), fake({ resolve: { ok: false, reason: "not-found", detail: { target: { role: "button", name: "Go" }, waitedMs: 2000 } } }).deps); const out = formatReport(r, "t.aite"); expect(out).toContain('t.aite:3: click role:button name:"Go" failed (not-found)'); expect(out).toContain("no element matched"); }); it("for an ambiguous target lists the candidates", async () => { const r = await runScript(script('test "t"\nstep "s"\n click text:"Dup"'), fake({ resolve: { ok: false, reason: "ambiguous", detail: { candidates: [{ xpath: "/html/body/button[1]", text: "Dup" }, { xpath: "/html/body/button[2]", text: "Dup" }] } } }).deps); const out = formatReport(r, "t.aite"); expect(out).toContain("2 elements matched"); expect(out).toContain("/html/body/button[1]"); expect(out).toContain("/html/body/button[2]"); }); it("shows skipped steps as skipped", async () => { const out = formatReport(await run({ resolve: { ok: false, reason: "not-found", detail: {} } }), "hello.aite"); expect(out).toContain(" skip the page greets us"); }); it("is plain text: no colours, no non-ASCII characters", async () => { const out = formatReport(await run({ verify: { ok: false, reason: "mismatch", detail: { expected: "a", found: "b", nearby: [] } } }), "hello.aite"); expect(out).toMatch(/^[\x20-\x7e\n]*$/); }); it("copes with a failure that has no detail", async () => { const out = formatReport(await run({ resolve: { ok: false, reason: "internal" } as never }), "hello.aite"); expect(out).toContain("(internal)"); });});