Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217import fc from "fast-check";import { describe, expect, it } from "vitest";import { runScript, type EngineDeps } from "../../runner/engine";import { parseScript } from "../../runner/parser";import type { Script } from "../../runner/parser";
const script = (text: string): Script => { const r = parseScript(text); if (!r.ok) throw new Error(JSON.stringify(r.errors)); return r.script;};
type Body = { cmd: string; [k: string]: unknown };type Answer = Record<string, unknown>;
/** A fake bridge. `answer` decides each request; the clock moves 10 ms per request; sleeps are recorded, not waited. */function fake(answer: (b: Body, n: number) => Answer = () => ({ ok: true }), extra: Partial<EngineDeps> = {}) { const sent: Body[] = []; const slept: number[] = []; let t = 0; const deps: EngineDeps = { request: async (body) => { sent.push(body); t += 10; const a = body.cmd === "resolve" ? { ok: true, descriptor: { tag: "button" }, ...answer(body, sent.length) } : answer(body, sent.length); return { id: sent.length, ...a } as never; }, now: () => t, sleep: async (ms) => void (slept.push(ms), (t += ms)), ...extra, }; return { deps, sent, slept };}const two = `test "t"\nstep "a"\n click text:"A"\n verify text:"x"\nstep "b"\n click text:"B"\n verify text:"y"`;
describe("P2-2.a: a failure skips the rest, a retry is counted", () => { it("a failed command fails its step and test, skips its later commands and every later step, and sends nothing more", async () => { const { deps, sent } = fake((b) => (b.cmd === "verify" ? { ok: false, reason: "mismatch", detail: { expected: "x", found: "z" } } : { ok: true })); const r = await runScript(script(two), deps); expect(r.ok).toBe(false); expect(r.status).toBe("failed"); expect(r.steps.map((s) => s.status)).toEqual(["failed", "skipped"]); expect(r.steps[0]!.commands.map((c) => c.status)).toEqual(["passed", "failed"]); expect(r.steps[1]!.commands.map((c) => c.status)).toEqual(["skipped", "skipped"]); expect(sent.map((s) => s.cmd)).toEqual(["resolve", "act", "verify"]); }); it("a later command in the same step is skipped, not run", async () => { const { deps, sent } = fake((b) => (b.cmd === "resolve" ? { ok: false, reason: "not-found", detail: {} } : { ok: true })); const r = await runScript(script('test "t"\nstep "a"\n click text:"A"\n verify text:"x"'), deps); expect(r.steps[0]!.commands.map((c) => c.status)).toEqual(["failed", "skipped"]); expect(sent.map((s) => s.cmd)).toEqual(["resolve"]); }); it("a command that passed after the extension retried it makes its step passed-after-retry, with retries=1", async () => { const { deps } = fake((b) => (b.cmd === "verify" ? { ok: true, retries: 1 } : { ok: true })); const r = await runScript(script(two), deps); expect(r.ok).toBe(true); expect(r.steps.map((s) => s.status)).toEqual(["passed-after-retry", "passed-after-retry"]); expect(r.steps[0]!.commands.map((c) => [c.status, c.retries])).toEqual([["passed", 0], ["passed-after-retry", 1]]); expect(r.steps[0]!.retries).toBe(1); expect(r.retries).toBe(2); expect(r.status).toBe("passed"); }); it("a step with no retries stays passed", async () => { const r = await runScript(script(two), fake().deps); expect(r.steps.map((s) => s.status)).toEqual(["passed", "passed"]); expect(r.retries).toBe(0); }); it("retries are added up over the requests of one command (resolve and act)", async () => { const { deps } = fake((b) => (b.cmd === "resolve" ? { retries: 2 } : b.cmd === "act" ? { ok: true, retries: 1 } : { ok: true })); const r = await runScript(script('test "t"\nstep "a"\n click text:"A"\n verify text:"x"'), deps); expect(r.steps[0]!.commands[0]).toMatchObject({ status: "passed-after-retry", retries: 3 }); }); it("a failed command keeps the retries it used, and its step is failed, never passed-after-retry", async () => { const { deps } = fake((b) => (b.cmd === "verify" ? { ok: false, reason: "timeout", detail: { waitedMs: 5 }, retries: 2 } : { ok: true })); const r = await runScript(script('test "t"\nstep "a"\n verify text:"x"'), deps); expect(r.steps[0]).toMatchObject({ status: "failed", retries: 2 }); expect(r.steps[0]!.commands[0]).toMatchObject({ status: "failed", retries: 2, reason: "timeout" }); }); it("ignores a retries value that is not a count", async () => { const { deps } = fake((b) => (b.cmd === "verify" ? { ok: true, retries: -4 } : { ok: true, retries: "many" })); const r = await runScript(script(two), deps); expect(r.retries).toBe(0); }); it("records what a failed command wanted and found", async () => { const { deps } = fake((b) => (b.cmd === "verify" ? { ok: false, reason: "mismatch", detail: { expected: "x", found: "z" } } : { ok: true })); const c = (await runScript(script('test "t"\nstep "a"\n verify text:"x"'), deps)).steps[0]!.commands[0]!; expect(c).toMatchObject({ line: 3, verb: "verify", source: 'verify text:"x"', status: "failed", reason: "mismatch", expected: "x", found: "z" }); expect(c.durationMs).toBeGreaterThan(0); });});
describe("P2-2.b: a test that checks nothing says so", () => { it("warns no-assertion when the test has no verify command", async () => { const r = await runScript(script('test "t"\nstep "a"\n click text:"A"'), fake().deps); expect(r.ok).toBe(true); expect(r.warnings).toEqual([{ code: "no-assertion", message: "this test has no verify command, so it cannot fail on what the page shows" }]); }); it("does not warn when there is a verify", async () => { expect((await runScript(script(two), fake().deps)).warnings).toEqual([]); }); it("warns on a step with no commands (a hand-built script; the parser refuses these)", async () => { const s = script('test "t"\nstep "a"\n verify text:"x"'); s.steps.push({ name: "empty", line: 9, commands: [] }); const r = await runScript(s, fake().deps); expect(r.steps[1]).toMatchObject({ status: "passed", warnings: [{ code: "no-assertion", message: "this step has no commands, so it checks nothing" }] }); });});
describe("P2-2.e: sleep is a visible warning", () => { it("waits without asking the extension, and warns with the line and the length", async () => { const { deps, sent, slept } = fake(); const r = await runScript(script('test "t"\nstep "a"\n sleep 2s\n verify text:"x"'), deps); expect(slept).toEqual([2000]); expect(sent.map((s) => s.cmd)).toEqual(["verify"]); expect(r.steps[0]!.commands[0]).toMatchObject({ verb: "sleep", status: "passed", warnings: [{ code: "sleep", line: 3 }] }); expect(r.steps[0]!.commands[0]!.warnings[0]!.message).toContain("2000 ms"); expect(r.steps[0]!.warnings.map((w) => w.code)).toEqual(["sleep"]); }); it("counts in the run's warning total", async () => { const r = await runScript(script('test "t"\nstep "a"\n sleep 1s\n sleep 1s\n verify text:"x"'), fake().deps); expect(r.warningCount).toBe(2); }); it("is not run when an earlier command failed", async () => { const { deps, slept } = fake((b) => (b.cmd === "verify" ? { ok: false, reason: "timeout", detail: {} } : { ok: true })); await runScript(script('test "t"\nstep "a"\n verify text:"x"\n sleep 1s'), deps); expect(slept).toEqual([]); });});
describe("variables", () => { it("set defines a variable and ${NAME} is replaced in verify text and in targets, with no request for set", async () => { const { deps, sent } = fake(); await runScript(script('test "t"\nstep "a"\n set who "Alex"\n click text:"Hi ${who}"\n verify text:"Hello, ${who}!"'), deps); expect(sent.map((s) => s.cmd)).toEqual(["resolve", "act", "verify"]); expect(sent[0]).toMatchObject({ target: { text: "Hi Alex" } }); expect(sent[2]).toMatchObject({ value: "Hello, Alex!" }); }); it("a set value may use an earlier variable", async () => { const { deps, sent } = fake(); await runScript(script('test "t"\nstep "a"\n set a "x"\n set b "${a}y"\n verify text:"${b}"'), deps); expect(sent[0]).toMatchObject({ value: "xy" }); }); it("a later set replaces the earlier value", async () => { const { deps, sent } = fake(); await runScript(script('test "t"\nstep "a"\n set a "1"\n set a "2"\n verify text:"${a}"'), deps); expect(sent[0]).toMatchObject({ value: "2" }); }); it("a variable that is not set when it is used (not possible from a parsed script) fails the command and sends nothing", async () => { const s = script('test "t"\nstep "a"\n set a "1"\n verify text:"${a}"'); (s.steps[0]!.commands[0] as { name: string }).name = "other"; const { deps, sent } = fake(); const r = await runScript(s, deps); expect(r.ok).toBe(false); expect(sent).toEqual([]); expect(r.steps[0]!.commands[1]).toMatchObject({ status: "failed", reason: "internal" }); expect(r.steps[0]!.failure!.detail).toMatchObject({ message: 'the variable "a" is not set' }); }); it("a $ that is not ${NAME} is left alone", async () => { const { deps, sent } = fake(); await runScript(script('test "t"\nstep "a"\n verify text:"cost $5 ${x"'), deps); expect(sent[0]).toMatchObject({ value: "cost $5 ${x" }); });});
describe("the screenshot warning is a step warning", () => { it("a screenshot that cannot be taken is a warning with a code, and the step still passes", async () => { const { deps } = fake((b) => (b.cmd === "screenshot" ? { ok: false, reason: "internal", detail: { message: "no" } } : { ok: true }), { saveScreenshot: async () => undefined }); const r = await runScript(script('test "t"\nstep "a"\n verify text:"x"'), deps); expect(r.steps[0]!.status).toBe("passed"); expect(r.steps[0]!.warnings).toEqual([{ code: "screenshot", message: "could not take a screenshot: no" }]); });});
describe("P2-2.c: the test passes if and only if every command passed", () => { type Outcome = "ok" | "retry" | "fail"; const kinds = ["click", "verify", "set", "sleep"] as const; const stepArb = fc.array(fc.constantFrom(...kinds), { minLength: 1, maxLength: 4 }); const planArb = fc.record({ steps: fc.array(stepArb, { minLength: 1, maxLength: 5 }), outcomes: fc.array(fc.constantFrom<Outcome>("ok", "ok", "ok", "retry", "fail"), { minLength: 30, maxLength: 30 }) }); const text = (steps: readonly (typeof kinds)[number][][]) => 'test "t"\n' + steps.map((cmds, i) => `step "s${i}"\n` + cmds.map((k) => (k === "click" ? ' click text:"A"' : k === "verify" ? ' verify text:"x"' : k === "set" ? ' set v "1"' : " sleep 1s")).join("\n")).join("\n");
it("holds for any mix of outcomes", async () => { await fc.assert( fc.asyncProperty(planArb, async ({ steps, outcomes }) => { let i = 0; const { deps, sent } = fake((b) => { if (b.cmd === "navigate") return { ok: true }; const o = outcomes[i++ % outcomes.length]!; return o === "fail" ? { ok: false, reason: "mismatch", detail: {} } : o === "retry" ? { ok: true, retries: 1 } : { ok: true }; }); const r = await runScript(script(text(steps)), deps); const commands = r.steps.flatMap((s) => s.commands); const anyFailed = commands.some((c) => c.status === "failed"); // the invariant expect(r.ok).toBe(!anyFailed); expect(r.status).toBe(anyFailed ? "failed" : "passed"); // nothing runs after the first failure, and nothing is lost const firstFail = commands.findIndex((c) => c.status === "failed"); if (firstFail >= 0) expect(commands.slice(firstFail + 1).every((c) => c.status === "skipped")).toBe(true); expect(commands).toHaveLength(steps.flat().length); // statuses agree with their commands for (const s of r.steps) { const failed = s.commands.some((c) => c.status === "failed"); const ran = s.commands.some((c) => c.status !== "skipped"); const retried = s.commands.some((c) => c.status === "passed-after-retry"); expect(s.status).toBe(failed ? "failed" : !ran ? "skipped" : retried ? "passed-after-retry" : "passed"); expect(s.retries).toBe(s.commands.reduce((n, c) => n + c.retries, 0)); } expect(r.retries).toBe(r.steps.reduce((n, s) => n + s.retries, 0)); // only commands that talk to the page were sent expect(sent.every((b) => ["resolve", "act", "verify"].includes(b.cmd))).toBe(true); }), { numRuns: 300 }, ); });});