From df13d2f4830644d586fbfee5abbd185fc4eaf0ac Mon Sep 17 00:00:00 2001 From: Alex Wykoff Date: Sun, 11 Oct 2026 13:04:59 -0500 Subject: [PATCH] feat(runner): pre-state failures (balance-at-least), block-separated read, functional page-logic tests A check can now report that the account was not in the state the test needs (pre-state): the report names the account, the balance and the shortfall and says it is not an app failure; giveVerdict refuses app-broken for it. read gained blocks so handles and amounts do not run together. Co-Authored-By: Claude Sonnet 5.5 --- extension/src/pageCommands.ts | 2 +- extension/src/pageConditions.ts | 6 +- extension/src/pageDom.ts | 25 ++++ report/render.ts | 1 + report/verdict.ts | 1 + report/verdictService.ts | 6 + runner/checks.ts | 20 +-- runner/engine.ts | 2 +- runner/preState.ts | 30 +++++ runner/run.ts | 4 +- scripts/checks/balance-at-least.ts | 9 ++ scripts/checks/session-check.ts | 2 +- test/functional/extension.page.logic.test.ts | 132 +++++++++++++++++++ test/integration/session.whole.test.ts | 25 +++- test/unit/checks.test.ts | 25 ++++ test/unit/pageTargets.test.ts | 26 ++++ test/unit/preState.test.ts | 45 +++++++ test/unit/reportRender.test.ts | 14 ++ test/unit/verdictService.test.ts | 29 ++++ 19 files changed, 387 insertions(+), 17 deletions(-) create mode 100644 runner/preState.ts create mode 100644 scripts/checks/balance-at-least.ts create mode 100644 test/functional/extension.page.logic.test.ts create mode 100644 test/unit/preState.test.ts diff --git a/extension/src/pageCommands.ts b/extension/src/pageCommands.ts index 11ef7c0..40a077e 100644 --- a/extension/src/pageCommands.ts +++ b/extension/src/pageCommands.ts @@ -49,7 +49,7 @@ export async function handleCommand(ctx: PageContext, req: Request): Promise { +export async function readRequest(ctx: PageContext, req: { target: Target; blocks?: boolean; timeoutMs?: number }): Promise { const found = await resolveElement(ctx, req.target, req.timeoutMs ?? DEFAULT_TIMEOUT_MS); if (!found.ok) return found.failure; const el = found.el as HTMLInputElement; const isField = ["INPUT", "TEXTAREA", "SELECT"].includes(el.tagName) && !["submit", "button", "reset"].includes((el.getAttribute("type") ?? "").toLowerCase()); - return { ok: true, text: isField ? String(el.value ?? "") : visibleText(el) }; + return { ok: true, text: isField ? String(el.value ?? "") : req.blocks === true ? blockText(el) : visibleText(el) }; } diff --git a/extension/src/pageDom.ts b/extension/src/pageDom.ts index 5046ca8..00fa135 100644 --- a/extension/src/pageDom.ts +++ b/extension/src/pageDom.ts @@ -68,6 +68,31 @@ export function isVisible(el: Element): boolean { return el.getClientRects().length > 0; } +/** + * The visible text of an element with a space between its blocks (paragraphs, headings, list items, divs, line breaks), so "Signed in as @a.test" followed by + * "Balance: 3" reads as two things and not as "@a.testBalance". Inline elements do not split a word. Used to read a whole page for a check; the text + * that targets and verifies compare is still `visibleText`, which joins blocks as it always did. + */ +export function blockText(el: Element): string { + const walk = (e: Element): string => { + const label = buttonLabel(e); + if (label !== undefined) return label; + let out = ""; + for (const node of Array.from(e.childNodes)) { + if (node.nodeType === 3) out += node.nodeValue ?? ""; + else if (node.nodeType === 1) { + const child = node as Element; + if (SKIP_TEXT.has(child.tagName) || hiddenByStyle(child)) continue; + const display = child.ownerDocument.defaultView?.getComputedStyle(child).display ?? "inline"; + const separate = child.tagName === "BR" || (display !== "inline" && display !== "contents"); + out += separate ? ` ${walk(child)} ` : walk(child); + } + } + return out; + }; + return normalize(walk(el)); +} + /** The text a person can see in an element: hidden parts left out, white space collapsed. */ export function visibleText(el: Element): string { const walk = (e: Element): string => { diff --git a/report/render.ts b/report/render.ts index 280bfb5..1fe568f 100644 --- a/report/render.ts +++ b/report/render.ts @@ -26,6 +26,7 @@ const REASONS: Record = { timeout: "The page did not get to the wanted state in time.", drifted: "A similar element is on the page, but not similar enough to trust: please confirm it is the one.", unlocked: "This run does not know what the target looked like before (run it once interactively first).", + "pre-state": "The test's starting state is wrong (for example a balance too low). This is not a failure of the app.", navigation: "The page navigated while the command was running.", "tab-closed": "The tab was closed.", "connection-lost": "The connection to the browser was lost.", diff --git a/report/verdict.ts b/report/verdict.ts index b667036..3506766 100644 --- a/report/verdict.ts +++ b/report/verdict.ts @@ -61,6 +61,7 @@ const REASONS: Record = { timeout: "The page did not get to the wanted state in time.", drifted: "A similar element is on the page, but not similar enough to trust.", unlocked: "This run did not know what the target looked like before.", + "pre-state": "The test's starting state was wrong (for example a balance too low): not a failure of the app.", }; export function failedStep(run: RunJson, stepIndex: number): { step: RunJsonStep; command: RunJsonCommand } { diff --git a/report/verdictService.ts b/report/verdictService.ts index 36ed74f..f98b4c5 100644 --- a/report/verdictService.ts +++ b/report/verdictService.ts @@ -74,6 +74,8 @@ export interface VerdictOutcome { export const FLAKY_RUNS = 3; +const failedStepReason = (run: RunJson, stepIndex: number): string | undefined => failedStep(run, stepIndex).command.reason; + /** A file name part from a test name: lower case words, hyphens, never empty, never a path. */ export function slug(text: string): string { const s = text.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40).replace(/-+$/g, ""); @@ -154,6 +156,10 @@ export async function giveVerdict(ctx: VerdictContext, req: VerdictRequest): Pro } // app-broken + if (failedStepReason(run, step.index) === "pre-state") { + const error = { kind: "pre-state", message: "this step failed because the test's starting state was wrong (for example a balance too low), not because the app is broken: no issue is filed. Fix the state and run again, or choose Script is out of date or Flaky." }; + return { status: "saved-locally", verdict: save({ ...base, error }), error }; + } const before = existing(); if (before?.issue !== undefined) return { status: "already", verdict: before, issue: { iid: Number(before.issue.split("#")[1]), url: before.issueUrl ?? "", reference: before.issue } }; if (ctx.gitlab === undefined) { diff --git a/runner/checks.ts b/runner/checks.ts index 0eb7126..181360d 100644 --- a/runner/checks.ts +++ b/runner/checks.ts @@ -5,8 +5,11 @@ import type { CheckOutcome } from "./engine"; export interface CheckContext { args: string[]; vars: Record; - /** The text of the element a target names (the same targets as in a script). Fails if the page cannot answer. */ - read(target: Record): Promise; + /** + * The text of the element a target names (the same targets as in a script). Fails if the page cannot answer. With `blocks` the text of neighbouring blocks is + * separated by a space, which is what reading a whole page for a handle or an amount needs. + */ + read(target: Record, options?: { blocks?: boolean }): Promise; /** The address of the page the run is on. */ url(): Promise; /** The profile the run uses, and the account it is signed in as when the config says so. */ @@ -14,7 +17,8 @@ export interface CheckContext { account: string | undefined; } -export type CheckResult = void | undefined | boolean | { ok: boolean; message?: string; expected?: unknown; found?: unknown }; +/** A check says the test's starting state is wrong (a balance too low, an account limited), not the app, with reason "pre-state". Any other reason it names is a mismatch. */ +export type CheckResult = void | undefined | boolean | { ok: boolean; reason?: "pre-state"; message?: string; expected?: unknown; found?: unknown }; export type CheckFn = (ctx: CheckContext) => CheckResult | Promise; export type CheckModuleLoader = (path: string) => Promise; @@ -36,7 +40,7 @@ export async function runCheck(options: { args: string[]; vars: Record; timeoutMs: number; - read(target: Record): Promise; + read(target: Record, options?: { blocks?: boolean }): Promise; url(): Promise; profile?: string; account?: string | undefined; @@ -70,15 +74,15 @@ export async function runCheck(options: { if (finished === undefined || finished === true) return { ok: true }; if (finished === false) return { ok: false, reason: "mismatch", message: `check ${options.name} returned false` }; if (typeof finished === "object" && finished !== null && typeof (finished as { ok?: unknown }).ok === "boolean") { - const r = finished as { ok: boolean; message?: string; expected?: unknown; found?: unknown }; + const r = finished as { ok: boolean; reason?: unknown; message?: string; expected?: unknown; found?: unknown }; if (r.ok) return { ok: true }; - return { ok: false, reason: "mismatch", message: r.message ?? `check ${options.name} failed`, ...(r.expected !== undefined ? { expected: r.expected } : {}), ...(r.found !== undefined ? { found: r.found } : {}) }; + return { ok: false, reason: r.reason === "pre-state" ? "pre-state" : "mismatch", message: r.message ?? `check ${options.name} failed`, ...(r.expected !== undefined ? { expected: r.expected } : {}), ...(r.found !== undefined ? { found: r.found } : {}) }; } return internal(`check ${options.name} returned something that is not a result (return nothing, true, false or { ok })`); } catch (err) { const e = err as { name?: string; message?: string; expected?: unknown; actual?: unknown }; - if (e?.name === "AssertionError" || e?.name === "CheckFailed") { - return { ok: false, reason: "mismatch", message: e.message ?? "assertion failed", ...(e.expected !== undefined ? { expected: e.expected } : {}), ...(e.actual !== undefined ? { found: e.actual } : {}) }; + if (e?.name === "AssertionError" || e?.name === "CheckFailed" || e?.name === "PreStateFailed") { + return { ok: false, reason: e.name === "PreStateFailed" ? "pre-state" : "mismatch", message: e.message ?? "assertion failed", ...(e.expected !== undefined ? { expected: e.expected } : {}), ...(e.actual !== undefined ? { found: e.actual } : {}) }; } return internal(`check ${options.name} crashed: ${err instanceof Error ? err.message : String(err)}`); } finally { diff --git a/runner/engine.ts b/runner/engine.ts index 67b6ea5..fd70bc0 100644 --- a/runner/engine.ts +++ b/runner/engine.ts @@ -40,7 +40,7 @@ export interface FailureToDiagnose { commandIndex: number; } -export type CheckOutcome = { ok: true } | { ok: false; reason: "mismatch" | "internal" | "timeout"; message: string; expected?: unknown; found?: unknown }; +export type CheckOutcome = { ok: true } | { ok: false; reason: "mismatch" | "internal" | "timeout" | "pre-state"; message: string; expected?: unknown; found?: unknown }; /** What a `call` hands to the runner: the check's name and arguments, the variables set so far, and the tab and time limit of the run. */ export interface CheckCall { diff --git a/runner/preState.ts b/runner/preState.ts new file mode 100644 index 0000000..549d3ff --- /dev/null +++ b/runner/preState.ts @@ -0,0 +1,30 @@ +/** + * The test's starting state, as opposed to the app: a balance that is too low to run a test that spends, an account that cannot do what the test needs. + * A failure of this kind has the reason "pre-state" (docs/design/protocol.md), so a report and a person can tell it from the app being wrong: + * nobody should file a bug against the app because a test account ran out of testnet money. + */ +export type PreStateVerdict = { ok: true } | { ok: false; reason: "pre-state" | "internal"; message: string; expected?: string; found?: string }; + +const escape = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +const num = (n: number): string => String(Math.round(n * 1e9) / 1e9); + +/** Whether the balance the page shows (a number after the label, such as "Balance: 12.5 MALLBUCKS") is at least what the test needs. */ +export function judgeBalance(input: { account: string; needed: number; label: string; pageText: string }): PreStateVerdict { + if (!Number.isFinite(input.needed) || input.needed < 0) return { ok: false, reason: "internal", message: "balance-at-least needs an amount of zero or more" }; + if (input.label.trim() === "") return { ok: false, reason: "internal", message: "balance-at-least needs the label the page shows next to the balance, for example Balance" }; + const text = input.pageText.replace(/\s+/g, " "); + const found = new RegExp(`(?= input.needed) return { ok: true }; + const shown = found[1]!.length > 30 ? `${found[1]!.slice(0, 30)}...` : found[1]!; + return { + ok: false, + reason: "pre-state", + message: `${input.account.slice(0, 80)} has ${shown} but this test needs ${num(input.needed)} (short by ${num(input.needed - have)}). This is about the test's starting state, not a failure of the app: refill the account (see docs/runbook-sessions.md) and run again.`, + expected: `>= ${num(input.needed)}`, + found: shown, + }; +} diff --git a/runner/run.ts b/runner/run.ts index e82885e..697ec1d 100644 --- a/runner/run.ts +++ b/runner/run.ts @@ -369,10 +369,10 @@ export async function runScriptFile(options: RunOptions, deps: RunDeps = realRun const seat = seats.get(key) ?? seats.get(defaultKey)!; return runCheck({ name: call.name, args: call.args, vars: call.vars, timeoutMs: call.timeoutMs, checksDir, profile: asProfile, account: config.profiles[asProfile]?.account, load: deps.importModule, exists: deps.fileExists, - read: async (target) => { + read: async (target, readOptions) => { // a read waits up to half the command's time, so a check whose element never comes can report that before it is stopped for taking too long const wait = Math.max(100, Math.floor(call.timeoutMs / 2)); - const a = (await request({ cmd: "read", target, ...(call.profile !== undefined ? { profile: call.profile } : {}), ...(call.tab !== undefined ? { tab: call.tab } : {}), timeoutMs: wait }, { timeoutMs: wait + 5000 })) as unknown as { ok?: boolean; text?: string; reason?: string; detail?: unknown }; + const a = (await request({ cmd: "read", target, ...(readOptions?.blocks === true ? { blocks: true } : {}), ...(call.profile !== undefined ? { profile: call.profile } : {}), ...(call.tab !== undefined ? { tab: call.tab } : {}), timeoutMs: wait }, { timeoutMs: wait + 5000 })) as unknown as { ok?: boolean; text?: string; reason?: string; detail?: unknown }; if (a.ok !== true || typeof a.text !== "string") throw new Error(`${a.reason ?? "internal"}: ${JSON.stringify(a.detail ?? {})}`); return a.text; }, diff --git a/scripts/checks/balance-at-least.ts b/scripts/checks/balance-at-least.ts new file mode 100644 index 0000000..9a5d8c0 --- /dev/null +++ b/scripts/checks/balance-at-least.ts @@ -0,0 +1,9 @@ +import { judgeBalance } from "../../runner/preState"; +import type { CheckContext } from "../../runner/checks"; + +/** + * `call balance-at-least AMOUNT [LABEL]`: a pre-state check. The page must show at least AMOUNT after LABEL (default "Balance") for the account of the profile, or the + * run fails as `pre-state`, not as an app failure. Put it first in a test that spends, after `call session-check`. + */ +export default async (ctx: CheckContext) => + judgeBalance({ account: ctx.account ?? ctx.profile, needed: Number(ctx.args[0]), label: ctx.args[1] ?? "Balance", pageText: await ctx.read({ css: "body" }, { blocks: true }) }); diff --git a/scripts/checks/session-check.ts b/scripts/checks/session-check.ts index d04f2e7..e318704 100644 --- a/scripts/checks/session-check.ts +++ b/scripts/checks/session-check.ts @@ -5,4 +5,4 @@ import type { CheckContext } from "../../runner/checks"; * `call session-check [HANDLE]`: fails the run, with what to do about it, unless the page the run is on shows the account the profile is signed in as * (HANDLE, or the profile's `account` in aite.config.json). The logic is in runner/sessionCheck.ts. Every script that runs on a signed-in profile starts with it. */ -export default async (ctx: CheckContext) => judgeSession({ profile: ctx.profile, expected: ctx.args[0] ?? ctx.account ?? "", pageText: await ctx.read({ css: "body" }) }); +export default async (ctx: CheckContext) => judgeSession({ profile: ctx.profile, expected: ctx.args[0] ?? ctx.account ?? "", pageText: await ctx.read({ css: "body" }, { blocks: true }) }); diff --git a/test/functional/extension.page.logic.test.ts b/test/functional/extension.page.logic.test.ts new file mode 100644 index 0000000..f9aa8cd --- /dev/null +++ b/test/functional/extension.page.logic.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from "vitest"; +import { JSDOM } from "jsdom"; +import { handleCommand } from "../../extension/src/pageCommands"; +import type { PageContext } from "../../extension/src/pageDom"; +import { applyLayout } from "../helpers/dom"; + +/** + * The page-side logic of the extension on a realistic fixture page, run by jsdom with the page's own script: every kind of verify, the outline, and + * reading text. (The same code in a real Chrome is covered by the end-to-end tests; this is the table of what each command says on a page.) + */ + +const PAGE = `Send money +

Send money

+

Amount: 1 MALLBUCKS

+ + +More +
  • One
  • Two
  • Three
+
Balance: 100
+ + +`; + +function load(url = "http://127.0.0.1:4780/send.html?x=1") { + const dom = new JSDOM(PAGE, { url, runScripts: "dangerously" }); + applyLayout(dom.window); + let t = 0; + const ctx: PageContext = { doc: dom.window.document, now: () => t, sleep: async (ms) => void (t += ms) }; + const ask = async (req: Record) => (await handleCommand(ctx, { id: 1, cmd: "verify", timeoutMs: 300, ...req } as never)) as { ok: boolean; reason?: string; found?: unknown; detail?: Record }; + return { dom, ctx, ask }; +} +const passes = async (req: Record) => expect((await load().ask(req)).ok, JSON.stringify(req)).toBe(true); +const fails = async (req: Record, reason = "mismatch") => { + const r = await load().ask(req); + expect(r.ok, JSON.stringify(req)).toBe(false); + expect(r.reason, JSON.stringify(req)).toBe(reason); +}; + +describe("verify on a page", () => { + it("text: exact, ignoring case, and as a part ignoring case", async () => { + await passes({ kind: "text", value: "Amount: 1 MALLBUCKS" }); + await fails({ kind: "text", value: "amount: 1 mallbucks" }); + await passes({ kind: "text", value: "amount: 1 mallbucks", match: "ignoreCase" }); + await passes({ kind: "text", value: "1 mallbucks", match: "containsIgnoreCase" }); + await fails({ kind: "text", value: "2 mallbucks", match: "containsIgnoreCase" }); + }); + it("url and title", async () => { + await passes({ kind: "url", contains: "send.html" }); + await fails({ kind: "url", contains: "nope" }); + await passes({ kind: "title", value: "Send money" }); + await fails({ kind: "title", value: "Other" }); + }); + it("exists and count", async () => { + await passes({ kind: "exists", target: { css: "#send" } }); + await fails({ kind: "exists", target: { css: "#nothing" } }, "not-found"); + await passes({ kind: "count", target: { css: ".row" }, count: 3 }); + await fails({ kind: "count", target: { css: ".row" }, count: 2 }); + }); + it("the text of a target, and the value of a field", async () => { + await passes({ kind: "targetText", target: { css: "#balance" }, value: "Balance: 100" }); + await fails({ kind: "targetText", target: { css: "#balance" }, value: "Balance: 99" }); + await passes({ kind: "value", target: { css: "#to" }, value: "qian.pds.malldao.xyz" }); + await fails({ kind: "value", target: { css: "#to" }, value: "someone" }); + }); + it("states: enabled, disabled, checked", async () => { + await passes({ kind: "state", target: { css: "#send" }, state: "enabled" }); + await passes({ kind: "state", target: { css: "#off" }, state: "disabled" }); + await passes({ kind: "state", target: { css: "#agree" }, state: "checked" }); + await fails({ kind: "state", target: { css: "#send" }, state: "disabled" }); + await fails({ kind: "state", target: { css: "#off" }, state: "enabled" }); + }); + it("attributes, as written, equal or containing", async () => { + await passes({ kind: "attr", target: { css: "#more" }, attr: "class", op: "equals", value: "link" }); + await passes({ kind: "attr", target: { css: "#more" }, attr: "href", op: "contains", value: "/docs/more" }); + await fails({ kind: "attr", target: { css: "#more" }, attr: "href", op: "equals", value: "/other" }); + await fails({ kind: "attr", target: { css: "#more" }, attr: "data-nothing", op: "contains", value: "x" }); + }); + it("a request that is missing what it needs, or of a kind nobody knows, is refused, never passed", async () => { + await fails({ kind: "text" }, "internal"); + await fails({ kind: "count", target: { css: ".row" } }, "internal"); + await fails({ kind: "state", target: { css: "#send" }, state: "sleepy" }, "internal"); + await fails({ kind: "url" }, "internal"); + await fails({ kind: "nonsense" }, "unsupported"); + await fails({}, "internal"); + }); + it("the click on the page's own button changes what is there, and verify sees the change", async () => { + const p = load(); + expect((await p.ask({ kind: "targetText", target: { css: "#balance" }, value: "Balance: 99" })).ok).toBe(false); + p.dom.window.document.getElementById("send")!.click(); + expect((await p.ask({ kind: "targetText", target: { css: "#balance" }, value: "Balance: 99" })).ok).toBe(true); + }); +}); + +describe("outline on a page", () => { + const draw = async (items: unknown[]) => { + const p = load(); + const a = (await handleCommand(p.ctx, { id: 1, cmd: "outline", items } as never)) as { ok: boolean; drawn?: number }; + return { p, a, boxes: () => p.dom.window.document.querySelectorAll("[data-aite-overlay]").length }; + }; + it("draws boxes for the elements found, labelled, and clears them leaving the page as it was", async () => { + const before = load().dom.window.document.documentElement.outerHTML; + const d = await draw([{ xpath: "/html/body/button", label: "wanted", tone: "wanted" }, { xpath: "/html/body/a", label: "maybe", tone: "candidate" }]); + expect(d.a).toMatchObject({ ok: true, drawn: 2 }); + expect(d.boxes()).toBe(2); + const cleared = (await handleCommand(d.p.ctx, { id: 2, cmd: "outline", clear: true } as never)) as { removed: number; remaining: number }; + expect(cleared).toMatchObject({ removed: 2, remaining: 0 }); + expect(d.p.dom.window.document.documentElement.outerHTML).toBe(before); + }); + it("skips what is not there, draws no more than ten, and refuses a request with nothing to do", async () => { + expect((await draw([{ xpath: "/html/body/nothing", label: "x", tone: "wanted" }])).a).toMatchObject({ ok: true, drawn: 0 }); + const many = Array.from({ length: 15 }, (_, i) => ({ xpath: `/html/body/ul/li[${(i % 3) + 1}]`, label: String(i), tone: "other" })); + expect((await draw(many)).a.drawn).toBe(10); + const p = load(); + expect(((await handleCommand(p.ctx, { id: 1, cmd: "outline" } as never)) as { ok: boolean }).ok).toBe(false); + }); +}); + +describe("read on a page", () => { + const read = async (target: Record, blocks?: boolean) => { + const p = load(); + return (await handleCommand(p.ctx, { id: 1, cmd: "read", target, ...(blocks ? { blocks: true } : {}), timeoutMs: 200 } as never)) as { ok: boolean; text?: string; reason?: string }; + }; + it("reads the text of an element, the value of a field, and a whole page with its blocks apart", async () => { + expect((await read({ css: "#balance" })).text).toBe("Balance: 100"); + expect((await read({ css: "#to" })).text).toBe("qian.pds.malldao.xyz"); + expect((await read({ css: "body" }, true)).text).toMatch(/^Send money Amount: 1 MALLBUCKS Handle /); + expect((await read({ css: "body" })).text).toContain("Balance: 100"); + }); + it("an element that is not there is not-found", async () => { + expect(await read({ css: "#nothing" })).toMatchObject({ ok: false, reason: "not-found" }); + }); +}); diff --git a/test/integration/session.whole.test.ts b/test/integration/session.whole.test.ts index 68a9c56..366178c 100644 --- a/test/integration/session.whole.test.ts +++ b/test/integration/session.whole.test.ts @@ -31,7 +31,7 @@ async function project(pageBody: string) { mkdirSync(join(repo.dir, "site"), { recursive: true }); mkdirSync(join(repo.dir, "scripts/checks"), { recursive: true }); repo.write("site/app.html", page(pageBody)); - repo.write("scripts/checks/session-check.ts", `export { default } from ${JSON.stringify(join(root, "scripts/checks/session-check.ts"))};\n`); + for (const name of ["session-check", "balance-at-least"]) repo.write(`scripts/checks/${name}.ts`, `export { default } from ${JSON.stringify(join(root, `scripts/checks/${name}.ts`))};\n`); const profileDir = join(repo.dir, "profiles/alice"); repo.write("aite.config.json", JSON.stringify({ profiles: { alice: { userDataDir: profileDir, chrome: "/unused/chrome", account: "alice.test" } }, allow: [origin] })); const configPath = join(repo.dir, "aite.config.json"); @@ -100,3 +100,26 @@ describe("aite profile open through the whole stack", () => { expect((await openProfile({ name: "alice", configPath: p.configPath, headless: true, noSandbox: true }, deps)).exitCode).toBe(2); }); }); + +describe("a pre-state check through the whole stack (P5-7.a)", () => { + const withBalance = async (balance: string, needed: string) => { + const p = await project(`

Mallbucks

Signed in as @alice.test

Balance: ${balance} MALLBUCKS

`); + return p.asAlice(` call session-check\n step "enough money"\n call balance-at-least ${needed}\n`); + }; + it("a balance that is enough passes", async () => { + const r = await withBalance("12.5", "5"); + expect(r.exitCode, r.output).toBe(0); + }); + it("a balance that is too low fails as pre-state, naming the account and the shortfall, and says it is not the app", async () => { + const r = await withBalance("3", "5"); + expect(r.exitCode).toBe(1); + expect(r.output).toContain("(pre-state)"); + expect(r.output).toContain("alice.test has 3 but this test needs 5 (short by 2)"); + const json = JSON.parse(readFileSync(join(/^results: (.+)$/m.exec(r.output)![1]!, "run.json"), "utf8")); + const failed = json.tests[0].steps.flatMap((st: { commands: Array<{ status: string; reason: string }> }) => st.commands).find((c: { status: string }) => c.status === "failed"); + expect(failed.reason).toBe("pre-state"); + const html = readFileSync(join(/^results: (.+)$/m.exec(r.output)![1]!, "report.html"), "utf8"); + expect(html).toMatch(/starting state is wrong/i); + }); +}); + diff --git a/test/unit/checks.test.ts b/test/unit/checks.test.ts index 1193a17..fbbbaf7 100644 --- a/test/unit/checks.test.ts +++ b/test/unit/checks.test.ts @@ -103,3 +103,28 @@ describe("what a check knows about the profile (P5-3)", () => { expect(seen).toEqual([["", undefined]]); }); }); + +describe("a check can say the test's starting state is wrong, not the app (P5-7)", () => { + it("{ ok:false, reason:'pre-state' } is a pre-state failure with the message, expected and found", async () => { + const l = loader({ default: () => ({ ok: false, reason: "pre-state", message: "qian has 3, needs 5", expected: ">= 5", found: "3" }) }); + expect(await runCheck({ ...base, name: "x", ...l })).toEqual({ ok: false, reason: "pre-state", message: "qian has 3, needs 5", expected: ">= 5", found: "3" }); + }); + it("any other reason a check names is still just a mismatch (a check cannot make up reasons)", async () => { + const l = loader({ default: () => ({ ok: false, reason: "app-broken", message: "m" }) }); + expect(await runCheck({ ...base, name: "x", ...l })).toMatchObject({ ok: false, reason: "mismatch" }); + }); + it("throwing a PreStateFailed is the same", async () => { + const l = loader({ default: () => { throw Object.assign(new Error("short of funds"), { name: "PreStateFailed", expected: ">= 5", actual: "3" }); } }); + expect(await runCheck({ ...base, name: "x", ...l })).toMatchObject({ ok: false, reason: "pre-state", message: "short of funds" }); + }); +}); + +describe("read with blocks reaches the check (P5-3)", () => { + it("ctx.read(target, { blocks: true }) hands the option on", async () => { + const calls: unknown[] = []; + const l = loader({ default: async (ctx: any) => void (await ctx.read({ css: "body" }, { blocks: true })) }); + await runCheck({ ...base, name: "x", ...l, read: async (t: unknown, o?: unknown) => (calls.push([t, o]), "text") }); + expect(calls).toEqual([[{ css: "body" }, { blocks: true }]]); + }); +}); + diff --git a/test/unit/pageTargets.test.ts b/test/unit/pageTargets.test.ts index ccd1a3a..aa5f829 100644 --- a/test/unit/pageTargets.test.ts +++ b/test/unit/pageTargets.test.ts @@ -95,3 +95,29 @@ describe("the body is a target too (session-check reads the whole page)", () => expect(findMatches(doc, { text: "just words" })).toEqual([doc.body]); }); }); + +describe("read with blocks: the text of a page with a space between its blocks (session-check, P5-3)", () => { + const read = async (html: string, blocks?: boolean) => { + const { ctx } = makeDom(html); + const a = (await handleCommand(ctx, { id: 1, cmd: "read", target: { css: "body" }, ...(blocks ? { blocks: true } : {}), timeoutMs: 100 })) as { ok: boolean; text: string }; + return a.text; + }; + it("without blocks the text of neighbouring blocks runs together, as it always did", async () => { + expect(await read(`

Signed in as @a.test

Balance: 3

`)).toBe("Signed in as @a.testBalance: 3"); + }); + it("with blocks, paragraphs, headings, list items, divs and line breaks are separated by a space", async () => { + expect(await read(`

Title

Signed in as @a.test

Balance: 3
  • one
  • two
line
break`, true)).toBe("Title Signed in as @a.test Balance: 3 one two line break"); + }); + it("inline elements do not split a word, and text around them keeps its own spaces", async () => { + expect(await read(`

Hello world, link.

`, true)).toBe("Hello world, link."); + }); + it("hidden things, scripts and styles are left out, as in the plain text", async () => { + expect(await read(`

shown

gone

also

`, true)).toBe("shown also"); + }); + it("a block inside an inline element still separates, and a button's own label is used", async () => { + expect(await read(`
Top
Bottom
`, true)).toBe("Top Bottom Go Send"); + }); + it("spaces are collapsed, so the result has no double spaces or leading or trailing space", async () => { + expect(await read(`

a b

\n\n

c

`, true)).toBe("a b c"); + }); +}); diff --git a/test/unit/preState.test.ts b/test/unit/preState.test.ts new file mode 100644 index 0000000..4855334 --- /dev/null +++ b/test/unit/preState.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { judgeBalance } from "../../runner/preState"; + +const base = { account: "qian.pds.malldao.xyz", needed: 5, label: "Balance" }; + +describe("judgeBalance (P5-7.a)", () => { + it("passes when the balance is enough, exactly enough, or more", () => { + for (const text of ["Balance: 5", "Balance: 12.5 MALLBUCKS", "Your Balance 5.00 MALLBUCKS"]) expect(judgeBalance({ ...base, pageText: text }), text).toEqual({ ok: true }); + }); + it("fails as a pre-state failure naming the account, what it has, what the test needs and the shortfall", () => { + const r = judgeBalance({ ...base, pageText: "Balance: 3 MALLBUCKS" }); + expect(r).toMatchObject({ ok: false, reason: "pre-state", expected: ">= 5", found: "3" }); + if (!r.ok) { + expect(r.message).toContain("qian.pds.malldao.xyz"); + expect(r.message).toContain("has 3"); + expect(r.message).toContain("needs 5"); + expect(r.message).toContain("short by 2"); + expect(r.message).toMatch(/not a failure of the app/i); + expect(r.message).toContain("refill"); + } + }); + it("copes with decimals without float noise in the shortfall", () => { + const r = judgeBalance({ ...base, needed: 0.3, pageText: "Balance: 0.1" }); + expect(r.ok === false && r.message).toContain("short by 0.2"); + }); + it("a page that shows no balance is a pre-state failure too, saying what it looked for", () => { + const r = judgeBalance({ ...base, pageText: "Loading..." }); + expect(r).toMatchObject({ ok: false, reason: "pre-state" }); + if (!r.ok) expect(r.message).toMatch(/cannot find a "Balance" amount/i); + }); + it("the label is matched with any case, a colon is optional, and a number inside another word is not a balance", () => { + expect(judgeBalance({ ...base, pageText: "balance 7" })).toEqual({ ok: true }); + expect(judgeBalance({ ...base, pageText: "Balances: 9" }).ok).toBe(false); + expect(judgeBalance({ ...base, pageText: "Rebalance 9" }).ok).toBe(false); + }); + it("a bad amount or label is refused rather than passing anything", () => { + expect(judgeBalance({ ...base, needed: Number.NaN, pageText: "Balance: 9" })).toMatchObject({ ok: false, reason: "internal" }); + expect(judgeBalance({ ...base, needed: -1, pageText: "Balance: 9" })).toMatchObject({ ok: false, reason: "internal" }); + expect(judgeBalance({ ...base, label: "", pageText: "Balance: 9" })).toMatchObject({ ok: false, reason: "internal" }); + }); + it("a hostile page cannot make the message long", () => { + const r = judgeBalance({ ...base, pageText: `Balance: ${"9".repeat(5000)}` }); + expect(r.ok || r.message.length < 800).toBe(true); + }); +}); diff --git a/test/unit/reportRender.test.ts b/test/unit/reportRender.test.ts index 2bbe9fe..b8d7cdf 100644 --- a/test/unit/reportRender.test.ts +++ b/test/unit/reportRender.test.ts @@ -216,3 +216,17 @@ describe("several profiles in the report (P5-6)", () => { expect(parse(render()).body.textContent).not.toContain(" as test-"); }); }); + +describe("a pre-state failure in the report (P5-7.a)", () => { + it("says the starting state is wrong and that it is not an app failure", () => { + const run = runJsonFixture(); + const c = run.tests[0]!.steps[2]!.commands[0]!; + c.reason = "pre-state"; + c.found = "3"; + c.saw = ["qian has 3 but this test needs 5 (short by 2)"]; + const why = parse(render(run)).querySelector("article.failed .reason")!.textContent!; + expect(why).toMatch(/starting state is wrong/i); + expect(why).toMatch(/not a failure of the app/i); + expect(why).toContain("(pre-state)"); + }); +}); diff --git a/test/unit/verdictService.test.ts b/test/unit/verdictService.test.ts index e71ec3f..dbe2771 100644 --- a/test/unit/verdictService.test.ts +++ b/test/unit/verdictService.test.ts @@ -254,3 +254,32 @@ describe("P4-5.d: an issue filed with nobody at the keyboard has no picture unle expect(body).not.toContain("SECRETSTATE"); }); }); + +describe("a pre-state failure is not an app bug (P5-7.a)", () => { + const preState = () => { + const fs = files(); + const run = JSON.parse(fs[`${RUN}/run.json`] as string); + const c = run.tests[0].steps[2].commands[0]; + c.reason = "pre-state"; + c.found = "3"; + c.expected = ">= 5"; + c.saw = ["qian has 3 but this test needs 5"]; + fs[`${RUN}/run.json`] = JSON.stringify(run); + return fs; + }; + it("App is broken is not filed: the verdict is kept with the reason, and GitLab is never asked", async () => { + const { c, g, m } = ctx({}, preState()); + const r = await giveVerdict(c, { stepIndex: 2, kind: "app-broken" }); + expect(r.status).toBe("saved-locally"); + expect(r.error).toMatchObject({ kind: "pre-state" }); + expect(r.error!.message).toMatch(/not because the app is broken/); + expect(g.calls).toHaveLength(0); + expect(g.issues).toHaveLength(0); + expect(Object.keys(m.files).some((f) => f.includes("/verdicts/"))).toBe(true); + }); + it("Script is out of date and Flaky still work on it", async () => { + const { c } = ctx({ rerun: async () => ({ exitCode: 0 }) }, preState()); + expect((await giveVerdict(c, { stepIndex: 2, kind: "script-out-of-date" })).status).toBe("written"); + expect((await giveVerdict(c, { stepIndex: 2, kind: "flaky" })).status).toBe("measured"); + }); +}); -- knotmirror 0.0.1