web: capture-set comparator for behaviour-preserving refactors
--compare BEFORE AFTER fails when a capture is missing or unexpected and lists pixel drift for manual review; live durations make byte equality the wrong gate.
This commit is contained in:
+7
-1
@@ -59,7 +59,13 @@ node test/ui-smoke.mjs --out /path/to/run/artifacts # browser, live, scree
|
||||
```
|
||||
|
||||
`ui-smoke.mjs` starts `serve.mjs`, launches the installed Chrome headless, drives
|
||||
the real page and writes PNGs into `--out`. **That is the UI evidence**: a review
|
||||
the real page and writes PNGs into `--out`. It picks a free port unless one is
|
||||
given, so concurrent checks never collide.
|
||||
|
||||
`capture-manifest.mjs` compares two capture directories. A behaviour-preserving
|
||||
refactor must keep the same capture SET; `--compare BEFORE AFTER` exits non-zero
|
||||
when a capture is missing or unexpected and lists the files whose pixels drifted
|
||||
(live durations legitimately change bytes, so inspect that list by eye). **That is the UI evidence**: a review
|
||||
stage must run it (or an extension of it) and report the capture paths, not
|
||||
describe what it thinks would render. Set `CHROME_PATH` if Chrome is not at the
|
||||
default macOS location.
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
// Compare two ui-smoke capture directories.
|
||||
//
|
||||
// node test/capture-manifest.mjs DIR # manifest: name, bytes, sha256
|
||||
// node test/capture-manifest.mjs --compare BEFORE AFTER
|
||||
//
|
||||
// A refactor that must not change behaviour has to produce the SAME capture set:
|
||||
// every screen the check drove before must still be captured after. Pixel bytes
|
||||
// legitimately drift on a live server (durations, "x minutes ago", run counts), so
|
||||
// the gate is the SET, with the differing files listed for a human to inspect:
|
||||
//
|
||||
// exit 0 same capture set (drift printed for review)
|
||||
// exit 1 a capture is missing or unexpected (a screen stopped being driven)
|
||||
//
|
||||
// Zero dependencies, no server needed.
|
||||
import { createHash } from "node:crypto";
|
||||
import { readdir, readFile, stat } from "node:fs/promises";
|
||||
import { basename, resolve } from "node:path";
|
||||
|
||||
async function manifest(directory) {
|
||||
const dir = resolve(directory);
|
||||
const entries = await readdir(dir).catch(() => {
|
||||
throw new Error(`no such capture directory: ${dir}`);
|
||||
});
|
||||
const files = entries.filter((name) => name.toLowerCase().endsWith(".png")).sort();
|
||||
const rows = [];
|
||||
for (const name of files) {
|
||||
const path = `${dir}/${name}`;
|
||||
const info = await stat(path);
|
||||
const bytes = await readFile(path);
|
||||
rows.push({
|
||||
name,
|
||||
bytes: info.size,
|
||||
sha: createHash("sha256").update(bytes).digest("hex").slice(0, 16),
|
||||
});
|
||||
}
|
||||
return { dir, rows };
|
||||
}
|
||||
|
||||
function printManifest({ dir, rows }) {
|
||||
console.log(`# ${dir} — ${rows.length} capture(s)`);
|
||||
for (const row of rows) console.log(`${row.name}\t${row.bytes}\t${row.sha}`);
|
||||
}
|
||||
|
||||
async function compare(beforeDir, afterDir) {
|
||||
const before = await manifest(beforeDir);
|
||||
const after = await manifest(afterDir);
|
||||
const beforeNames = new Set(before.rows.map((row) => row.name));
|
||||
const afterNames = new Set(after.rows.map((row) => row.name));
|
||||
const missing = [...beforeNames].filter((name) => !afterNames.has(name)).sort();
|
||||
const extra = [...afterNames].filter((name) => !beforeNames.has(name)).sort();
|
||||
const beforeByName = new Map(before.rows.map((row) => [row.name, row]));
|
||||
const drift = [];
|
||||
for (const row of after.rows) {
|
||||
const prior = beforeByName.get(row.name);
|
||||
if (!prior || prior.sha === row.sha) continue;
|
||||
drift.push({ name: row.name, before: prior.bytes, after: row.bytes, delta: row.bytes - prior.bytes });
|
||||
}
|
||||
console.log(`before: ${before.dir} — ${before.rows.length} capture(s)`);
|
||||
console.log(`after: ${after.dir} — ${after.rows.length} capture(s)`);
|
||||
console.log(`same set: ${missing.length === 0 && extra.length === 0 ? "yes" : "NO"}`);
|
||||
if (missing.length) console.log(`missing after: ${missing.join(", ")}`);
|
||||
if (extra.length) console.log(`unexpected after: ${extra.join(", ")}`);
|
||||
console.log(
|
||||
`pixel drift: ${drift.length} capture(s)` +
|
||||
(drift.length ? " — inspect these for real visual changes:" : ""),
|
||||
);
|
||||
for (const row of drift) console.log(` ${row.name} ${row.before} -> ${row.after} bytes (${row.delta >= 0 ? "+" : ""}${row.delta})`);
|
||||
if (missing.length || extra.length) process.exitCode = 1;
|
||||
}
|
||||
|
||||
import { pathToFileURL } from "node:url";
|
||||
const runAsScript =
|
||||
process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href;
|
||||
const [first, second, third] = process.argv.slice(2);
|
||||
if (!runAsScript) {
|
||||
// Imported as a module (for example from a test): no CLI side effects.
|
||||
} else if (first === "--compare") {
|
||||
if (!second || !third) {
|
||||
console.error("usage: node test/capture-manifest.mjs --compare BEFORE_DIR AFTER_DIR");
|
||||
process.exit(2);
|
||||
}
|
||||
compare(second, third).catch((error) => {
|
||||
console.error(error.message);
|
||||
process.exit(2);
|
||||
});
|
||||
} else if (first) {
|
||||
manifest(first).then(printManifest).catch((error) => {
|
||||
console.error(error.message);
|
||||
process.exit(2);
|
||||
});
|
||||
} else {
|
||||
console.error("usage: node test/capture-manifest.mjs DIR | --compare BEFORE AFTER");
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
export { compare, manifest };
|
||||
Reference in New Issue
Block a user