Files
pipelines/app/tool/changes_probe.mjs
alan b60ad6d98f Remove alternate hosts from tooling probes
Keep the probe suite on the Material composition so browser fixtures and capture drivers no longer render or select Cupertino hosts. Record the final residual legacy arguments and implementation checks for the delivery gate.
2026-11-11 21:38:30 +00:00

588 lines
24 KiB
JavaScript

// Real-browser evidence for the Changes viewer fixture.
//
// Usage (from app/):
// flutter build web --target tool/changes_probe_entry.dart --output build/probe-web
// node tool/changes_probe.mjs --root build/probe-web --out DIR
//
// Serves the fixture over loopback with an explicit text/html index, drives
// headless Chrome through the Chrome DevTools Protocol, enables Flutter
// semantics, clicks exact labels, verifies the body/labels change, copies the
// raw patch and writes real PNG captures at phone and desktop widths.
//
// The Changes page shows one recorded visit at a time, so visit-only evidence
// (capture states, visit numbers) is collected by paging with Previous/Next
// rather than from a single body snapshot.
import { createServer } from 'node:http';
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { spawn } from 'node:child_process';
import { once } from 'node:events';
import { setTimeout as delay } from 'node:timers/promises';
import { extname, join, resolve, sep } from 'node:path';
import { tmpdir } from 'node:os';
const args = new Map();
for (let i = 2; i < process.argv.length; i++) {
const value = process.argv[i];
if (value.startsWith('--')) args.set(value.slice(2), process.argv[++i]);
}
const root = resolve(args.get('root') ?? 'build/probe-web');
const out = resolve(args.get('out') ?? 'build/changes-probe');
const chromePath =
process.env.CHROME_PATH ?? '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
await mkdir(out, { recursive: true });
const mime = {
'.html': 'text/html; charset=utf-8',
'.js': 'text/javascript; charset=utf-8',
'.mjs': 'text/javascript; charset=utf-8',
'.json': 'application/json; charset=utf-8',
'.wasm': 'application/wasm',
'.png': 'image/png',
'.svg': 'image/svg+xml',
'.ico': 'image/x-icon',
'.ttf': 'font/ttf',
'.otf': 'font/otf',
'.woff': 'font/woff',
'.woff2': 'font/woff2',
'.bin': 'application/octet-stream',
'.frag': 'application/octet-stream',
};
const server = createServer(async (req, res) => {
try {
const pathname = decodeURIComponent(new URL(req.url, 'http://localhost').pathname);
let filePath = resolve(root, '.' + pathname);
if (filePath !== root && !filePath.startsWith(root + sep)) {
res.writeHead(403).end();
return;
}
if (pathname === '/' || pathname.endsWith('/')) {
filePath = join(filePath, 'index.html');
}
const file = await readFile(filePath);
const type = mime[extname(filePath)] ?? 'application/octet-stream';
res.writeHead(200, { 'content-type': type }).end(file);
} catch {
res.writeHead(404).end('not found');
}
});
server.listen(0, '127.0.0.1');
await once(server, 'listening');
const port = server.address().port;
const origin = `http://127.0.0.1:${port}`;
const profile = await mkdtemp(join(tmpdir(), 'd1-changes-chrome-'));
const chrome = spawn(
chromePath,
[
'--headless=new',
'--no-first-run',
'--no-default-browser-check',
'--disable-gpu',
'--remote-debugging-port=0',
`--user-data-dir=${profile}`,
'about:blank',
],
{ stdio: ['ignore', 'ignore', 'pipe'] },
);
let stderr = '';
chrome.stderr.on('data', (data) => {
stderr += data;
});
const waitFor = async (fn, what) => {
for (let i = 0; i < 120; i++) {
const value = await fn();
if (value) return value;
await delay(100);
}
throw new Error(`Timed out waiting for ${what}: ${stderr.slice(-1500)}`);
};
let ws;
const evidence = { origin, captures: [], checks: [] };
try {
const endpoint = await waitFor(
() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1],
'Chrome DevTools',
);
ws = new WebSocket(endpoint);
await once(ws, 'open');
let id = 0;
const pending = new Map();
ws.addEventListener('message', (event) => {
const message = JSON.parse(event.data);
if (message.id && pending.has(message.id)) {
pending.get(message.id)(message);
pending.delete(message.id);
}
});
const send = (method, params = {}, sessionId) =>
new Promise((resolveCall) => {
const callId = ++id;
pending.set(callId, resolveCall);
ws.send(JSON.stringify({ id: callId, method, params, ...(sessionId ? { sessionId } : {}) }));
});
const created = await send('Target.createTarget', { url: 'about:blank' });
if (!created.result) throw new Error(`Target creation failed: ${JSON.stringify(created)}`);
const target = created.result.targetId;
const session = (await send('Target.attachToTarget', { targetId: target, flatten: true })).result
.sessionId;
await send('Page.enable', {}, session);
await send('Runtime.enable', {}, session);
await send('Emulation.setDeviceMetricsOverride', {
width: 390,
height: 844,
deviceScaleFactor: 1,
mobile: true,
}, session);
const evaluate = async (expression) => {
const response = await send(
'Runtime.evaluate',
{ expression, returnByValue: true, awaitPromise: true },
session,
);
if (response.error) throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
return response.result?.result?.value;
};
const labels = () =>
evaluate(
"[...document.querySelectorAll('flt-semantics')].map(x => x.getAttribute('aria-label') || (x.innerText || '').trim()).filter(Boolean)",
);
const bodyText = () => evaluate('document.body.innerText');
const hasLabel = async (label) => (await labels()).includes(label);
const clickExact = async (label) => {
const clicked = await evaluate(`(() => {
const wanted = ${JSON.stringify(label)};
const nodes = [...document.querySelectorAll('flt-semantics')];
const node = nodes.find(x => x.getAttribute('aria-label') === wanted)
|| nodes.find(x => (x.innerText || '').trim() === wanted);
if (!node) return false;
node.click();
return true;
})()`);
if (!clicked) throw new Error(`No exact semantics label ${JSON.stringify(label)}: ${JSON.stringify(await labels())}`);
await delay(450);
};
// File rows announce their visible kind and meta after the path, so click by
// the stable action+path prefix rather than a fully spelled label.
const clickPrefix = async (prefix) => {
const clicked = await evaluate(`(() => {
const wanted = ${JSON.stringify(prefix)};
const nodes = [...document.querySelectorAll('flt-semantics')];
// A container's innerText also starts with its first row, so click the
// most specific (shortest-text) node that starts with the row prefix.
const candidates = nodes.filter((x) =>
(x.getAttribute('aria-label') || '').startsWith(wanted)
|| (x.innerText || '').trim().startsWith(wanted));
if (candidates.length === 0) return false;
candidates.sort((a, b) => (a.innerText || '').length - (b.innerText || '').length);
candidates[0].click();
return true;
})()`);
if (!clicked) throw new Error(`No semantics label starting ${JSON.stringify(prefix)}: ${JSON.stringify(await labels())}`);
await delay(450);
};
// The disabled Previous/Next is a semantics button with aria-disabled; the
// pager must stop at both ends instead of wrapping.
const buttonDisabled = (label) => evaluate(`(() => {
const wanted = ${JSON.stringify(label)};
const node = [...document.querySelectorAll('flt-semantics[role="button"]')]
.find(x => (x.getAttribute('aria-label') || (x.innerText || '').trim()) === wanted);
return node ? node.getAttribute('aria-disabled') === 'true' : null;
})()`);
// Clipboard writes need a trusted user gesture, which a scripted semantics
// `.click()` is not; dispatch real mouse events at the control's centre.
const tapExact = async (label) => {
const point = await evaluate(`(() => {
const wanted = ${JSON.stringify(label)};
const nodes = [...document.querySelectorAll('flt-semantics')];
const node = nodes.find(x => x.getAttribute('aria-label') === wanted)
|| nodes.find(x => (x.innerText || '').trim() === wanted);
if (!node) return null;
const rect = node.getBoundingClientRect();
return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
})()`);
if (!point) throw new Error(`No semantics node for ${JSON.stringify(label)}: ${JSON.stringify(await labels())}`);
for (const type of ['mousePressed', 'mouseReleased']) {
await send('Input.dispatchMouseEvent', { type, x: point.x, y: point.y, button: 'left', clickCount: 1 }, session);
}
await delay(450);
};
const visitPage = (text) => text.match(/\d+ of \d+ recorded visits/)?.[0] ?? null;
const visitStage = (text, stage) => text.includes(`\n${stage}\n`);
const headerLine = (text) =>
text.split('\n').find((line) => line.startsWith('Changes · ')) ?? null;
const activate = async () => {
await delay(3500);
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
await delay(1000);
const current = await labels();
if (!current.some((label) => label.includes('Changes probe'))) {
throw new Error(`Flutter semantics unavailable: ${JSON.stringify(current)}`);
}
};
const navigate = async (url) => {
const navigation = await send('Page.navigate', { url }, session);
if (navigation.error || navigation.result?.errorText) throw new Error(JSON.stringify(navigation));
await activate();
};
const capture = async (name, width, height, scale = 1) => {
await send('Emulation.setDeviceMetricsOverride', {
width,
height,
deviceScaleFactor: scale,
mobile: width < 600,
}, session);
await delay(350);
const data = (await send('Page.captureScreenshot', { format: 'png' }, session)).result.data;
const file = join(out, `${name}.png`);
await writeFile(file, Buffer.from(data, 'base64'));
evidence.captures.push(file);
return file;
};
await navigate(`${origin}/`);
const contentType = await evaluate(
"fetch('/').then(r => r.headers.get('content-type'))",
);
evidence.checks.push({ check: 'index content-type is text/html', value: contentType });
if (!String(contentType).toLowerCase().startsWith('text/html')) {
throw new Error(`Root index is not text/html: ${contentType}`);
}
// Diff text inherits native host typography; it must not add a custom face.
const manifest = await evaluate("fetch('/assets/FontManifest.json').then(r => r.json())");
const codeEntry = (Array.isArray(manifest) ? manifest : []).find(
(entry) => entry.family === 'RobotoMono',
);
if (codeEntry) throw new Error('Diff build still overrides native typography');
evidence.checks.push({
check: 'the web build does not register a custom diff font',
});
const firstLabels = await labels();
const firstBody = await bodyText();
const firstRow = 'Open diff for lib/features/changes/domain/unified_patch.dart';
const firstRowPresent = firstLabels.some((label) => label.startsWith(firstRow));
// The run-wide aggregate header and the pager describe different scopes: the
// header covers every visit, the pager position scopes the shown visit.
const firstPage = {
header: headerLine(firstBody),
page: visitPage(firstBody),
stage: visitStage(firstBody, 'implementer'),
visit: firstBody.includes('Visit 1 · completed'),
previousDisabled: await buttonDisabled('Previous visit'),
nextDisabled: await buttonDisabled('Next visit'),
};
evidence.checks.push({
check: 'page 1 opens on the first recorded visit',
...firstPage,
fileRow: firstRowPresent,
});
if (
firstPage.header !== 'Changes · capture unavailable' ||
firstPage.page !== '1 of 5 recorded visits' ||
!firstPage.stage ||
!firstPage.visit
) {
throw new Error(`Run summary or first visit missing: ${JSON.stringify(firstBody)}`);
}
if (firstPage.previousDisabled !== true || firstPage.nextDisabled !== false) {
throw new Error(`Page 1 pager did not clamp at the start: ${JSON.stringify(firstPage)}`);
}
if (!firstRowPresent) throw new Error('Page 1 is missing the implementer file row');
await capture('d1-changes-phone-list', 390, 844);
// Page through every recorded visit. Each page keeps its own stage/visit
// heading and the run-wide header, and Next/Previous clamp at both ends.
await clickExact('Next visit');
const secondBody = await bodyText();
const secondPage = {
page: visitPage(secondBody),
stage: visitStage(secondBody, 'reviewer'),
visit: secondBody.includes('Visit 1 · completed'),
headerUnchanged: headerLine(secondBody) === firstPage.header,
previousDisabled: await buttonDisabled('Previous visit'),
rowPresent: (await labels()).some((label) =>
label.startsWith('Open diff for lib/features/changes/widgets/file_diff_viewer.dart'),
),
};
evidence.checks.push({ check: 'Next opens the next recorded visit', ...secondPage });
if (
secondPage.page !== '2 of 5 recorded visits' ||
!secondPage.stage ||
!secondPage.visit ||
!secondPage.headerUnchanged ||
secondPage.previousDisabled !== false ||
!secondPage.rowPresent
) {
throw new Error(`Second visit page is wrong: ${JSON.stringify(secondBody)}`);
}
await capture('d1-changes-phone-page2', 390, 844);
await clickExact('Next visit');
const missingBody = await bodyText();
evidence.checks.push({
check: 'a later visit reports a missing capture',
page: visitPage(missingBody),
stage: visitStage(missingBody, 'ui_reviewer'),
captureUnavailable: missingBody.includes('Capture unavailable'),
});
if (
visitPage(missingBody) !== '3 of 5 recorded visits' ||
!visitStage(missingBody, 'ui_reviewer') ||
!missingBody.includes('Capture unavailable')
) {
throw new Error(`Missing-capture visit page is wrong: ${JSON.stringify(missingBody)}`);
}
await capture('d1-changes-phone-page3-missing', 390, 844);
await clickExact('Next visit');
const unavailableBody = await bodyText();
evidence.checks.push({
check: 'a later visit reports an unavailable capture',
page: visitPage(unavailableBody),
stage: visitStage(unavailableBody, 'committer'),
diagnostic: unavailableBody.includes('Workspace snapshot unavailable'),
});
if (
visitPage(unavailableBody) !== '4 of 5 recorded visits' ||
!visitStage(unavailableBody, 'committer') ||
!unavailableBody.includes('Workspace snapshot unavailable')
) {
throw new Error(`Unavailable-capture visit page is wrong: ${JSON.stringify(unavailableBody)}`);
}
await capture('d1-changes-phone-page4-unavailable', 390, 844);
await clickExact('Next visit');
const lastBody = await bodyText();
evidence.checks.push({
check: 'the last visit is a recorded no-change and Next is disabled',
page: visitPage(lastBody),
stage: visitStage(lastBody, 'verifier'),
empty: lastBody.includes('No changes observed'),
nextDisabled: await buttonDisabled('Next visit'),
});
if (
visitPage(lastBody) !== '5 of 5 recorded visits' ||
!visitStage(lastBody, 'verifier') ||
!lastBody.includes('No changes observed') ||
(await buttonDisabled('Next visit')) !== true
) {
throw new Error(`Last visit page is wrong: ${JSON.stringify(lastBody)}`);
}
await capture('d1-changes-phone-page5-last', 390, 844);
await clickExact('Previous visit');
const previousBody = await bodyText();
evidence.checks.push({
check: 'Previous returns to the prior visit',
page: visitPage(previousBody),
stage: visitStage(previousBody, 'committer'),
});
if (visitPage(previousBody) !== '4 of 5 recorded visits') {
throw new Error(`Previous lost the visit: ${JSON.stringify(previousBody)}`);
}
await navigate(`${origin}/`);
const before = await bodyText();
await clickPrefix('Open diff for lib/features/changes/domain/unified_patch.dart');
const after = await bodyText();
if (after === before) throw new Error('Opening the diff did not change the document body');
for (const label of ['Rendered', 'Raw patch', 'Copy patch', 'Close diff']) {
if (!(await hasLabel(label))) throw new Error(`Open diff is missing ${label}: ${JSON.stringify(await labels())}`);
}
evidence.checks.push({ check: 'body changed and diff controls appeared', bodyChanged: true });
const renderedFile = await capture('d1-changes-phone-diff', 390, 844);
const renderedBody = await bodyText();
await clickExact('Raw patch');
const rawSemantics = (await labels()).some((label) =>
label.startsWith('Raw patch for lib/features/changes/domain/unified_patch.dart'),
);
if (!rawSemantics) {
throw new Error(`Raw patch view semantics missing: ${JSON.stringify(await labels())}`);
}
const rawBody = await bodyText();
const rawFile = await capture('d1-changes-phone-raw', 390, 844);
const [renderedBytes, rawBytes] = await Promise.all([
readFile(renderedFile),
readFile(rawFile),
]);
if (renderedBytes.equals(rawBytes)) {
throw new Error('Raw patch capture is identical to the rendered capture');
}
evidence.checks.push({
check: 'Raw patch switched the body and the rendered pixels',
bodyChanged: rawBody !== renderedBody,
pixelsChanged: true,
rawSemantics,
});
await send('Browser.grantPermissions', {
origin,
permissions: ['clipboardReadWrite', 'clipboardSanitizedWrite'],
});
await tapExact('Copy patch');
let clipboard = null;
try {
clipboard = await evaluate('navigator.clipboard.readText()');
} catch (error) {
evidence.checks.push({ check: 'clipboard readback unavailable', value: String(error) });
}
if (clipboard != null) {
const startsWithHeader = String(clipboard).startsWith(
'--- "a/lib/features/changes/domain/unified_patch.dart"',
);
if (!startsWithHeader) throw new Error('Clipboard did not contain the recorded raw patch');
evidence.checks.push({
check: 'Copy patch copied the raw patch',
startsWithHeader,
length: String(clipboard).length,
});
}
await capture('d1-changes-phone-copied', 390, 844);
await clickExact('Close diff');
if (await hasLabel('Raw patch')) throw new Error('Close diff left the viewer open');
evidence.checks.push({ check: 'Close diff returned to the list', bodyChanged: (await bodyText()) !== after });
// Filter changed filenames through the real text field when Flutter exposes
// it as a semantics input; record honestly when it does not.
const focused = await evaluate(`(() => {
const nodes = [...document.querySelectorAll('input, textarea')];
const node = nodes.find(x => ((x.getAttribute('aria-label') || '') + ' ' + (x.placeholder || '')).toLowerCase().includes('filter changed files'));
if (!node) return false;
node.focus();
node.click();
return true;
})()`);
if (focused) {
await send('Input.insertText', { text: 'unified' }, session);
await delay(500);
const filtered = await labels();
// A row is one single-line semantics node; a list container repeats its
// rows with newlines and must not be counted as an extra match.
const rows = (prefix) =>
filtered.filter((label) => label.startsWith(prefix) && !label.includes('\n'));
const unifiedRows = rows('Open diff for lib/features/changes/domain/unified_patch.dart');
const binaryRows = rows('Open diff for assets/pipelines-mark.png');
evidence.checks.push({
check: 'filtering by changed filename',
unifiedRows: unifiedRows.length,
binaryRows: binaryRows.length,
});
if (unifiedRows.length === 0) {
throw new Error('Filtering removed the matching file');
}
if (binaryRows.length !== 0) {
throw new Error('Filtering left a non-matching file visible');
}
await capture('d1-changes-phone-filtered', 390, 844);
} else {
evidence.checks.push({ check: 'filter field not exposed as a semantics input', value: false });
}
// Visit numbering: two stages appear twice in this compact fixture. Each
// stage numbers its own visits, so paging must collect one number per page:
// the implementer's visits are position-derived (1, 2) and the resumed
// reviewer visit records control.visit 6, which must win over its position.
await navigate(`${origin}/?visits=1`);
const visitSequence = [];
for (let step = 0; step < 4; step++) {
const current = await bodyText();
const stage = visitStage(current, 'implementer')
? 'implementer'
: visitStage(current, 'reviewer')
? 'reviewer'
: '?';
visitSequence.push(`${stage}:${current.match(/Visit (\d+) ·/)?.[1] ?? '?'}`);
if (step < 3) await clickExact('Next visit');
}
const lastVisitBody = await bodyText();
evidence.checks.push({
check: 'paged visit numbering keeps per-stage numbers and control.visit',
sequence: visitSequence,
page: visitPage(lastVisitBody),
nextDisabled: await buttonDisabled('Next visit'),
});
if (visitSequence.join(',') !== 'implementer:1,reviewer:1,implementer:2,reviewer:6') {
throw new Error(`Visit numbers are not paged per stage: ${JSON.stringify(visitSequence)}`);
}
if (visitPage(lastVisitBody) !== '4 of 4 recorded visits') {
throw new Error(`Visit fixture did not reach its last page: ${JSON.stringify(lastVisitBody)}`);
}
const visitFile = await capture('d1-changes-phone-visits', 390, 844);
await navigate(`${origin}/`);
await capture('d1-changes-desktop-list', 1280, 800);
// The reviewer's file row lives on the second visit page.
await clickExact('Next visit');
await capture('d1-changes-desktop-page2', 1280, 800);
await clickPrefix('Open diff for lib/features/changes/widgets/file_diff_viewer.dart');
await capture('d1-changes-desktop-diff', 1280, 800);
await clickExact('Close diff');
// Presentation revision: the light theme and a narrow Material surface, at
// 2x so glyph advance, line tints and the popup inset are inspectable.
await navigate(`${origin}/?theme=light`);
await capture('d1-changes-phone-light-list', 390, 844);
await clickPrefix('Open diff for lib/features/changes/domain/unified_patch.dart');
await capture('d1-changes-phone-light-diff', 390, 844);
await clickExact('Raw patch');
await capture('d1-changes-phone-light-raw', 390, 844);
await clickExact('Rendered');
// The 2x capture is last in its state: Chrome only re-lays out Flutter when
// the CSS size changes, so scaling back down on the same page would capture a
// half-size frame.
await capture('d1-changes-phone-light-diff-2x', 390, 844, 2);
await navigate(`${origin}/?theme=light`);
await capture('d1-changes-desktop-light-list', 1280, 800);
// The reviewer's row is on the second visit page in the light theme too.
await clickExact('Next visit');
await clickPrefix('Open diff for lib/features/changes/widgets/file_diff_viewer.dart');
await capture('d1-changes-desktop-light-diff', 1280, 800);
await clickExact('Close diff');
await navigate(`${origin}/?theme=light`);
await capture('d1-changes-material-light-list', 390, 844);
// The Material pager uses native buttons; paging keeps the visit heading.
await clickExact('Next visit');
const materialSecond = await bodyText();
evidence.checks.push({
check: 'Material pages to the second visit',
page: visitPage(materialSecond),
stage: visitStage(materialSecond, 'reviewer'),
});
if (visitPage(materialSecond) !== '2 of 5 recorded visits') {
throw new Error(`Material pager did not advance: ${JSON.stringify(materialSecond)}`);
}
await capture('d1-changes-material-light-page2', 390, 844);
await clickExact('Previous visit');
await clickPrefix('Open diff for lib/features/changes/domain/unified_patch.dart');
await capture('d1-changes-material-light-diff', 390, 844);
await capture('d1-changes-material-light-diff-2x', 390, 844, 2);
await navigate(`${origin}/`);
const materialLabels = await labels();
if (!materialLabels.some((label) => label.includes('Changes probe'))) {
throw new Error('Material fixture did not load');
}
await clickPrefix('Open diff for lib/features/changes/domain/unified_patch.dart');
await capture('d1-changes-material-diff', 1280, 800);
evidence.checks.push({ check: 'Material viewer opened', value: true });
evidence.checks.push({ check: 'visit capture written', value: visitFile });
console.log(JSON.stringify(evidence, null, 2));
} finally {
ws?.close();
chrome.kill('SIGTERM');
await Promise.race([once(chrome, 'exit'), delay(3000)]);
server.close();
await rm(profile, { recursive: true, force: true });
}