b60ad6d98f
Keep the probe suite on the Material composition so browser fixtures and capture drivers no longer render or select Cupertino hosts. Record the final residual legacy arguments and implementation checks for the delivery gate.
395 lines
18 KiB
JavaScript
395 lines
18 KiB
JavaScript
// Real-browser acceptance for the run detail's recorded-visit pane.
|
|
//
|
|
// Serves the probe build (`flutter build web -t tool/run_detail_visit_probe.dart`)
|
|
// on an OS-assigned loopback port and drives the actual RunDetailView over a
|
|
// server-shaped fixture. It checks that a Graph stage click opens the recorded
|
|
// visit in Timeline with its facts and native tabs, that the tabs show the
|
|
// recorded Input, Artifacts, Session and Definition, that the visit's artifact
|
|
// row opens a gallery listing exactly that visit's attributed files, that Back
|
|
// returns to the rows, that a sparse visit does not invent Duration or
|
|
// provider/model, that a visit with no artifact field says the capture is
|
|
// unavailable rather than empty, and that the Session section renders the
|
|
// recorded export state: a complete export with a read-only failed repair and
|
|
// its open option, and a partial export with its state word and recorded
|
|
// reason. It also samples the rendered page so PROBE_SCHEME=dark cannot pass
|
|
// while the fixture silently renders light.
|
|
//
|
|
// PROBE_SCHEME=dark|light PROBE_SCALE=1|2 \
|
|
// PROBE_WIDTH=1280 PROBE_HEIGHT=900 PROBE_WEB_ROOT=<probe build> \
|
|
// OPENCODE_ARTIFACT_DIR=<dir> node tool/run_detail_visit_probe.mjs
|
|
import { createServer } from 'node:http';
|
|
import { spawn } from 'node:child_process';
|
|
import { once } from 'node:events';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { join, extname, resolve, sep } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
|
|
const root = resolve(
|
|
process.env.PROBE_WEB_ROOT ??
|
|
join(process.cwd(), 'build/probe-run-detail-visit'),
|
|
);
|
|
const variant = 'material';
|
|
const scheme = process.env.PROBE_SCHEME ?? 'dark';
|
|
const scale = process.env.PROBE_SCALE ?? '1';
|
|
const width = Number(process.env.PROBE_WIDTH ?? 1280);
|
|
const height = Number(process.env.PROBE_HEIGHT ?? 900);
|
|
const chromePath = process.env.CHROME_PATH ??
|
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
|
|
const artifactDir = process.env.OPENCODE_ARTIFACT_DIR;
|
|
if (!artifactDir) throw new Error('Set OPENCODE_ARTIFACT_DIR for the probe captures');
|
|
await mkdir(artifactDir, { recursive: true });
|
|
const prefix = `run-detail-visit-${variant}-${scheme}-${scale}x-${width}x${height}`;
|
|
const mime = {
|
|
'.html': 'text/html',
|
|
'.js': 'text/javascript',
|
|
'.json': 'application/json',
|
|
'.wasm': 'application/wasm',
|
|
'.png': 'image/png',
|
|
'.ico': 'image/x-icon',
|
|
'.otf': 'font/otf',
|
|
'.ttf': 'font/ttf',
|
|
};
|
|
const server = createServer(async (req, res) => {
|
|
try {
|
|
const path = resolve(root, '.' + decodeURIComponent(new URL(req.url, 'http://localhost').pathname));
|
|
if (path !== root && !path.startsWith(root + sep)) return res.writeHead(403).end();
|
|
const filePath = path === root || path.endsWith('/') ? join(path, 'index.html') : path;
|
|
const file = await readFile(filePath);
|
|
res.writeHead(200, { 'content-type': mime[extname(filePath)] ?? 'application/octet-stream' }).end(file);
|
|
} catch {
|
|
res.writeHead(404).end('not found');
|
|
}
|
|
});
|
|
server.listen(0, '127.0.0.1');
|
|
await once(server, 'listening');
|
|
const port = server.address().port;
|
|
const profile = await mkdtemp(join(tmpdir(), 'run-detail-visit-chrome-'));
|
|
const chrome = spawn(chromePath, [
|
|
'--headless=new', '--no-first-run', '--no-default-browser-check', '--disable-gpu',
|
|
'--remote-debugging-port=0', `--user-data-dir=${profile}`, 'about:blank',
|
|
], { stdio: ['ignore', 'ignore', 'pipe'] });
|
|
let stderr = '';
|
|
chrome.stderr.on('data', (data) => { stderr += data; });
|
|
const waitFor = async (fn) => {
|
|
for (let i = 0; i < 140; i++) {
|
|
const value = await fn();
|
|
if (value) return value;
|
|
await delay(100);
|
|
}
|
|
throw new Error('Timed out waiting for Chrome DevTools');
|
|
};
|
|
let ws;
|
|
try {
|
|
const endpoint = await waitFor(() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1]);
|
|
ws = new WebSocket(endpoint);
|
|
await once(ws, 'open');
|
|
let id = 0;
|
|
const pending = new Map();
|
|
ws.addEventListener('message', (event) => {
|
|
const message = JSON.parse(event.data);
|
|
if (message.method === 'Runtime.exceptionThrown') {
|
|
console.error('PAGE EXCEPTION', JSON.stringify(message.params?.exceptionDetails?.exception?.description));
|
|
}
|
|
if (message.id && pending.has(message.id)) {
|
|
pending.get(message.id)(message);
|
|
pending.delete(message.id);
|
|
}
|
|
});
|
|
const send = (method, params = {}, sessionId) => new Promise((resolveCall) => {
|
|
const callId = ++id;
|
|
pending.set(callId, resolveCall);
|
|
ws.send(JSON.stringify({ id: callId, method, params, ...(sessionId ? { sessionId } : {}) }));
|
|
});
|
|
const created = await send('Target.createTarget', { url: 'about:blank' });
|
|
const target = created.result.targetId;
|
|
const session = (await send('Target.attachToTarget', { targetId: target, flatten: true })).result.sessionId;
|
|
await send('Page.enable', {}, session);
|
|
await send('Runtime.enable', {}, session);
|
|
await send('Emulation.setEmulatedMedia', {
|
|
features: [{ name: 'prefers-color-scheme', value: scheme }],
|
|
}, session);
|
|
await send('Emulation.setDeviceMetricsOverride', {
|
|
width, height, deviceScaleFactor: 1, mobile: width < 600,
|
|
}, session);
|
|
|
|
const evaluate = async (expression) => {
|
|
const response = await send('Runtime.evaluate', {
|
|
expression, returnByValue: true, awaitPromise: true,
|
|
}, session);
|
|
if (response.error) throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
|
|
return response.result?.result?.value;
|
|
};
|
|
const semantics = () => evaluate(`[...document.querySelectorAll('flt-semantics')].map(x => {
|
|
const r = x.getBoundingClientRect();
|
|
return {
|
|
label: (x.getAttribute('aria-label') || (x.innerText || '')).trim(),
|
|
selected: x.getAttribute('aria-selected') || x.getAttribute('aria-current'),
|
|
tappable: x.hasAttribute('flt-tappable'),
|
|
x: r.x, y: r.y, width: r.width, height: r.height,
|
|
visible: r.width > 0 && r.height > 0 && r.bottom > 0 && r.top < innerHeight,
|
|
};
|
|
})`);
|
|
const labels = async () => (await semantics()).map((item) => item.label).filter(Boolean);
|
|
const bodyText = async () => [await evaluate('document.body.innerText'), ...(await labels())].join('\n');
|
|
const waitForText = async (needle, context) => {
|
|
for (let attempt = 0; attempt < 60; attempt++) {
|
|
if ((await bodyText()).includes(needle)) return;
|
|
await delay(250);
|
|
}
|
|
throw new Error(`${context}: missing ${JSON.stringify(needle)}\n${await bodyText()}`);
|
|
};
|
|
const clickItem = async (item) => {
|
|
const x = item.x + item.width / 2;
|
|
const y = item.y + item.height / 2;
|
|
await send('Input.dispatchMouseEvent', {
|
|
type: 'mousePressed', x, y, button: 'left', clickCount: 1,
|
|
}, session);
|
|
await send('Input.dispatchMouseEvent', {
|
|
type: 'mouseReleased', x, y, button: 'left', clickCount: 1,
|
|
}, session);
|
|
await delay(1000);
|
|
};
|
|
const findVisible = async (predicate) => {
|
|
const items = (await semantics()).filter((item) => item.visible && predicate(item));
|
|
items.sort((a, b) => a.width * a.height - b.width * b.height);
|
|
return items[0] ?? null;
|
|
};
|
|
const clickSemantics = async (wanted, { lowest = false, prefix: asPrefix = false } = {}) => {
|
|
const clicked = await evaluate(`(() => {
|
|
const wanted = ${JSON.stringify(wanted)};
|
|
const lowest = ${lowest ? 'true' : 'false'};
|
|
const asPrefix = ${asPrefix ? 'true' : 'false'};
|
|
const text = (x) => (x.getAttribute('aria-label') || x.innerText || '').trim();
|
|
const matches = [...document.querySelectorAll('flt-semantics')]
|
|
.filter((x) => x.hasAttribute('flt-tappable'))
|
|
.filter((x) => asPrefix ? text(x).startsWith(wanted) : text(x) === wanted);
|
|
if (!matches.length) return false;
|
|
matches.sort((a, b) => {
|
|
const ra = a.getBoundingClientRect();
|
|
const rb = b.getBoundingClientRect();
|
|
if (lowest) return rb.bottom - ra.bottom;
|
|
return ra.width * ra.height - rb.width * rb.height;
|
|
});
|
|
matches[0].click();
|
|
return true;
|
|
})()`);
|
|
if (!clicked) throw new Error(`Missing tappable ${JSON.stringify(wanted)}: ${JSON.stringify(await labels())}`);
|
|
await delay(1000);
|
|
};
|
|
const clickLabel = (label, options) => clickSemantics(label, options);
|
|
const clickPrefix = (prefixText) => clickSemantics(prefixText, { prefix: true });
|
|
const scrollPage = async (deltaY) => {
|
|
await send('Input.dispatchMouseEvent', {
|
|
type: 'mouseWheel', x: width / 2, y: height - 24, deltaX: 0, deltaY,
|
|
}, session);
|
|
await delay(700);
|
|
};
|
|
// A short tab body at 320/2x builds its rows lazily: scroll toward the row
|
|
// before requiring it, and leave the page there so the capture follows the
|
|
// facts. Semantics can include clipped rows, so the 320 capture proves the
|
|
// rows are in the tree; legibility at rest is checked by the UI review.
|
|
const ensureText = async (needle, context) => {
|
|
for (let attempt = 0; attempt < 8; attempt++) {
|
|
if ((await bodyText()).includes(needle)) return;
|
|
await scrollPage(240);
|
|
}
|
|
throw new Error(`${context}: missing ${JSON.stringify(needle)}\n${await bodyText()}`);
|
|
};
|
|
const capture = async (name) => {
|
|
const data = (await send('Page.captureScreenshot', {
|
|
format: 'png', captureBeyondViewport: false,
|
|
}, session)).result.data;
|
|
const file = `${prefix}-${name}.png`;
|
|
await writeFile(join(artifactDir, file), Buffer.from(data, 'base64'));
|
|
return file;
|
|
};
|
|
// Mean luminance of the actual render. The fixture receives the scheme through
|
|
// its own query parameter, so a mismatch between PROBE_SCHEME and the painted
|
|
// theme (as happened when the driver sent theme=1 and the fixture read
|
|
// 'dark') must fail rather than label a light capture as dark.
|
|
const pageLuminance = async () => {
|
|
const shot = (await send('Page.captureScreenshot', {
|
|
format: 'png', captureBeyondViewport: false,
|
|
}, session)).result.data;
|
|
return evaluate(`(async () => {
|
|
const img = new Image();
|
|
img.src = 'data:image/png;base64,${shot}';
|
|
await img.decode();
|
|
const canvas = document.createElement('canvas');
|
|
canvas.width = img.width;
|
|
canvas.height = img.height;
|
|
const ctx = canvas.getContext('2d');
|
|
ctx.drawImage(img, 0, 0);
|
|
const data = ctx.getImageData(0, 0, img.width, img.height).data;
|
|
let sum = 0;
|
|
let samples = 0;
|
|
for (let y = 0; y < img.height; y += 8) {
|
|
for (let x = 0; x < img.width; x += 8) {
|
|
const i = (y * img.width + x) * 4;
|
|
sum += (0.2126 * data[i] + 0.7152 * data[i + 1] + 0.0722 * data[i + 2]) / 255;
|
|
samples += 1;
|
|
}
|
|
}
|
|
return { mean: sum / samples, samples };
|
|
})()`);
|
|
};
|
|
const evidence = { variant, scheme, scale, width, height, checks: [], captures: [] };
|
|
const check = (text, value) => evidence.checks.push({ check: text, value });
|
|
|
|
const navigation = await send('Page.navigate', {
|
|
url: `http://127.0.0.1:${port}/?theme=${scheme}&scale=${scale}`,
|
|
}, session);
|
|
if (navigation.error || navigation.result?.errorText) throw new Error(JSON.stringify(navigation));
|
|
await delay(3200);
|
|
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
|
|
await delay(900);
|
|
await waitForText('Release verification', 'fixture load');
|
|
check('the real run detail opened over the recorded fixture', true);
|
|
|
|
const render = await pageLuminance();
|
|
evidence.pageLuminance = render.mean;
|
|
evidence.pageLuminanceSamples = render.samples;
|
|
const renderedDark = render.mean < 0.5;
|
|
if (renderedDark !== (scheme === 'dark')) {
|
|
throw new Error(
|
|
`PROBE_SCHEME=${scheme} painted the other theme (mean luminance ${render.mean.toFixed(3)})`,
|
|
);
|
|
}
|
|
check(`the ${scheme} scheme actually renders ${scheme === 'dark' ? 'dark' : 'light'}`, true);
|
|
|
|
await clickLabel('Graph');
|
|
await waitForText('Executed', 'graph legend');
|
|
await clickPrefix('plan');
|
|
await waitForText('plan · visit 1', 'visit pane after graph click');
|
|
await waitForText('Provider/model', 'visit facts');
|
|
await waitForText('anthropic / claude-x', 'visit facts');
|
|
await waitForText('2m 5s', 'recorded duration');
|
|
await waitForText('Plan checked.', 'recorded diagnostic');
|
|
await waitForText('approve', 'recorded verdict');
|
|
const paneLabels = await labels();
|
|
for (const section of ['Input', 'Output', 'Artifacts', 'Session', 'Definition']) {
|
|
if (!paneLabels.includes(section)) {
|
|
throw new Error(`Visit section ${section} is missing: ${JSON.stringify(paneLabels)}`);
|
|
}
|
|
}
|
|
check('Graph stage click opens the recorded visit pane with its tabs', true);
|
|
evidence.captures.push(await capture('visit-pane'));
|
|
|
|
await clickLabel('Definition', { lowest: true });
|
|
await waitForText('Type model', 'authored definition');
|
|
await waitForText('Prompt', 'authored definition prompt');
|
|
await waitForText('Draft the plan.', 'pinned prompt');
|
|
check('Definition shows the pinned authored source', true);
|
|
evidence.captures.push(await capture('definition'));
|
|
|
|
await clickLabel('Session', { lowest: true });
|
|
await waitForText('sess-1', 'recorded session pointer');
|
|
await waitForText('Open session.html', 'session export link');
|
|
check('Session shows the recorded pointer and export open', true);
|
|
evidence.captures.push(await capture('session'));
|
|
|
|
await clickLabel('Artifacts', { lowest: true });
|
|
await waitForText('plan.csv', 'visit artifact');
|
|
check('Artifacts lists the visit recorded files', true);
|
|
evidence.captures.push(await capture('artifacts'));
|
|
|
|
await clickLabel('Back to timeline');
|
|
await waitForText('build · visit 2', 'timeline rows after back');
|
|
if ((await bodyText()).includes('Plan checked.')) {
|
|
throw new Error('Back left the visit pane open');
|
|
}
|
|
check('Back returns to the recorded rows', true);
|
|
evidence.captures.push(await capture('timeline-rows'));
|
|
|
|
await clickPrefix('build · visit 2');
|
|
await waitForText('build · visit 2', 'sparse visit pane');
|
|
const sparse = await bodyText();
|
|
for (const absent of ['Provider/model', 'Duration', '2m 5s', 'anthropic / claude-x']) {
|
|
if (sparse.includes(absent)) {
|
|
throw new Error(`A sparse visit invented ${JSON.stringify(absent)}\n${sparse}`);
|
|
}
|
|
}
|
|
check('a sparse visit does not invent duration or provider/model', true);
|
|
evidence.captures.push(await capture('sparse-visit'));
|
|
|
|
await clickLabel('Artifacts', { lowest: true });
|
|
await waitForText('Artifacts · capture unavailable', 'sparse capture state');
|
|
check('a visit without an artifact field says capture unavailable', true);
|
|
evidence.captures.push(await capture('sparse-artifacts'));
|
|
|
|
await clickLabel('Back to timeline');
|
|
await waitForText('plan · visit 1', 'rows before the export-state checks');
|
|
await clickPrefix('audit · visit 3');
|
|
await waitForText('audit · visit 3', 'recorded export visit');
|
|
await clickLabel('Session', { lowest: true });
|
|
await ensureText('Open repaired export', 'recorded repair option');
|
|
await ensureText(
|
|
'failed · 0 messages. The repair read failed.',
|
|
'recorded repair label',
|
|
);
|
|
await ensureText('Open session.html', 'recorded export open');
|
|
const auditText = await bodyText();
|
|
if (!auditText.includes('Export complete')) {
|
|
throw new Error(`The complete export state is missing\n${auditText}`);
|
|
}
|
|
if (auditText.includes('Reason')) {
|
|
throw new Error(`A complete export showed a Reason row\n${auditText}`);
|
|
}
|
|
check('a complete export shows its read-only repair attempt and option', true);
|
|
evidence.captures.push(await capture('export-complete-repair'));
|
|
|
|
await clickLabel('Back to timeline');
|
|
await waitForText('plan · visit 1', 'rows before the partial export check');
|
|
await ensureText('partial · visit 4', 'partial export row');
|
|
await clickPrefix('partial · visit 4');
|
|
await waitForText('partial · visit 4', 'partial export visit');
|
|
await clickLabel('Session', { lowest: true });
|
|
await ensureText(
|
|
'This export is failed; full session evidence is not available.',
|
|
'partial export sentence',
|
|
);
|
|
await ensureText('The native read was refused.', 'partial export reason');
|
|
const partialText = await bodyText();
|
|
if (!partialText.includes('Export partial')) {
|
|
throw new Error(`The partial export state is missing\n${partialText}`);
|
|
}
|
|
if (!partialText.includes('Reason The native read was refused.')) {
|
|
throw new Error(`The partial export reason is missing\n${partialText}`);
|
|
}
|
|
check('a partial export shows its state word and recorded reason', true);
|
|
evidence.captures.push(await capture('export-partial-reason'));
|
|
|
|
await clickLabel('Back to timeline');
|
|
await waitForText('plan · visit 1', 'rows before the gallery check');
|
|
await clickPrefix('plan · visit 1');
|
|
await waitForText('Provider/model', 'plan visit for the gallery check');
|
|
await clickLabel('Artifacts', { lowest: true });
|
|
await waitForText('plan.csv', 'plan visit artifacts');
|
|
await clickPrefix('plan.csv');
|
|
await waitForText('Artifacts · 1 recorded', 'visit gallery summary');
|
|
await waitForText('plan · visit 1', 'visit gallery group');
|
|
const galleryText = await bodyText();
|
|
if (galleryText.includes('session.html')) {
|
|
throw new Error(`The visit gallery listed the session evidence file\n${galleryText}`);
|
|
}
|
|
check('the artifact row opens exactly the visit attributed group', true);
|
|
evidence.captures.push(await capture('visit-gallery'));
|
|
|
|
console.log(JSON.stringify(evidence, null, 2));
|
|
} finally {
|
|
if (ws?.readyState === WebSocket.OPEN) {
|
|
ws.send(JSON.stringify({ id: 2147483647, method: 'Browser.close' }));
|
|
}
|
|
await Promise.race([once(chrome, 'exit'), delay(3000)]);
|
|
ws?.close();
|
|
if (chrome.exitCode === null) chrome.kill('SIGTERM');
|
|
chrome.stderr.destroy();
|
|
chrome.unref();
|
|
await new Promise((resolveClose) => {
|
|
server.close(resolveClose);
|
|
server.closeAllConnections();
|
|
});
|
|
await rm(profile, { recursive: true, force: true });
|
|
}
|