3ee5d0e407
Split the export model/classification from the session wording, add the no-session, selected-repair and missing-evidence coverage, and update the parity matrix and R4 note. The visit probe now passes the scheme name its fixture reads, asserts the painted theme before capture and re-captures the export states.
395 lines
18 KiB
JavaScript
395 lines
18 KiB
JavaScript
// Real-browser acceptance for the run detail's recorded-visit pane.
|
|
//
|
|
// Serves the probe build (`flutter build web -t tool/run_detail_visit_probe.dart`)
|
|
// on an OS-assigned loopback port and drives the actual RunDetailView over a
|
|
// server-shaped fixture. It checks that a Graph stage click opens the recorded
|
|
// visit in Timeline with its facts and native tabs, that the tabs show the
|
|
// recorded Input, Artifacts, Session and Definition, that the visit's artifact
|
|
// row opens a gallery listing exactly that visit's attributed files, that Back
|
|
// returns to the rows, that a sparse visit does not invent Duration or
|
|
// provider/model, that a visit with no artifact field says the capture is
|
|
// unavailable rather than empty, and that the Session section renders the
|
|
// recorded export state: a complete export with a read-only failed repair and
|
|
// its open option, and a partial export with its state word and recorded
|
|
// reason. It also samples the rendered page so PROBE_SCHEME=dark cannot pass
|
|
// while the fixture silently renders light.
|
|
//
|
|
// PROBE_VARIANT=material|cupertino PROBE_SCHEME=dark|light PROBE_SCALE=1|2 \
|
|
// PROBE_WIDTH=1280 PROBE_HEIGHT=900 PROBE_WEB_ROOT=<probe build> \
|
|
// OPENCODE_ARTIFACT_DIR=<dir> node tool/run_detail_visit_probe.mjs
|
|
import { createServer } from 'node:http';
|
|
import { spawn } from 'node:child_process';
|
|
import { once } from 'node:events';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { join, extname, resolve, sep } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
|
|
const root = resolve(
|
|
process.env.PROBE_WEB_ROOT ??
|
|
join(process.cwd(), 'build/probe-run-detail-visit'),
|
|
);
|
|
const variant = process.env.PROBE_VARIANT ?? 'material';
|
|
const scheme = process.env.PROBE_SCHEME ?? 'dark';
|
|
const scale = process.env.PROBE_SCALE ?? '1';
|
|
const width = Number(process.env.PROBE_WIDTH ?? 1280);
|
|
const height = Number(process.env.PROBE_HEIGHT ?? 900);
|
|
const chromePath = process.env.CHROME_PATH ??
|
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
|
|
const artifactDir = process.env.OPENCODE_ARTIFACT_DIR;
|
|
if (!artifactDir) throw new Error('Set OPENCODE_ARTIFACT_DIR for the probe captures');
|
|
await mkdir(artifactDir, { recursive: true });
|
|
const prefix = `run-detail-visit-${variant}-${scheme}-${scale}x-${width}x${height}`;
|
|
const mime = {
|
|
'.html': 'text/html',
|
|
'.js': 'text/javascript',
|
|
'.json': 'application/json',
|
|
'.wasm': 'application/wasm',
|
|
'.png': 'image/png',
|
|
'.ico': 'image/x-icon',
|
|
'.otf': 'font/otf',
|
|
'.ttf': 'font/ttf',
|
|
};
|
|
const server = createServer(async (req, res) => {
|
|
try {
|
|
const path = resolve(root, '.' + decodeURIComponent(new URL(req.url, 'http://localhost').pathname));
|
|
if (path !== root && !path.startsWith(root + sep)) return res.writeHead(403).end();
|
|
const filePath = path === root || path.endsWith('/') ? join(path, 'index.html') : path;
|
|
const file = await readFile(filePath);
|
|
res.writeHead(200, { 'content-type': mime[extname(filePath)] ?? 'application/octet-stream' }).end(file);
|
|
} catch {
|
|
res.writeHead(404).end('not found');
|
|
}
|
|
});
|
|
server.listen(0, '127.0.0.1');
|
|
await once(server, 'listening');
|
|
const port = server.address().port;
|
|
const profile = await mkdtemp(join(tmpdir(), 'run-detail-visit-chrome-'));
|
|
const chrome = spawn(chromePath, [
|
|
'--headless=new', '--no-first-run', '--no-default-browser-check', '--disable-gpu',
|
|
'--remote-debugging-port=0', `--user-data-dir=${profile}`, 'about:blank',
|
|
], { stdio: ['ignore', 'ignore', 'pipe'] });
|
|
let stderr = '';
|
|
chrome.stderr.on('data', (data) => { stderr += data; });
|
|
const waitFor = async (fn) => {
|
|
for (let i = 0; i < 140; i++) {
|
|
const value = await fn();
|
|
if (value) return value;
|
|
await delay(100);
|
|
}
|
|
throw new Error('Timed out waiting for Chrome DevTools');
|
|
};
|
|
let ws;
|
|
try {
|
|
const endpoint = await waitFor(() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1]);
|
|
ws = new WebSocket(endpoint);
|
|
await once(ws, 'open');
|
|
let id = 0;
|
|
const pending = new Map();
|
|
ws.addEventListener('message', (event) => {
|
|
const message = JSON.parse(event.data);
|
|
if (message.method === 'Runtime.exceptionThrown') {
|
|
console.error('PAGE EXCEPTION', JSON.stringify(message.params?.exceptionDetails?.exception?.description));
|
|
}
|
|
if (message.id && pending.has(message.id)) {
|
|
pending.get(message.id)(message);
|
|
pending.delete(message.id);
|
|
}
|
|
});
|
|
const send = (method, params = {}, sessionId) => new Promise((resolveCall) => {
|
|
const callId = ++id;
|
|
pending.set(callId, resolveCall);
|
|
ws.send(JSON.stringify({ id: callId, method, params, ...(sessionId ? { sessionId } : {}) }));
|
|
});
|
|
const created = await send('Target.createTarget', { url: 'about:blank' });
|
|
const target = created.result.targetId;
|
|
const session = (await send('Target.attachToTarget', { targetId: target, flatten: true })).result.sessionId;
|
|
await send('Page.enable', {}, session);
|
|
await send('Runtime.enable', {}, session);
|
|
await send('Emulation.setEmulatedMedia', {
|
|
features: [{ name: 'prefers-color-scheme', value: scheme }],
|
|
}, session);
|
|
await send('Emulation.setDeviceMetricsOverride', {
|
|
width, height, deviceScaleFactor: 1, mobile: width < 600,
|
|
}, session);
|
|
|
|
const evaluate = async (expression) => {
|
|
const response = await send('Runtime.evaluate', {
|
|
expression, returnByValue: true, awaitPromise: true,
|
|
}, session);
|
|
if (response.error) throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
|
|
return response.result?.result?.value;
|
|
};
|
|
const semantics = () => evaluate(`[...document.querySelectorAll('flt-semantics')].map(x => {
|
|
const r = x.getBoundingClientRect();
|
|
return {
|
|
label: (x.getAttribute('aria-label') || (x.innerText || '')).trim(),
|
|
selected: x.getAttribute('aria-selected') || x.getAttribute('aria-current'),
|
|
tappable: x.hasAttribute('flt-tappable'),
|
|
x: r.x, y: r.y, width: r.width, height: r.height,
|
|
visible: r.width > 0 && r.height > 0 && r.bottom > 0 && r.top < innerHeight,
|
|
};
|
|
})`);
|
|
const labels = async () => (await semantics()).map((item) => item.label).filter(Boolean);
|
|
const bodyText = async () => [await evaluate('document.body.innerText'), ...(await labels())].join('\n');
|
|
const waitForText = async (needle, context) => {
|
|
for (let attempt = 0; attempt < 60; attempt++) {
|
|
if ((await bodyText()).includes(needle)) return;
|
|
await delay(250);
|
|
}
|
|
throw new Error(`${context}: missing ${JSON.stringify(needle)}\n${await bodyText()}`);
|
|
};
|
|
const clickItem = async (item) => {
|
|
const x = item.x + item.width / 2;
|
|
const y = item.y + item.height / 2;
|
|
await send('Input.dispatchMouseEvent', {
|
|
type: 'mousePressed', x, y, button: 'left', clickCount: 1,
|
|
}, session);
|
|
await send('Input.dispatchMouseEvent', {
|
|
type: 'mouseReleased', x, y, button: 'left', clickCount: 1,
|
|
}, session);
|
|
await delay(1000);
|
|
};
|
|
const findVisible = async (predicate) => {
|
|
const items = (await semantics()).filter((item) => item.visible && predicate(item));
|
|
items.sort((a, b) => a.width * a.height - b.width * b.height);
|
|
return items[0] ?? null;
|
|
};
|
|
const clickSemantics = async (wanted, { lowest = false, prefix: asPrefix = false } = {}) => {
|
|
const clicked = await evaluate(`(() => {
|
|
const wanted = ${JSON.stringify(wanted)};
|
|
const lowest = ${lowest ? 'true' : 'false'};
|
|
const asPrefix = ${asPrefix ? 'true' : 'false'};
|
|
const text = (x) => (x.getAttribute('aria-label') || x.innerText || '').trim();
|
|
const matches = [...document.querySelectorAll('flt-semantics')]
|
|
.filter((x) => x.hasAttribute('flt-tappable'))
|
|
.filter((x) => asPrefix ? text(x).startsWith(wanted) : text(x) === wanted);
|
|
if (!matches.length) return false;
|
|
matches.sort((a, b) => {
|
|
const ra = a.getBoundingClientRect();
|
|
const rb = b.getBoundingClientRect();
|
|
if (lowest) return rb.bottom - ra.bottom;
|
|
return ra.width * ra.height - rb.width * rb.height;
|
|
});
|
|
matches[0].click();
|
|
return true;
|
|
})()`);
|
|
if (!clicked) throw new Error(`Missing tappable ${JSON.stringify(wanted)}: ${JSON.stringify(await labels())}`);
|
|
await delay(1000);
|
|
};
|
|
const clickLabel = (label, options) => clickSemantics(label, options);
|
|
const clickPrefix = (prefixText) => clickSemantics(prefixText, { prefix: true });
|
|
const scrollPage = async (deltaY) => {
|
|
await send('Input.dispatchMouseEvent', {
|
|
type: 'mouseWheel', x: width / 2, y: height - 24, deltaX: 0, deltaY,
|
|
}, session);
|
|
await delay(700);
|
|
};
|
|
// A short tab body at 320/2x builds its rows lazily: scroll toward the row
|
|
// before requiring it, and leave the page there so the capture follows the
|
|
// facts. Semantics can include clipped rows, so the 320 capture proves the
|
|
// rows are in the tree; legibility at rest is checked by the UI review.
|
|
const ensureText = async (needle, context) => {
|
|
for (let attempt = 0; attempt < 8; attempt++) {
|
|
if ((await bodyText()).includes(needle)) return;
|
|
await scrollPage(240);
|
|
}
|
|
throw new Error(`${context}: missing ${JSON.stringify(needle)}\n${await bodyText()}`);
|
|
};
|
|
const capture = async (name) => {
|
|
const data = (await send('Page.captureScreenshot', {
|
|
format: 'png', captureBeyondViewport: false,
|
|
}, session)).result.data;
|
|
const file = `${prefix}-${name}.png`;
|
|
await writeFile(join(artifactDir, file), Buffer.from(data, 'base64'));
|
|
return file;
|
|
};
|
|
// Mean luminance of the actual render. The fixture receives the scheme through
|
|
// its own query parameter, so a mismatch between PROBE_SCHEME and the painted
|
|
// theme (as happened when the driver sent theme=1 and the fixture read
|
|
// 'dark') must fail rather than label a light capture as dark.
|
|
const pageLuminance = async () => {
|
|
const shot = (await send('Page.captureScreenshot', {
|
|
format: 'png', captureBeyondViewport: false,
|
|
}, session)).result.data;
|
|
return evaluate(`(async () => {
|
|
const img = new Image();
|
|
img.src = 'data:image/png;base64,${shot}';
|
|
await img.decode();
|
|
const canvas = document.createElement('canvas');
|
|
canvas.width = img.width;
|
|
canvas.height = img.height;
|
|
const ctx = canvas.getContext('2d');
|
|
ctx.drawImage(img, 0, 0);
|
|
const data = ctx.getImageData(0, 0, img.width, img.height).data;
|
|
let sum = 0;
|
|
let samples = 0;
|
|
for (let y = 0; y < img.height; y += 8) {
|
|
for (let x = 0; x < img.width; x += 8) {
|
|
const i = (y * img.width + x) * 4;
|
|
sum += (0.2126 * data[i] + 0.7152 * data[i + 1] + 0.0722 * data[i + 2]) / 255;
|
|
samples += 1;
|
|
}
|
|
}
|
|
return { mean: sum / samples, samples };
|
|
})()`);
|
|
};
|
|
const evidence = { variant, scheme, scale, width, height, checks: [], captures: [] };
|
|
const check = (text, value) => evidence.checks.push({ check: text, value });
|
|
|
|
const navigation = await send('Page.navigate', {
|
|
url: `http://127.0.0.1:${port}/?platform=${variant}&theme=${scheme}&scale=${scale}`,
|
|
}, session);
|
|
if (navigation.error || navigation.result?.errorText) throw new Error(JSON.stringify(navigation));
|
|
await delay(3200);
|
|
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
|
|
await delay(900);
|
|
await waitForText('Release verification', 'fixture load');
|
|
check('the real run detail opened over the recorded fixture', true);
|
|
|
|
const render = await pageLuminance();
|
|
evidence.pageLuminance = render.mean;
|
|
evidence.pageLuminanceSamples = render.samples;
|
|
const renderedDark = render.mean < 0.5;
|
|
if (renderedDark !== (scheme === 'dark')) {
|
|
throw new Error(
|
|
`PROBE_SCHEME=${scheme} painted the other theme (mean luminance ${render.mean.toFixed(3)})`,
|
|
);
|
|
}
|
|
check(`the ${scheme} scheme actually renders ${scheme === 'dark' ? 'dark' : 'light'}`, true);
|
|
|
|
await clickLabel('Graph');
|
|
await waitForText('Executed', 'graph legend');
|
|
await clickPrefix('plan');
|
|
await waitForText('plan · visit 1', 'visit pane after graph click');
|
|
await waitForText('Provider/model', 'visit facts');
|
|
await waitForText('anthropic / claude-x', 'visit facts');
|
|
await waitForText('2m 5s', 'recorded duration');
|
|
await waitForText('Plan checked.', 'recorded diagnostic');
|
|
await waitForText('approve', 'recorded verdict');
|
|
const paneLabels = await labels();
|
|
for (const section of ['Input', 'Output', 'Artifacts', 'Session', 'Definition']) {
|
|
if (!paneLabels.includes(section)) {
|
|
throw new Error(`Visit section ${section} is missing: ${JSON.stringify(paneLabels)}`);
|
|
}
|
|
}
|
|
check('Graph stage click opens the recorded visit pane with its tabs', true);
|
|
evidence.captures.push(await capture('visit-pane'));
|
|
|
|
await clickLabel('Definition', { lowest: true });
|
|
await waitForText('Type model', 'authored definition');
|
|
await waitForText('Prompt', 'authored definition prompt');
|
|
await waitForText('Draft the plan.', 'pinned prompt');
|
|
check('Definition shows the pinned authored source', true);
|
|
evidence.captures.push(await capture('definition'));
|
|
|
|
await clickLabel('Session', { lowest: true });
|
|
await waitForText('sess-1', 'recorded session pointer');
|
|
await waitForText('Open session.html', 'session export link');
|
|
check('Session shows the recorded pointer and export open', true);
|
|
evidence.captures.push(await capture('session'));
|
|
|
|
await clickLabel('Artifacts', { lowest: true });
|
|
await waitForText('plan.csv', 'visit artifact');
|
|
check('Artifacts lists the visit recorded files', true);
|
|
evidence.captures.push(await capture('artifacts'));
|
|
|
|
await clickLabel('Back to timeline');
|
|
await waitForText('build · visit 2', 'timeline rows after back');
|
|
if ((await bodyText()).includes('Plan checked.')) {
|
|
throw new Error('Back left the visit pane open');
|
|
}
|
|
check('Back returns to the recorded rows', true);
|
|
evidence.captures.push(await capture('timeline-rows'));
|
|
|
|
await clickPrefix('build · visit 2');
|
|
await waitForText('build · visit 2', 'sparse visit pane');
|
|
const sparse = await bodyText();
|
|
for (const absent of ['Provider/model', 'Duration', '2m 5s', 'anthropic / claude-x']) {
|
|
if (sparse.includes(absent)) {
|
|
throw new Error(`A sparse visit invented ${JSON.stringify(absent)}\n${sparse}`);
|
|
}
|
|
}
|
|
check('a sparse visit does not invent duration or provider/model', true);
|
|
evidence.captures.push(await capture('sparse-visit'));
|
|
|
|
await clickLabel('Artifacts', { lowest: true });
|
|
await waitForText('Artifacts · capture unavailable', 'sparse capture state');
|
|
check('a visit without an artifact field says capture unavailable', true);
|
|
evidence.captures.push(await capture('sparse-artifacts'));
|
|
|
|
await clickLabel('Back to timeline');
|
|
await waitForText('plan · visit 1', 'rows before the export-state checks');
|
|
await clickPrefix('audit · visit 3');
|
|
await waitForText('audit · visit 3', 'recorded export visit');
|
|
await clickLabel('Session', { lowest: true });
|
|
await ensureText('Open repaired export', 'recorded repair option');
|
|
await ensureText(
|
|
'failed · 0 messages. The repair read failed.',
|
|
'recorded repair label',
|
|
);
|
|
await ensureText('Open session.html', 'recorded export open');
|
|
const auditText = await bodyText();
|
|
if (!auditText.includes('Export complete')) {
|
|
throw new Error(`The complete export state is missing\n${auditText}`);
|
|
}
|
|
if (auditText.includes('Reason')) {
|
|
throw new Error(`A complete export showed a Reason row\n${auditText}`);
|
|
}
|
|
check('a complete export shows its read-only repair attempt and option', true);
|
|
evidence.captures.push(await capture('export-complete-repair'));
|
|
|
|
await clickLabel('Back to timeline');
|
|
await waitForText('plan · visit 1', 'rows before the partial export check');
|
|
await ensureText('partial · visit 4', 'partial export row');
|
|
await clickPrefix('partial · visit 4');
|
|
await waitForText('partial · visit 4', 'partial export visit');
|
|
await clickLabel('Session', { lowest: true });
|
|
await ensureText(
|
|
'This export is failed; full session evidence is not available.',
|
|
'partial export sentence',
|
|
);
|
|
await ensureText('The native read was refused.', 'partial export reason');
|
|
const partialText = await bodyText();
|
|
if (!partialText.includes('Export partial')) {
|
|
throw new Error(`The partial export state is missing\n${partialText}`);
|
|
}
|
|
if (!partialText.includes('Reason The native read was refused.')) {
|
|
throw new Error(`The partial export reason is missing\n${partialText}`);
|
|
}
|
|
check('a partial export shows its state word and recorded reason', true);
|
|
evidence.captures.push(await capture('export-partial-reason'));
|
|
|
|
await clickLabel('Back to timeline');
|
|
await waitForText('plan · visit 1', 'rows before the gallery check');
|
|
await clickPrefix('plan · visit 1');
|
|
await waitForText('Provider/model', 'plan visit for the gallery check');
|
|
await clickLabel('Artifacts', { lowest: true });
|
|
await waitForText('plan.csv', 'plan visit artifacts');
|
|
await clickPrefix('plan.csv');
|
|
await waitForText('Artifacts · 1 recorded', 'visit gallery summary');
|
|
await waitForText('plan · visit 1', 'visit gallery group');
|
|
const galleryText = await bodyText();
|
|
if (galleryText.includes('session.html')) {
|
|
throw new Error(`The visit gallery listed the session evidence file\n${galleryText}`);
|
|
}
|
|
check('the artifact row opens exactly the visit attributed group', true);
|
|
evidence.captures.push(await capture('visit-gallery'));
|
|
|
|
console.log(JSON.stringify(evidence, null, 2));
|
|
} finally {
|
|
if (ws?.readyState === WebSocket.OPEN) {
|
|
ws.send(JSON.stringify({ id: 2147483647, method: 'Browser.close' }));
|
|
}
|
|
await Promise.race([once(chrome, 'exit'), delay(3000)]);
|
|
ws?.close();
|
|
if (chrome.exitCode === null) chrome.kill('SIGTERM');
|
|
chrome.stderr.destroy();
|
|
chrome.unref();
|
|
await new Promise((resolveClose) => {
|
|
server.close(resolveClose);
|
|
server.closeAllConnections();
|
|
});
|
|
await rm(profile, { recursive: true, force: true });
|
|
}
|