Files
pipelines/app/tool/runs_probe.mjs
T

599 lines
25 KiB
JavaScript

import { spawn, execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
import { once } from 'node:events';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import { startFixture } from './runs_probe_server.mjs';
// Browser acceptance for the Runs destination.
//
// Build one of the bundles first, then run from app/:
// flutter build web
// flutter build web --target=tool/runs_probe_entry.dart -o build/probe_web
// OPENCODE_ARTIFACT_DIR=<dir> node tool/runs_probe.mjs build/web
//
// Requires Google Chrome and Tesseract (OCR assertions cover canvas-rendered
// selectable text that Flutter web does not expose through semantics).
const output = process.env.OPENCODE_ARTIFACT_DIR;
if (!output) throw new Error('Set OPENCODE_ARTIFACT_DIR for the probe captures');
await mkdir(output, { recursive: true });
const root = resolve(process.argv[2] ?? 'build/probe_web');
const fixture = await startFixture({ root });
const html = await fetch(`${fixture.url}/`);
const mime = html.headers.get('content-type') ?? '';
if (html.status !== 200 || !mime.startsWith('text/html')) {
throw new Error(`Fixture index must be HTML: ${html.status} ${mime}`);
}
await html.text();
const profile = await mkdtemp(join(tmpdir(), 'i1-runs-chrome-'));
const chrome = spawn('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', [
'--headless=new',
'--no-first-run',
'--no-default-browser-check',
'--disable-gpu',
'--remote-debugging-port=0',
`--user-data-dir=${profile}`,
'about:blank',
], { stdio: ['ignore', 'ignore', 'pipe'] });
let stderr = '';
chrome.stderr.on('data', (data) => { stderr += data; });
let ws;
try {
const endpoint = await waitFor(() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1], 'Chrome DevTools');
ws = new WebSocket(endpoint);
await once(ws, 'open');
let id = 0;
const pending = new Map();
ws.addEventListener('message', (event) => {
const message = JSON.parse(event.data);
if (message.method?.includes('detached') || message.method?.includes('crashed')) {
console.error('CDP event', JSON.stringify(message));
}
if (message.method === 'Runtime.exceptionThrown') {
console.error('PAGE EXCEPTION', JSON.stringify(message.params?.exceptionDetails));
}
if (message.method === 'Log.entryAdded') {
console.error('PAGE LOG', JSON.stringify(message.params?.entry));
}
if (message.method === 'Runtime.consoleAPICalled') {
console.error('PAGE CONSOLE', JSON.stringify(message.params));
}
if (message.id && pending.has(message.id)) {
pending.get(message.id)(message);
pending.delete(message.id);
}
});
const send = (method, params = {}, sessionId) => new Promise((resolveCall) => {
const callId = ++id;
pending.set(callId, resolveCall);
ws.send(JSON.stringify({ id: callId, method, params, ...(sessionId ? { sessionId } : {}) }));
});
const target = (await send('Target.createTarget', { url: 'about:blank' })).result.targetId;
const session = (await send('Target.attachToTarget', { targetId: target, flatten: true })).result.sessionId;
await send('Page.enable', {}, session);
await send('Runtime.enable', {}, session);
await send('Log.enable', {}, session);
const evaluate = async (expression) => {
const response = await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }, session);
if (response.error) throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
if (response.result?.exceptionDetails) {
throw new Error(`Page exception: ${JSON.stringify(response.result.exceptionDetails)}`);
}
return response.result?.result?.value;
};
const enableSemantics = async () => {
await waitFor(
() => evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))"),
'Flutter semantics placeholder',
);
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
await delay(600);
};
const navigation = await send('Page.navigate', { url: `${fixture.url}/` }, session);
if (navigation.error || navigation.result?.errorText) throw new Error(JSON.stringify(navigation));
await enableSemantics();
await waitFor(
async () => (await evaluate('location.pathname')) === '/pipelines',
'boot route correction to /pipelines',
);
const semanticText = () => evaluate(`(() => {
const chunks = [];
for (const el of document.querySelectorAll('flt-semantics, [aria-label], [aria-valuetext]')) {
const label = el.getAttribute('aria-label');
if (label) chunks.push(label);
const value = el.getAttribute('aria-valuetext');
if (value) chunks.push(value);
if (el.value) chunks.push(String(el.value));
if (!el.children.length && el.innerText) chunks.push(el.innerText);
}
for (const el of document.querySelectorAll('input, textarea')) {
const label = el.getAttribute('aria-label');
if (label) chunks.push(label);
if (el.value) chunks.push(String(el.value));
}
if (document.body?.innerText) chunks.push(document.body.innerText);
return [...new Set(chunks)].join('\\n');
})()`);
const assertBody = async (text) => {
try {
await waitFor(
async () => (await semanticText()).includes(text),
`body to contain ${JSON.stringify(text)}`,
);
} catch (error) {
const textNow = await semanticText();
let extra = '';
if (process.env.PROBE_DEBUG) {
extra = `\nNodes: ${JSON.stringify(await evaluate(`[...document.querySelectorAll('flt-semantics, input, textarea')]
.map((el) => ({ tag: el.tagName, role: el.getAttribute('role'), label: el.getAttribute('aria-label'), value: el.getAttribute('aria-valuetext') ?? el.value, text: el.innerText }))
.filter((item) => item.label || item.value || item.text)`))}`;
}
throw new Error(`${error.message}\nBody now:\n${textNow.slice(0, 3000)}${extra}`);
}
};
const clickText = async (text, { contains = false } = {}) => {
const clicked = await evaluate(`(() => {
const wanted = ${JSON.stringify(text)};
const candidates = [...document.querySelectorAll('flt-semantics')].filter((el) => {
const label = (el.getAttribute('aria-label') || el.innerText || '').trim();
return label === wanted || (${contains} && label.includes(wanted));
});
const interactive = candidates.find((el) =>
['button', 'tab', 'link', 'checkbox', 'radio', 'menuitem'].includes(el.getAttribute('role')),
);
const target = interactive ?? candidates[0];
if (!target) return false;
target.click();
return true;
})()`);
if (!clicked) {
const labels = await evaluate(`[...document.querySelectorAll('flt-semantics')]
.map((el) => (el.getAttribute('aria-label') || '').trim()).filter(Boolean)`);
throw new Error(`Could not click ${JSON.stringify(text)}. Labels: ${JSON.stringify(labels)}`);
}
await delay(300);
};
const capture = async (name) => {
const data = (await send('Page.captureScreenshot', { format: 'png', captureBeyondViewport: false }, session)).result.data;
const path = join(output, `${name}.png`);
await writeFile(path, Buffer.from(data, 'base64'));
return path;
};
const ocr = async (path) => {
const { stdout } = await promisify(execFile)('tesseract', [path, 'stdout']);
return stdout;
};
const assertOcr = async (path, text) => {
const read = await ocr(path);
if (!read.toLowerCase().includes(text.toLowerCase())) {
throw new Error(`OCR of ${path} lacks ${JSON.stringify(text)}:\n${read.slice(0, 3000)}`);
}
};
const wheel = (deltaY) => send('Input.dispatchMouseEvent', {
type: 'mouseWheel',
x: 195,
y: 500,
deltaX: 0,
deltaY,
}, session);
const setSize = async (width, height) => {
await send('Emulation.setDeviceMetricsOverride', {
width,
height,
deviceScaleFactor: 1,
mobile: width < 600,
}, session);
await delay(500);
};
await setSize(390, 844);
await clickText('Runs');
if ((await evaluate('location.pathname')) !== '/runs') {
throw new Error(`Runs tab did not update the URL: ${await evaluate('location.pathname')}`);
}
await assertBody('Fixture live run');
await assertBody('Fixture pending run');
await assertBody('Fixture failing run');
const captures = [await capture('i1-runs-phone-list')];
await assertOcr(captures[0], 'Fixture live run');
await setSize(1280, 800);
await assertBody('Fixture live run');
captures.push(await capture('i1-runs-desktop-list'));
await assertOcr(captures[1], 'Fixture live run');
await setSize(390, 844);
const live = await fetch(`${fixture.url}/fixture/live`, { method: 'POST' });
if (!live.ok) throw new Error(`Fixture live trigger failed: ${live.status}`);
await assertBody('succeeded');
captures.push(await capture('i1-runs-phone-list-live'));
await assertOcr(captures[2], 'succeeded');
await clickText('Fixture pending run', { contains: true });
await assertBody('Loading run details…');
await assertBody('Back to list');
captures.push(await capture('i1-runs-phone-pending'));
await assertOcr(captures[3], 'Loading run details');
await clickText('Back to list', { contains: true });
await assertBody('Fixture failing run');
await clickText('Fixture failing run', { contains: true });
await assertBody('Fixture detail failure: run 3 is unavailable');
await assertBody('Back to list');
captures.push(await capture('i1-runs-phone-error'));
await assertOcr(captures[4], 'run 3 is unavailable');
await clickText('Back to list', { contains: true });
await assertBody('Fixture live run');
await clickText('Fixture live run', { contains: true });
if ((await evaluate('location.pathname')) !== '/runs/1') {
throw new Error(`Run selection did not update the URL: ${await evaluate('location.pathname')}`);
}
await assertBody('Original prompt');
await assertBody('Timeline');
// The recorded visits live on their own tab: the detail opens on Summary, so
// the timeline is only present after switching, exactly as on the web.
await clickText('Timeline');
await assertBody('build · visit 3');
await clickText('Summary');
await assertBody('Original prompt');
const detailCapture = await capture('i1-runs-phone-detail');
captures.push(detailCapture);
await assertOcr(detailCapture, 'fixture prompt with spaces');
// The recorded Changes action opens the landed viewer for this run's steps:
// the recorded file, its rendered diff, and back to the same detail.
await clickText('Changes');
await assertBody('1 file changed');
await assertBody('lib/features/fixture.dart');
captures.push(await capture('changes-phone-list'));
await clickText('lib/features/fixture.dart', { contains: true });
await assertBody('Rendered');
await assertBody('added fixture line');
await assertBody('removed fixture line');
captures.push(await capture('changes-phone-diff'));
await clickText('Close diff');
await assertBody('lib/features/fixture.dart');
await delay(600);
await clickText('Back', { contains: true });
await assertBody('Original prompt');
// The recorded artifacts action lists each visit's own files, keeps a
// repeated filename distinct, and shows the selected record's facts.
await clickText('Artifacts');
await assertBody('build · visit 1');
await assertBody('test · visit 1');
await assertBody('build-report.html');
await assertBody('result.csv');
captures.push(await capture('artifacts-phone-list'));
await delay(500);
await clickText('result.csv', { contains: true });
await assertBody('text/csv');
await assertBody('4.0');
captures.push(await capture('artifacts-phone-selected'));
await clickText('Close');
await delay(400);
await clickText('Back', { contains: true });
await assertBody('Original prompt');
await setSize(1280, 800);
await assertBody('Original prompt');
const desktopDetail = await capture('i1-runs-desktop-detail');
captures.push(desktopDetail);
await assertOcr(desktopDetail, 'fixture prompt with spaces');
await setSize(390, 844);
await clickText('Input');
await wheel(250);
await delay(300);
const inputCapture = await capture('i1-runs-phone-detail-input');
captures.push(inputCapture);
await assertOcr(inputCapture, 'input payload');
await clickText('Output');
await wheel(250);
await delay(300);
const outputCapture = await capture('i1-runs-phone-detail-output');
captures.push(outputCapture);
await assertOcr(outputCapture, 'answer');
// Browser Back/Forward move between the in-app routes, not out of the app.
const path = () => evaluate('location.pathname');
await evaluate('history.back()');
await waitFor(
async () => (await path()) === '/runs',
'browser Back to /runs',
);
await waitFor(
async () => (await semanticText()).includes('Fixture live run'),
'browser Back to the runs list',
);
await waitFor(
async () => (await semanticText()).includes('Fixture live run'),
'browser Back to the runs list',
);
const backListCapture = await capture('i1-runs-phone-browser-back-list');
captures.push(backListCapture);
await assertOcr(backListCapture, 'Fixture live run');
await evaluate('history.forward()');
await waitFor(
async () => (await path()) === '/runs/1' && (await semanticText()).includes('Original prompt'),
'browser Forward to /runs/1',
);
captures.push(await capture('i1-runs-phone-browser-forward-detail'));
// A direct /runs/1 load is a real deep link: the detail opens and the URL stays.
await send('Page.navigate', { url: `${fixture.url}/runs/1` }, session);
await enableSemantics();
await assertBody('Original prompt');
if ((await path()) !== '/runs/1') throw new Error(`Deep link moved to ${await path()}`);
captures.push(await capture('i1-deeplink-run-1-phone'));
// A drop keeps the open detail, reports reconnecting rather than unavailable,
// and has no raw transport exception anywhere on the page.
const beforeDrop = await (await fetch(`${fixture.url}/fixture/state`)).json();
const refused = await fetch(`${fixture.url}/fixture/refuse`, { method: 'POST' });
if (!refused.ok) throw new Error(`Fixture refuse failed: ${refused.status}`);
await assertBody('Reconnecting to Pipelines…');
const duringDrop = await semanticText();
for (const raw of ['WebSocketException', 'WebSocketChannelException', 'Failed to connect']) {
if (duringDrop.includes(raw)) throw new Error(`Raw transport text shown: ${raw}`);
}
if ((await path()) !== '/runs/1') throw new Error(`Drop moved the URL to ${await path()}`);
const retainedCapture = await capture('i1-reconnect-detail-retained-phone');
captures.push(retainedCapture);
await assertOcr(retainedCapture, 'Original prompt');
// Accepting again plus Retry restores the same detail without a remount.
const accepted = await fetch(`${fixture.url}/fixture/accept`, { method: 'POST' });
if (!accepted.ok) throw new Error(`Fixture accept failed: ${accepted.status}`);
await clickText('Retry');
await waitFor(
async () =>
!(await semanticText()).includes('Reconnecting to Pipelines…') &&
(await semanticText()).includes('Original prompt'),
'detail restored after reconnect',
);
if ((await path()) !== '/runs/1') throw new Error(`Reconnect moved the URL to ${await path()}`);
captures.push(await capture('i1-reconnect-detail-restored-phone'));
// A first connection while the server refuses upgrades is friendly: the gate
// never prints the transport exception and does not flicker to "connecting".
await fetch(`${fixture.url}/fixture/refuse`, { method: 'POST' });
await send('Page.navigate', { url: `${fixture.url}/runs` }, session);
await enableSemantics();
await assertBody('Pipelines server unavailable');
const gate = await semanticText();
for (const raw of ['WebSocketException', 'WebSocketChannelException', 'Failed to connect']) {
if (gate.includes(raw)) throw new Error(`Gate printed raw transport text: ${raw}`);
}
const gateCapture = await capture('i1-gate-unavailable-friendly-phone');
captures.push(gateCapture);
await delay(1500);
if (!(await semanticText()).includes('Pipelines server unavailable')) {
throw new Error('Gate left the unavailable state while retrying');
}
await assertOcr(gateCapture, 'Pipelines server unavailable');
await fetch(`${fixture.url}/fixture/accept`, { method: 'POST' });
await clickText('Retry');
await assertBody('Fixture live run');
captures.push(await capture('i1-gate-recovered-list-phone'));
await clickText('Settings');
await assertBody('Server URL');
await assertBody('Fixture settings build');
if ((await path()) !== '/settings') throw new Error('Settings did not update route');
captures.push(await capture('settings-cupertino-registered-phone'));
await clickText('Runs');
await clickText('Fixture cancel run', { contains: true });
await assertBody('Original prompt');
await clickText('Cancel run');
await assertBody('Cancel run 4?');
captures.push(await capture('cancel-cupertino-confirm'));
await clickText('Keep');
await assertBody('Original prompt');
const kept = await (await fetch(`${fixture.url}/fixture/state`)).json();
if (kept.cancelRequests.length) throw new Error('Keep sent a cancellation');
await clickText('Back', { contains: true });
await clickText('Fixture human run', { contains: true });
captures.push(await capture('decision-cupertino-card'));
await assertBody('Waiting for your decision');
await clickText('Decide');
await assertBody('Please review this recorded checkpoint.');
captures.push(await capture('decision-cupertino-confirm'));
await clickText('Cancel');
// Material host: the same deep link and route must render standard Material
// controls (the Cupertino host was selected by the macOS browser above).
await send(
'Emulation.setUserAgentOverride',
{
userAgent:
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' +
'(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36',
platform: 'Linux x86_64',
},
session,
);
await send('Page.navigate', { url: `${fixture.url}/runs/1` }, session);
await enableSemantics();
await assertBody('Original prompt');
await assertBody('Back to list');
if ((await path()) !== '/runs/1') {
throw new Error(`Material deep link moved to ${await path()}`);
}
captures.push(await capture('i1-material-deeplink-run-1-phone'));
const beforeSettings = await (await fetch(`${fixture.url}/fixture/state`)).json();
await clickText('Settings');
await assertBody('Server URL');
await assertBody('Fixture settings build');
const afterSettings = await (await fetch(`${fixture.url}/fixture/state`)).json();
if (beforeSettings.subscribed !== afterSettings.subscribed || afterSettings.connections !== 1) {
throw new Error('Settings opened another connection instead of sharing the app socket');
}
captures.push(await capture('settings-material-registered-phone'));
// The registered Chat destination opens a real conversation over the shared
// socket: list row -> session route -> stored turn, then Back to the list.
await clickText('Chat');
await assertBody('Probe conversation');
captures.push(await capture('chat-material-list'));
await clickText('Probe conversation', { contains: true });
await assertBody('User message');
await assertBody('Assistant message');
if ((await path()) !== '/chat/ses_probechat') {
throw new Error(`Chat selection did not update the route: ${await path()}`);
}
await delay(900);
const threadCapture = await capture('chat-material-thread');
captures.push(threadCapture);
await assertOcr(threadCapture, 'Read-only probe question.');
await assertOcr(threadCapture, 'Stored probe answer.');
await clickText('Back', { contains: true });
await assertBody('Probe conversation');
if ((await path()) !== '/chat') {
throw new Error(`Chat Back left the route at ${await path()}`);
}
await clickText('Runs');
await clickText('Fixture cancel run', { contains: true });
await assertBody('Original prompt');
await clickText('Cancel run');
await assertBody('Cancel run 4?');
captures.push(await capture('cancel-material-confirm'));
await clickText('Cancel run');
await assertBody('canceled');
captures.push(await capture('cancel-material-accepted'));
await clickText('Back to list');
await assertBody('Fixture cancel run');
await assertBody('canceled');
await clickText('Fixture human run', { contains: true });
await assertBody('Waiting for your decision');
captures.push(await capture('decision-material-card'));
await clickText('Decide');
await assertBody('Please review this recorded checkpoint.');
captures.push(await capture('decision-material-confirm'));
const pendingDecision = await (await fetch(`${fixture.url}/fixture/state`)).json();
if (pendingDecision.decisionRequests.length) throw new Error('Opening decision sent a reply');
await clickText('approve', { contains: true });
await clickText('Confirm decision');
await assertBody('succeeded');
if ((await semanticText()).includes('Waiting for your decision')) {
throw new Error('Accepted human decision left the waiting card behind');
}
captures.push(await capture('decision-material-accepted'));
await clickText('Back to list');
await clickText('Chat');
await clickText('Probe conversation', { contains: true });
await assertBody('User message');
// The empty composer exposes a hint, not an accessible label. Locate the
// semantics text field and click its centre: a fixed point misses once an
// attachment strip grows the composer, and JS focus alone is not Flutter's
// editing connection.
const composerPoint = await evaluate(`(() => {
const fields = [...document.querySelectorAll('flt-semantics[role="textbox"], input, textarea')];
const rects = fields
.map((el) => el.getBoundingClientRect())
.filter((r) => r.width > 40 && r.height > 10);
if (!rects.length) return null;
const lowest = rects.sort((a, b) => b.bottom - a.bottom)[0];
return { x: lowest.left + lowest.width / 2, y: lowest.top + lowest.height / 2 };
})()`);
if (!composerPoint) throw new Error('The composer text field is not exposed');
for (const type of ['mousePressed', 'mouseReleased']) {
await send('Input.dispatchMouseEvent', {
type, x: composerPoint.x, y: composerPoint.y, button: 'left', clickCount: 1,
}, session);
}
await delay(400);
await send('Input.insertText', { text: 'Probe composed message' }, session);
await delay(500);
if (!(await semanticText()).includes('Probe composed message')) {
throw new Error('The composer field did not receive the typed text');
}
// Send through the primary control; the Ctrl/Cmd+Enter path is covered by the
// widget probe, and a synthetic key event is not a reliable stand-in for the
// engine's hardware-keyboard state in headless Chrome.
await clickText('Send');
await delay(900);
const afterSend = await (await fetch(`${fixture.url}/fixture/state`)).json();
if (afterSend.chatPrompts.length !== 1) {
throw new Error(`Composer did not send exactly one prompt: ${JSON.stringify(afterSend.chatPrompts)}`);
}
const sent = afterSend.chatPrompts[0];
if (sent.text !== 'Probe composed message' || sent.session_id !== 'ses_probechat' ||
!sent.operation_id || !sent.native_message_id) {
throw new Error(`Wrong prompt payload: ${JSON.stringify(sent)}`);
}
const composerCapture = await capture('chat-composer-material-sent');
captures.push(composerCapture);
await assertOcr(composerCapture, 'Probe composed message');
await clickText('Runs');
await delay(500);
await assertBody('Fixture cancel run');
await assertBody('canceled');
const fixtureState = await (await fetch(`${fixture.url}/fixture/state`)).json();
if (JSON.stringify(fixtureState.decisionRequests) !==
JSON.stringify([{ run_id: 5, stage: 'gate', option: 'approve' }])) {
throw new Error('Decision sent the wrong payload or repeated the request');
}
if (JSON.stringify(fixtureState.cancelRequests) !== JSON.stringify([{ run_id: 4 }])) {
throw new Error('Cancel sent the wrong payload or repeated the request');
}
if (fixtureState.subscribed < 2) throw new Error(`App never resubscribed: ${JSON.stringify(fixtureState)}`);
if (fixtureState.listRuns < 2) throw new Error(`App never reloaded: ${JSON.stringify(fixtureState)}`);
if (fixtureState.getRun < 2) throw new Error(`Detail was not rehydrated: ${JSON.stringify(fixtureState)}`);
if (fixtureState.dropped < 1) throw new Error(`Fixture never dropped: ${JSON.stringify(fixtureState)}`);
console.log(JSON.stringify({
ok: true,
fixture: fixture.url,
subscribed: fixtureState.subscribed,
listRuns: fixtureState.listRuns,
getRun: fixtureState.getRun,
dropped: fixtureState.dropped,
live: fixtureState.live,
captures,
}, null, 2));
} finally {
ws?.close();
chrome.kill('SIGTERM');
await Promise.race([once(chrome, 'exit'), delay(3000)]);
await fixture.close();
await rm(profile, { recursive: true, force: true });
}
async function waitFor(check, label, attempts = 160) {
for (let attempt = 0; attempt < attempts; attempt++) {
const value = await check();
if (value) return value;
await delay(100);
}
throw new Error(`Timed out waiting for ${label}`);
}