import { spawn, execFile } from 'node:child_process'; import { promisify } from 'node:util'; import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; import { once } from 'node:events'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; import { startFixture } from './runs_probe_server.mjs'; // Browser acceptance for the Runs destination. // // Build one of the bundles first, then run from app/: // flutter build web // flutter build web --target=tool/runs_probe_entry.dart -o build/probe_web // OPENCODE_ARTIFACT_DIR= node tool/runs_probe.mjs build/web // // Requires Google Chrome and Tesseract (OCR assertions cover canvas-rendered // selectable text that Flutter web does not expose through semantics). const output = process.env.OPENCODE_ARTIFACT_DIR; if (!output) throw new Error('Set OPENCODE_ARTIFACT_DIR for the probe captures'); await mkdir(output, { recursive: true }); const root = resolve(process.argv[2] ?? 'build/probe_web'); const fixture = await startFixture({ root }); const html = await fetch(`${fixture.url}/`); const mime = html.headers.get('content-type') ?? ''; if (html.status !== 200 || !mime.startsWith('text/html')) { throw new Error(`Fixture index must be HTML: ${html.status} ${mime}`); } await html.text(); const profile = await mkdtemp(join(tmpdir(), 'i1-runs-chrome-')); const chrome = spawn('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', [ '--headless=new', '--no-first-run', '--no-default-browser-check', '--disable-gpu', '--remote-debugging-port=0', `--user-data-dir=${profile}`, 'about:blank', ], { stdio: ['ignore', 'ignore', 'pipe'] }); let stderr = ''; chrome.stderr.on('data', (data) => { stderr += data; }); let ws; try { const endpoint = await waitFor(() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1], 'Chrome DevTools'); ws = new WebSocket(endpoint); await once(ws, 'open'); let id = 0; const pending = new Map(); ws.addEventListener('message', (event) => { const message = JSON.parse(event.data); if (message.method?.includes('detached') || message.method?.includes('crashed')) { console.error('CDP event', JSON.stringify(message)); } if (message.method === 'Runtime.exceptionThrown') { console.error('PAGE EXCEPTION', JSON.stringify(message.params?.exceptionDetails)); } if (message.method === 'Log.entryAdded') { console.error('PAGE LOG', JSON.stringify(message.params?.entry)); } if (message.method === 'Runtime.consoleAPICalled') { console.error('PAGE CONSOLE', JSON.stringify(message.params)); } if (message.id && pending.has(message.id)) { pending.get(message.id)(message); pending.delete(message.id); } }); const send = (method, params = {}, sessionId) => new Promise((resolveCall) => { const callId = ++id; pending.set(callId, resolveCall); ws.send(JSON.stringify({ id: callId, method, params, ...(sessionId ? { sessionId } : {}) })); }); const target = (await send('Target.createTarget', { url: 'about:blank' })).result.targetId; const session = (await send('Target.attachToTarget', { targetId: target, flatten: true })).result.sessionId; await send('Page.enable', {}, session); await send('Runtime.enable', {}, session); await send('Log.enable', {}, session); const evaluate = async (expression) => { const response = await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }, session); if (response.error) throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`); if (response.result?.exceptionDetails) { throw new Error(`Page exception: ${JSON.stringify(response.result.exceptionDetails)}`); } return response.result?.result?.value; }; const enableSemantics = async () => { await waitFor( () => evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))"), 'Flutter semantics placeholder', ); await evaluate("document.querySelector('flt-semantics-placeholder')?.click()"); await delay(600); }; const navigation = await send('Page.navigate', { url: `${fixture.url}/` }, session); if (navigation.error || navigation.result?.errorText) throw new Error(JSON.stringify(navigation)); await enableSemantics(); await waitFor( async () => (await evaluate('location.pathname')) === '/pipelines', 'boot route correction to /pipelines', ); const semanticText = () => evaluate(`(() => { const chunks = []; for (const el of document.querySelectorAll('flt-semantics, [aria-label], [aria-valuetext]')) { const label = el.getAttribute('aria-label'); if (label) chunks.push(label); const value = el.getAttribute('aria-valuetext'); if (value) chunks.push(value); if (el.value) chunks.push(String(el.value)); if (!el.children.length && el.innerText) chunks.push(el.innerText); } for (const el of document.querySelectorAll('input, textarea')) { const label = el.getAttribute('aria-label'); if (label) chunks.push(label); if (el.value) chunks.push(String(el.value)); } if (document.body?.innerText) chunks.push(document.body.innerText); return [...new Set(chunks)].join('\\n'); })()`); const assertBody = async (text) => { try { await waitFor( async () => (await semanticText()).includes(text), `body to contain ${JSON.stringify(text)}`, ); } catch (error) { const textNow = await semanticText(); let extra = ''; if (process.env.PROBE_DEBUG) { extra = `\nNodes: ${JSON.stringify(await evaluate(`[...document.querySelectorAll('flt-semantics, input, textarea')] .map((el) => ({ tag: el.tagName, role: el.getAttribute('role'), label: el.getAttribute('aria-label'), value: el.getAttribute('aria-valuetext') ?? el.value, text: el.innerText })) .filter((item) => item.label || item.value || item.text)`))}`; } throw new Error(`${error.message}\nBody now:\n${textNow.slice(0, 3000)}${extra}`); } }; const clickText = async (text, { contains = false } = {}) => { const clicked = await evaluate(`(() => { const wanted = ${JSON.stringify(text)}; const candidates = [...document.querySelectorAll('flt-semantics')].filter((el) => { const label = (el.getAttribute('aria-label') || el.innerText || '').trim(); return label === wanted || (${contains} && label.includes(wanted)); }); const interactive = candidates.find((el) => ['button', 'tab', 'link', 'checkbox', 'radio', 'menuitem'].includes(el.getAttribute('role')), ); const target = interactive ?? candidates[0]; if (!target) return false; target.click(); return true; })()`); if (!clicked) { const labels = await evaluate(`[...document.querySelectorAll('flt-semantics')] .map((el) => (el.getAttribute('aria-label') || '').trim()).filter(Boolean)`); throw new Error(`Could not click ${JSON.stringify(text)}. Labels: ${JSON.stringify(labels)}`); } await delay(300); }; const capture = async (name) => { const data = (await send('Page.captureScreenshot', { format: 'png', captureBeyondViewport: false }, session)).result.data; const path = join(output, `${name}.png`); await writeFile(path, Buffer.from(data, 'base64')); return path; }; const ocr = async (path) => { const { stdout } = await promisify(execFile)('tesseract', [path, 'stdout']); return stdout; }; const assertOcr = async (path, text) => { const read = await ocr(path); if (!read.toLowerCase().includes(text.toLowerCase())) { throw new Error(`OCR of ${path} lacks ${JSON.stringify(text)}:\n${read.slice(0, 3000)}`); } }; const wheel = (deltaY) => send('Input.dispatchMouseEvent', { type: 'mouseWheel', x: 195, y: 500, deltaX: 0, deltaY, }, session); const setSize = async (width, height) => { await send('Emulation.setDeviceMetricsOverride', { width, height, deviceScaleFactor: 1, mobile: width < 600, }, session); await delay(500); }; await setSize(390, 844); await clickText('Runs'); if ((await evaluate('location.pathname')) !== '/runs') { throw new Error(`Runs tab did not update the URL: ${await evaluate('location.pathname')}`); } await assertBody('Fixture live run'); await assertBody('Fixture pending run'); await assertBody('Fixture failing run'); const captures = [await capture('i1-runs-phone-list')]; await assertOcr(captures[0], 'Fixture live run'); await setSize(1280, 800); await assertBody('Fixture live run'); captures.push(await capture('i1-runs-desktop-list')); await assertOcr(captures[1], 'Fixture live run'); await setSize(390, 844); const live = await fetch(`${fixture.url}/fixture/live`, { method: 'POST' }); if (!live.ok) throw new Error(`Fixture live trigger failed: ${live.status}`); await assertBody('succeeded'); captures.push(await capture('i1-runs-phone-list-live')); await assertOcr(captures[2], 'succeeded'); await clickText('Fixture pending run', { contains: true }); await assertBody('Loading run details…'); await assertBody('Back to list'); captures.push(await capture('i1-runs-phone-pending')); await assertOcr(captures[3], 'Loading run details'); await clickText('Back to list', { contains: true }); await assertBody('Fixture failing run'); await clickText('Fixture failing run', { contains: true }); await assertBody('Fixture detail failure: run 3 is unavailable'); await assertBody('Back to list'); captures.push(await capture('i1-runs-phone-error')); await assertOcr(captures[4], 'run 3 is unavailable'); await clickText('Back to list', { contains: true }); await assertBody('Fixture live run'); await clickText('Fixture live run', { contains: true }); if ((await evaluate('location.pathname')) !== '/runs/1') { throw new Error(`Run selection did not update the URL: ${await evaluate('location.pathname')}`); } await assertBody('Original prompt'); await assertBody('Timeline'); // The recorded visits live on their own tab: the detail opens on Summary, so // the timeline is only present after switching, exactly as on the web. await clickText('Timeline'); await assertBody('build · visit 3'); await clickText('Summary'); await assertBody('Original prompt'); const detailCapture = await capture('i1-runs-phone-detail'); captures.push(detailCapture); await assertOcr(detailCapture, 'fixture prompt with spaces'); // The recorded Changes action opens the landed viewer for this run's steps: // the recorded file, its rendered diff, and back to the same detail. await clickText('Changes'); await assertBody('1 file changed'); await assertBody('lib/features/fixture.dart'); captures.push(await capture('changes-phone-list')); await clickText('lib/features/fixture.dart', { contains: true }); await assertBody('Rendered'); await assertBody('added fixture line'); await assertBody('removed fixture line'); captures.push(await capture('changes-phone-diff')); await clickText('Close diff'); await assertBody('lib/features/fixture.dart'); await delay(600); await clickText('Back', { contains: true }); await assertBody('Original prompt'); // The recorded artifacts action lists each visit's own files, keeps a // repeated filename distinct, and shows the selected record's facts. await clickText('Artifacts'); await assertBody('build · visit 1'); await assertBody('test · visit 1'); await assertBody('build-report.html'); await assertBody('result.csv'); captures.push(await capture('artifacts-phone-list')); await delay(500); await clickText('result.csv', { contains: true }); await assertBody('text/csv'); await assertBody('4.0'); captures.push(await capture('artifacts-phone-selected')); await clickText('Close'); await delay(400); await clickText('Back', { contains: true }); await assertBody('Original prompt'); await setSize(1280, 800); await assertBody('Original prompt'); const desktopDetail = await capture('i1-runs-desktop-detail'); captures.push(desktopDetail); await assertOcr(desktopDetail, 'fixture prompt with spaces'); await setSize(390, 844); await clickText('Input'); await wheel(250); await delay(300); const inputCapture = await capture('i1-runs-phone-detail-input'); captures.push(inputCapture); await assertOcr(inputCapture, 'input payload'); await clickText('Output'); await wheel(250); await delay(300); const outputCapture = await capture('i1-runs-phone-detail-output'); captures.push(outputCapture); await assertOcr(outputCapture, 'answer'); // Browser Back/Forward move between the in-app routes, not out of the app. const path = () => evaluate('location.pathname'); await evaluate('history.back()'); await waitFor( async () => (await path()) === '/runs', 'browser Back to /runs', ); await waitFor( async () => (await semanticText()).includes('Fixture live run'), 'browser Back to the runs list', ); await waitFor( async () => (await semanticText()).includes('Fixture live run'), 'browser Back to the runs list', ); const backListCapture = await capture('i1-runs-phone-browser-back-list'); captures.push(backListCapture); await assertOcr(backListCapture, 'Fixture live run'); await evaluate('history.forward()'); await waitFor( async () => (await path()) === '/runs/1' && (await semanticText()).includes('Original prompt'), 'browser Forward to /runs/1', ); captures.push(await capture('i1-runs-phone-browser-forward-detail')); // A direct /runs/1 load is a real deep link: the detail opens and the URL stays. await send('Page.navigate', { url: `${fixture.url}/runs/1` }, session); await enableSemantics(); await assertBody('Original prompt'); if ((await path()) !== '/runs/1') throw new Error(`Deep link moved to ${await path()}`); captures.push(await capture('i1-deeplink-run-1-phone')); // A drop keeps the open detail, reports reconnecting rather than unavailable, // and has no raw transport exception anywhere on the page. const beforeDrop = await (await fetch(`${fixture.url}/fixture/state`)).json(); const refused = await fetch(`${fixture.url}/fixture/refuse`, { method: 'POST' }); if (!refused.ok) throw new Error(`Fixture refuse failed: ${refused.status}`); await assertBody('Reconnecting to Pipelines…'); const duringDrop = await semanticText(); for (const raw of ['WebSocketException', 'WebSocketChannelException', 'Failed to connect']) { if (duringDrop.includes(raw)) throw new Error(`Raw transport text shown: ${raw}`); } if ((await path()) !== '/runs/1') throw new Error(`Drop moved the URL to ${await path()}`); const retainedCapture = await capture('i1-reconnect-detail-retained-phone'); captures.push(retainedCapture); await assertOcr(retainedCapture, 'Original prompt'); // Accepting again plus Retry restores the same detail without a remount. const accepted = await fetch(`${fixture.url}/fixture/accept`, { method: 'POST' }); if (!accepted.ok) throw new Error(`Fixture accept failed: ${accepted.status}`); await clickText('Retry'); await waitFor( async () => !(await semanticText()).includes('Reconnecting to Pipelines…') && (await semanticText()).includes('Original prompt'), 'detail restored after reconnect', ); if ((await path()) !== '/runs/1') throw new Error(`Reconnect moved the URL to ${await path()}`); captures.push(await capture('i1-reconnect-detail-restored-phone')); // A first connection while the server refuses upgrades is friendly: the gate // never prints the transport exception and does not flicker to "connecting". await fetch(`${fixture.url}/fixture/refuse`, { method: 'POST' }); await send('Page.navigate', { url: `${fixture.url}/runs` }, session); await enableSemantics(); await assertBody('Pipelines server unavailable'); const gate = await semanticText(); for (const raw of ['WebSocketException', 'WebSocketChannelException', 'Failed to connect']) { if (gate.includes(raw)) throw new Error(`Gate printed raw transport text: ${raw}`); } const gateCapture = await capture('i1-gate-unavailable-friendly-phone'); captures.push(gateCapture); await delay(1500); if (!(await semanticText()).includes('Pipelines server unavailable')) { throw new Error('Gate left the unavailable state while retrying'); } await assertOcr(gateCapture, 'Pipelines server unavailable'); await fetch(`${fixture.url}/fixture/accept`, { method: 'POST' }); await clickText('Retry'); await assertBody('Fixture live run'); captures.push(await capture('i1-gate-recovered-list-phone')); await clickText('Settings'); await assertBody('Server URL'); await assertBody('Fixture settings build'); if ((await path()) !== '/settings') throw new Error('Settings did not update route'); captures.push(await capture('settings-cupertino-registered-phone')); await clickText('Runs'); await clickText('Fixture cancel run', { contains: true }); await assertBody('Original prompt'); await clickText('Cancel run'); await assertBody('Cancel run 4?'); captures.push(await capture('cancel-cupertino-confirm')); await clickText('Keep'); await assertBody('Original prompt'); const kept = await (await fetch(`${fixture.url}/fixture/state`)).json(); if (kept.cancelRequests.length) throw new Error('Keep sent a cancellation'); await clickText('Back', { contains: true }); await clickText('Fixture human run', { contains: true }); captures.push(await capture('decision-cupertino-card')); await assertBody('Waiting for your decision'); await clickText('Decide'); await assertBody('Please review this recorded checkpoint.'); captures.push(await capture('decision-cupertino-confirm')); await clickText('Cancel'); // Material host: the same deep link and route must render standard Material // controls (the Cupertino host was selected by the macOS browser above). await send( 'Emulation.setUserAgentOverride', { userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36', platform: 'Linux x86_64', }, session, ); await send('Page.navigate', { url: `${fixture.url}/runs/1` }, session); await enableSemantics(); await assertBody('Original prompt'); await assertBody('Back to list'); if ((await path()) !== '/runs/1') { throw new Error(`Material deep link moved to ${await path()}`); } captures.push(await capture('i1-material-deeplink-run-1-phone')); const beforeSettings = await (await fetch(`${fixture.url}/fixture/state`)).json(); await clickText('Settings'); await assertBody('Server URL'); await assertBody('Fixture settings build'); const afterSettings = await (await fetch(`${fixture.url}/fixture/state`)).json(); if (beforeSettings.subscribed !== afterSettings.subscribed || afterSettings.connections !== 1) { throw new Error('Settings opened another connection instead of sharing the app socket'); } captures.push(await capture('settings-material-registered-phone')); // The registered Chat destination opens a real conversation over the shared // socket: list row -> session route -> stored turn, then Back to the list. await clickText('Chat'); await assertBody('Probe conversation'); captures.push(await capture('chat-material-list')); await clickText('Probe conversation', { contains: true }); await assertBody('User message'); await assertBody('Assistant message'); if ((await path()) !== '/chat/ses_probechat') { throw new Error(`Chat selection did not update the route: ${await path()}`); } await delay(900); const threadCapture = await capture('chat-material-thread'); captures.push(threadCapture); await assertOcr(threadCapture, 'Read-only probe question.'); await assertOcr(threadCapture, 'Stored probe answer.'); await clickText('Back', { contains: true }); await assertBody('Probe conversation'); if ((await path()) !== '/chat') { throw new Error(`Chat Back left the route at ${await path()}`); } await clickText('Runs'); await clickText('Fixture cancel run', { contains: true }); await assertBody('Original prompt'); await clickText('Cancel run'); await assertBody('Cancel run 4?'); captures.push(await capture('cancel-material-confirm')); await clickText('Cancel run'); await assertBody('canceled'); captures.push(await capture('cancel-material-accepted')); await clickText('Back to list'); await assertBody('Fixture cancel run'); await assertBody('canceled'); await clickText('Fixture human run', { contains: true }); await assertBody('Waiting for your decision'); captures.push(await capture('decision-material-card')); await clickText('Decide'); await assertBody('Please review this recorded checkpoint.'); captures.push(await capture('decision-material-confirm')); const pendingDecision = await (await fetch(`${fixture.url}/fixture/state`)).json(); if (pendingDecision.decisionRequests.length) throw new Error('Opening decision sent a reply'); await clickText('approve', { contains: true }); await clickText('Confirm decision'); await assertBody('succeeded'); if ((await semanticText()).includes('Waiting for your decision')) { throw new Error('Accepted human decision left the waiting card behind'); } captures.push(await capture('decision-material-accepted')); await clickText('Back to list'); await clickText('Chat'); await clickText('Probe conversation', { contains: true }); await assertBody('User message'); // The empty composer exposes a hint, not an accessible label. Locate the // semantics text field and click its centre: a fixed point misses once an // attachment strip grows the composer, and JS focus alone is not Flutter's // editing connection. const composerPoint = await evaluate(`(() => { const fields = [...document.querySelectorAll('flt-semantics[role="textbox"], input, textarea')]; const rects = fields .map((el) => el.getBoundingClientRect()) .filter((r) => r.width > 40 && r.height > 10); if (!rects.length) return null; const lowest = rects.sort((a, b) => b.bottom - a.bottom)[0]; return { x: lowest.left + lowest.width / 2, y: lowest.top + lowest.height / 2 }; })()`); if (!composerPoint) throw new Error('The composer text field is not exposed'); for (const type of ['mousePressed', 'mouseReleased']) { await send('Input.dispatchMouseEvent', { type, x: composerPoint.x, y: composerPoint.y, button: 'left', clickCount: 1, }, session); } await delay(400); await send('Input.insertText', { text: 'Probe composed message' }, session); await delay(500); if (!(await semanticText()).includes('Probe composed message')) { throw new Error('The composer field did not receive the typed text'); } // Send through the primary control; the Ctrl/Cmd+Enter path is covered by the // widget probe, and a synthetic key event is not a reliable stand-in for the // engine's hardware-keyboard state in headless Chrome. await clickText('Send'); await delay(900); const afterSend = await (await fetch(`${fixture.url}/fixture/state`)).json(); if (afterSend.chatPrompts.length !== 1) { throw new Error(`Composer did not send exactly one prompt: ${JSON.stringify(afterSend.chatPrompts)}`); } const sent = afterSend.chatPrompts[0]; if (sent.text !== 'Probe composed message' || sent.session_id !== 'ses_probechat' || !sent.operation_id || !sent.native_message_id) { throw new Error(`Wrong prompt payload: ${JSON.stringify(sent)}`); } const composerCapture = await capture('chat-composer-material-sent'); captures.push(composerCapture); await assertOcr(composerCapture, 'Probe composed message'); await clickText('Runs'); await delay(500); await assertBody('Fixture cancel run'); await assertBody('canceled'); const fixtureState = await (await fetch(`${fixture.url}/fixture/state`)).json(); if (JSON.stringify(fixtureState.decisionRequests) !== JSON.stringify([{ run_id: 5, stage: 'gate', option: 'approve' }])) { throw new Error('Decision sent the wrong payload or repeated the request'); } if (JSON.stringify(fixtureState.cancelRequests) !== JSON.stringify([{ run_id: 4 }])) { throw new Error('Cancel sent the wrong payload or repeated the request'); } if (fixtureState.subscribed < 2) throw new Error(`App never resubscribed: ${JSON.stringify(fixtureState)}`); if (fixtureState.listRuns < 2) throw new Error(`App never reloaded: ${JSON.stringify(fixtureState)}`); if (fixtureState.getRun < 2) throw new Error(`Detail was not rehydrated: ${JSON.stringify(fixtureState)}`); if (fixtureState.dropped < 1) throw new Error(`Fixture never dropped: ${JSON.stringify(fixtureState)}`); console.log(JSON.stringify({ ok: true, fixture: fixture.url, subscribed: fixtureState.subscribed, listRuns: fixtureState.listRuns, getRun: fixtureState.getRun, dropped: fixtureState.dropped, live: fixtureState.live, captures, }, null, 2)); } finally { ws?.close(); chrome.kill('SIGTERM'); await Promise.race([once(chrome, 'exit'), delay(3000)]); await fixture.close(); await rm(profile, { recursive: true, force: true }); } async function waitFor(check, label, attempts = 160) { for (let attempt = 0; attempt < attempts; attempt++) { const value = await check(); if (value) return value; await delay(100); } throw new Error(`Timed out waiting for ${label}`); }