import { spawn, execFile } from 'node:child_process'; import { promisify } from 'node:util'; import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; import { once } from 'node:events'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; import { startFixture } from './runs_probe_server.mjs'; // Browser acceptance for the Runs destination. // // Build one of the bundles first, then run from app/: // flutter build web // flutter build web --target=tool/runs_probe_entry.dart -o build/probe_web // OPENCODE_ARTIFACT_DIR= node tool/runs_probe.mjs build/web // // Requires Google Chrome and Tesseract (OCR assertions cover canvas-rendered // selectable text that Flutter web does not expose through semantics). const output = process.env.OPENCODE_ARTIFACT_DIR; if (!output) throw new Error('Set OPENCODE_ARTIFACT_DIR for the probe captures'); await mkdir(output, { recursive: true }); const root = resolve(process.argv[2] ?? 'build/probe_web'); const fixture = await startFixture({ root }); const html = await fetch(`${fixture.url}/`); const mime = html.headers.get('content-type') ?? ''; if (html.status !== 200 || !mime.startsWith('text/html')) { throw new Error(`Fixture index must be HTML: ${html.status} ${mime}`); } await html.text(); const profile = await mkdtemp(join(tmpdir(), 'i1-runs-chrome-')); const chrome = spawn('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', [ '--headless=new', '--no-first-run', '--no-default-browser-check', '--disable-gpu', '--remote-debugging-port=0', `--user-data-dir=${profile}`, 'about:blank', ], { stdio: ['ignore', 'ignore', 'pipe'] }); let stderr = ''; chrome.stderr.on('data', (data) => { stderr += data; }); let ws; try { const endpoint = await waitFor(() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1], 'Chrome DevTools'); ws = new WebSocket(endpoint); await once(ws, 'open'); let id = 0; const pending = new Map(); ws.addEventListener('message', (event) => { const message = JSON.parse(event.data); if (message.method?.includes('detached') || message.method?.includes('crashed')) { console.error('CDP event', JSON.stringify(message)); } if (message.method === 'Runtime.exceptionThrown') { console.error('PAGE EXCEPTION', JSON.stringify(message.params?.exceptionDetails)); } if (message.method === 'Log.entryAdded') { console.error('PAGE LOG', JSON.stringify(message.params?.entry)); } if (message.method === 'Runtime.consoleAPICalled') { console.error('PAGE CONSOLE', JSON.stringify(message.params)); } if (message.id && pending.has(message.id)) { pending.get(message.id)(message); pending.delete(message.id); } }); const send = (method, params = {}, sessionId) => new Promise((resolveCall) => { const callId = ++id; pending.set(callId, resolveCall); ws.send(JSON.stringify({ id: callId, method, params, ...(sessionId ? { sessionId } : {}) })); }); const target = (await send('Target.createTarget', { url: 'about:blank' })).result.targetId; const session = (await send('Target.attachToTarget', { targetId: target, flatten: true })).result.sessionId; await send('Page.enable', {}, session); await send('Runtime.enable', {}, session); await send('Log.enable', {}, session); const evaluate = async (expression) => { const response = await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }, session); if (response.error) throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`); if (response.result?.exceptionDetails) { throw new Error(`Page exception: ${JSON.stringify(response.result.exceptionDetails)}`); } return response.result?.result?.value; }; const enableSemantics = async () => { await waitFor( () => evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))"), 'Flutter semantics placeholder', ); await evaluate("document.querySelector('flt-semantics-placeholder')?.click()"); await delay(600); }; const navigation = await send('Page.navigate', { url: `${fixture.url}/` }, session); if (navigation.error || navigation.result?.errorText) throw new Error(JSON.stringify(navigation)); await enableSemantics(); await waitFor( async () => (await evaluate('location.pathname')) === '/pipelines', 'boot route correction to /pipelines', ); const semanticText = () => evaluate(`(() => { const chunks = []; for (const el of document.querySelectorAll('flt-semantics')) { const label = el.getAttribute('aria-label'); if (label) chunks.push(label); const value = el.getAttribute('aria-valuetext'); if (value) chunks.push(value); if (el.value) chunks.push(String(el.value)); if (!el.children.length && el.innerText) chunks.push(el.innerText); } for (const el of document.querySelectorAll('input, textarea')) { const label = el.getAttribute('aria-label'); if (label) chunks.push(label); if (el.value) chunks.push(String(el.value)); } if (document.body?.innerText) chunks.push(document.body.innerText); return [...new Set(chunks)].join('\\n'); })()`); const assertBody = async (text) => { try { await waitFor( async () => (await semanticText()).includes(text), `body to contain ${JSON.stringify(text)}`, ); } catch (error) { const textNow = await semanticText(); let extra = ''; if (process.env.PROBE_DEBUG) { extra = `\nNodes: ${JSON.stringify(await evaluate(`[...document.querySelectorAll('flt-semantics, input, textarea')] .map((el) => ({ tag: el.tagName, role: el.getAttribute('role'), label: el.getAttribute('aria-label'), value: el.getAttribute('aria-valuetext') ?? el.value, text: el.innerText })) .filter((item) => item.label || item.value || item.text)`))}`; } throw new Error(`${error.message}\nBody now:\n${textNow.slice(0, 3000)}${extra}`); } }; const clickText = async (text, { contains = false } = {}) => { const clicked = await evaluate(`(() => { const wanted = ${JSON.stringify(text)}; const candidates = [...document.querySelectorAll('flt-semantics')].filter((el) => { const label = (el.getAttribute('aria-label') || el.innerText || '').trim(); return label === wanted || (${contains} && label.includes(wanted)); }); const interactive = candidates.find((el) => ['button', 'tab', 'link', 'checkbox', 'menuitem'].includes(el.getAttribute('role')), ); const target = interactive ?? candidates[0]; if (!target) return false; target.click(); return true; })()`); if (!clicked) { const labels = await evaluate(`[...document.querySelectorAll('flt-semantics')] .map((el) => (el.getAttribute('aria-label') || '').trim()).filter(Boolean)`); throw new Error(`Could not click ${JSON.stringify(text)}. Labels: ${JSON.stringify(labels)}`); } await delay(300); }; const capture = async (name) => { const data = (await send('Page.captureScreenshot', { format: 'png', captureBeyondViewport: false }, session)).result.data; const path = join(output, `${name}.png`); await writeFile(path, Buffer.from(data, 'base64')); return path; }; const ocr = async (path) => { const { stdout } = await promisify(execFile)('tesseract', [path, 'stdout']); return stdout; }; const assertOcr = async (path, text) => { const read = await ocr(path); if (!read.toLowerCase().includes(text.toLowerCase())) { throw new Error(`OCR of ${path} lacks ${JSON.stringify(text)}:\n${read.slice(0, 3000)}`); } }; const wheel = (deltaY) => send('Input.dispatchMouseEvent', { type: 'mouseWheel', x: 195, y: 500, deltaX: 0, deltaY, }, session); const scrollUntil = async (text) => { for (let attempt = 0; attempt < 20; attempt++) { if ((await semanticText()).includes(text)) return; await wheel(400); await delay(180); } throw new Error(`Could not scroll to ${JSON.stringify(text)}`); }; const setSize = async (width, height) => { await send('Emulation.setDeviceMetricsOverride', { width, height, deviceScaleFactor: 1, mobile: width < 600, }, session); await delay(500); }; await setSize(390, 844); await clickText('Runs'); if ((await evaluate('location.pathname')) !== '/runs') { throw new Error(`Runs tab did not update the URL: ${await evaluate('location.pathname')}`); } await assertBody('Fixture live run'); await assertBody('Fixture pending run'); await assertBody('Fixture failing run'); const captures = [await capture('i1-runs-phone-list')]; await assertOcr(captures[0], 'Fixture live run'); await setSize(1280, 800); await assertBody('Fixture live run'); captures.push(await capture('i1-runs-desktop-list')); await assertOcr(captures[1], 'Fixture live run'); await setSize(390, 844); const live = await fetch(`${fixture.url}/fixture/live`, { method: 'POST' }); if (!live.ok) throw new Error(`Fixture live trigger failed: ${live.status}`); await assertBody('succeeded'); captures.push(await capture('i1-runs-phone-list-live')); await assertOcr(captures[2], 'succeeded'); await clickText('Fixture pending run', { contains: true }); await assertBody('Loading run details…'); await assertBody('Back to list'); captures.push(await capture('i1-runs-phone-pending')); await assertOcr(captures[3], 'Loading run details'); await clickText('Back to list', { contains: true }); await assertBody('Fixture failing run'); await clickText('Fixture failing run', { contains: true }); await assertBody('Fixture detail failure: run 3 is unavailable'); await assertBody('Back to list'); captures.push(await capture('i1-runs-phone-error')); await assertOcr(captures[4], 'run 3 is unavailable'); await clickText('Back to list', { contains: true }); await assertBody('Fixture live run'); await clickText('Fixture live run', { contains: true }); if ((await evaluate('location.pathname')) !== '/runs/1') { throw new Error(`Run selection did not update the URL: ${await evaluate('location.pathname')}`); } await assertBody('Original prompt'); await assertBody('Timeline'); await assertBody('build · visit 3'); captures.push(await capture('i1-runs-phone-detail')); await assertOcr(captures[5], 'fixture prompt with spaces'); await setSize(1280, 800); await assertBody('Original prompt'); captures.push(await capture('i1-runs-desktop-detail')); await assertOcr(captures[6], 'fixture prompt with spaces'); await setSize(390, 844); await clickText('Input'); await wheel(250); await delay(300); captures.push(await capture('i1-runs-phone-detail-input')); await assertOcr(captures[7], 'input payload'); await scrollUntil('Output'); await clickText('Output'); await wheel(250); await delay(300); captures.push(await capture('i1-runs-phone-detail-output')); await assertOcr(captures[8], 'answer'); await scrollUntil('Metrics'); await clickText('Metrics'); await wheel(300); await delay(300); captures.push(await capture('i1-runs-phone-detail-metrics')); await assertOcr(captures[9], 'Total tokens'); await assertOcr(captures[9], 'Reported cost'); // Browser Back/Forward move between the in-app routes, not out of the app. const path = () => evaluate('location.pathname'); await evaluate('history.back()'); await waitFor( async () => (await path()) === '/runs', 'browser Back to /runs', ); await waitFor( async () => (await semanticText()).includes('Fixture live run'), 'browser Back to the runs list', ); await waitFor( async () => (await semanticText()).includes('Fixture live run'), 'browser Back to the runs list', ); captures.push(await capture('i1-runs-phone-browser-back-list')); await assertOcr(captures[10], 'Fixture live run'); await evaluate('history.forward()'); await waitFor( async () => (await path()) === '/runs/1' && (await semanticText()).includes('Original prompt'), 'browser Forward to /runs/1', ); captures.push(await capture('i1-runs-phone-browser-forward-detail')); // A direct /runs/1 load is a real deep link: the detail opens and the URL stays. await send('Page.navigate', { url: `${fixture.url}/runs/1` }, session); await enableSemantics(); await assertBody('Original prompt'); if ((await path()) !== '/runs/1') throw new Error(`Deep link moved to ${await path()}`); captures.push(await capture('i1-deeplink-run-1-phone')); // A drop keeps the open detail, reports reconnecting rather than unavailable, // and has no raw transport exception anywhere on the page. const beforeDrop = await (await fetch(`${fixture.url}/fixture/state`)).json(); const refused = await fetch(`${fixture.url}/fixture/refuse`, { method: 'POST' }); if (!refused.ok) throw new Error(`Fixture refuse failed: ${refused.status}`); await assertBody('Reconnecting to Pipelines…'); const duringDrop = await semanticText(); for (const raw of ['WebSocketException', 'WebSocketChannelException', 'Failed to connect']) { if (duringDrop.includes(raw)) throw new Error(`Raw transport text shown: ${raw}`); } if ((await path()) !== '/runs/1') throw new Error(`Drop moved the URL to ${await path()}`); captures.push(await capture('i1-reconnect-detail-retained-phone')); await assertOcr(captures[13], 'Original prompt'); // Accepting again plus Retry restores the same detail without a remount. const accepted = await fetch(`${fixture.url}/fixture/accept`, { method: 'POST' }); if (!accepted.ok) throw new Error(`Fixture accept failed: ${accepted.status}`); await clickText('Retry'); await waitFor( async () => !(await semanticText()).includes('Reconnecting to Pipelines…') && (await semanticText()).includes('Original prompt'), 'detail restored after reconnect', ); if ((await path()) !== '/runs/1') throw new Error(`Reconnect moved the URL to ${await path()}`); captures.push(await capture('i1-reconnect-detail-restored-phone')); // A first connection while the server refuses upgrades is friendly: the gate // never prints the transport exception and does not flicker to "connecting". await fetch(`${fixture.url}/fixture/refuse`, { method: 'POST' }); await send('Page.navigate', { url: `${fixture.url}/runs` }, session); await enableSemantics(); await assertBody('Pipelines server unavailable'); const gate = await semanticText(); for (const raw of ['WebSocketException', 'WebSocketChannelException', 'Failed to connect']) { if (gate.includes(raw)) throw new Error(`Gate printed raw transport text: ${raw}`); } captures.push(await capture('i1-gate-unavailable-friendly-phone')); await delay(1500); if (!(await semanticText()).includes('Pipelines server unavailable')) { throw new Error('Gate left the unavailable state while retrying'); } await assertOcr(captures[15], 'Pipelines server unavailable'); await fetch(`${fixture.url}/fixture/accept`, { method: 'POST' }); await clickText('Retry'); await assertBody('Fixture live run'); captures.push(await capture('i1-gate-recovered-list-phone')); await clickText('Settings'); await assertBody('Server URL'); await assertBody('Fixture settings build'); if ((await path()) !== '/settings') throw new Error('Settings did not update route'); captures.push(await capture('settings-cupertino-registered-phone')); // Material host: the same deep link and route must render standard Material // controls (the Cupertino host was selected by the macOS browser above). await send( 'Emulation.setUserAgentOverride', { userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36', platform: 'Linux x86_64', }, session, ); await send('Page.navigate', { url: `${fixture.url}/runs/1` }, session); await enableSemantics(); await assertBody('Original prompt'); await assertBody('Back to list'); if ((await path()) !== '/runs/1') { throw new Error(`Material deep link moved to ${await path()}`); } captures.push(await capture('i1-material-deeplink-run-1-phone')); const beforeSettings = await (await fetch(`${fixture.url}/fixture/state`)).json(); await clickText('Settings'); await assertBody('Server URL'); await assertBody('Fixture settings build'); const afterSettings = await (await fetch(`${fixture.url}/fixture/state`)).json(); if (beforeSettings.subscribed !== afterSettings.subscribed || afterSettings.connections !== 1) { throw new Error('Settings opened another connection instead of sharing the app socket'); } captures.push(await capture('settings-material-registered-phone')); const fixtureState = await (await fetch(`${fixture.url}/fixture/state`)).json(); if (fixtureState.subscribed < 2) throw new Error(`App never resubscribed: ${JSON.stringify(fixtureState)}`); if (fixtureState.listRuns < 2) throw new Error(`App never reloaded: ${JSON.stringify(fixtureState)}`); if (fixtureState.getRun < 2) throw new Error(`Detail was not rehydrated: ${JSON.stringify(fixtureState)}`); if (fixtureState.dropped < 1) throw new Error(`Fixture never dropped: ${JSON.stringify(fixtureState)}`); console.log(JSON.stringify({ ok: true, fixture: fixture.url, subscribed: fixtureState.subscribed, listRuns: fixtureState.listRuns, getRun: fixtureState.getRun, dropped: fixtureState.dropped, live: fixtureState.live, captures, }, null, 2)); } finally { ws?.close(); chrome.kill('SIGTERM'); await Promise.race([once(chrome, 'exit'), delay(3000)]); await fixture.close(); await rm(profile, { recursive: true, force: true }); } async function waitFor(check, label, attempts = 160) { for (let attempt = 0; attempt < attempts; attempt++) { const value = await check(); if (value) return value; await delay(100); } throw new Error(`Timed out waiting for ${label}`); }