Files
pipelines/app/tool/chat_composer_probe.mjs
T

309 lines
14 KiB
JavaScript

import { createServer } from 'node:http';
import { readFile, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { spawn, execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { once } from 'node:events';
import { setTimeout as delay } from 'node:timers/promises';
import { join, extname, resolve, sep } from 'node:path';
import { tmpdir } from 'node:os';
// Real Chrome acceptance for the chat composer strip.
//
// Usage (from app/):
// flutter build web -t tool/chat_composer_probe_entry.dart \
// --dart-define=CHAT_COMPOSER_PROBE_STYLE=material -o build/composer_probe_material
// flutter build web -t tool/chat_composer_probe_entry.dart \
// --dart-define=CHAT_COMPOSER_PROBE_STYLE=cupertino -o build/composer_probe_cupertino
// OPENCODE_ARTIFACT_DIR=<dir> PROBE_VARIANT=material \
// node tool/chat_composer_probe.mjs build/composer_probe_material
// OPENCODE_ARTIFACT_DIR=<dir> PROBE_VARIANT=cupertino \
// node tool/chat_composer_probe.mjs build/composer_probe_cupertino
//
// The fixture is served over loopback only. Flutter paints text on a canvas, so
// assertions read the semantics tree (aria-label/innerText) and the focused
// editing element; captures are PNGs for visual inspection.
const variant = process.env.PROBE_VARIANT ?? 'material';
const artifactDir = process.env.OPENCODE_ARTIFACT_DIR;
if (!artifactDir) throw new Error('Set OPENCODE_ARTIFACT_DIR for the captures');
await mkdir(artifactDir, { recursive: true });
const root = resolve(process.argv[2] ?? join(process.cwd(), 'build/composer_probe_material'));
const mime = {
'.html': 'text/html',
'.js': 'text/javascript',
'.json': 'application/json',
'.wasm': 'application/wasm',
'.png': 'image/png',
'.ico': 'image/x-icon',
'.ttf': 'font/ttf',
'.otf': 'font/otf',
'.woff': 'font/woff',
'.woff2': 'font/woff2',
};
const server = createServer(async (req, res) => {
try {
const path = resolve(root, '.' + decodeURIComponent(new URL(req.url, 'http://localhost').pathname));
if (path !== root && !path.startsWith(root + sep)) return res.writeHead(403).end();
const filePath = path === root || path.endsWith('/') ? join(path, 'index.html') : path;
const file = await readFile(filePath);
res.writeHead(200, { 'content-type': mime[extname(filePath)] ?? 'application/octet-stream' }).end(file);
} catch {
res.writeHead(404).end('not found');
}
});
server.listen(0, '127.0.0.1');
await once(server, 'listening');
const port = server.address().port;
const profile = await mkdtemp(join(tmpdir(), 'chat-composer-chrome-'));
const chrome = spawn('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', [
'--headless=new', '--no-first-run', '--no-default-browser-check', '--disable-gpu',
'--remote-debugging-port=0', `--user-data-dir=${profile}`, 'about:blank',
], { stdio: ['ignore', 'ignore', 'pipe'] });
let stderr = '';
chrome.stderr.on('data', data => { stderr += data; });
let ws;
try {
const waitFor = async (fn, what) => {
for (let i = 0; i < 120; i++) { const value = await fn(); if (value) return value; await delay(100); }
throw new Error(`Timed out waiting for ${what}`);
};
const endpoint = await waitFor(
() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1],
'Chrome DevTools',
);
ws = new WebSocket(endpoint);
await once(ws, 'open');
let id = 0;
const pending = new Map();
ws.addEventListener('message', event => {
const msg = JSON.parse(event.data);
if (msg.method?.includes('detached') || msg.method?.includes('crashed')) {
console.error('CDP event', JSON.stringify(msg));
}
if (msg.method === 'Runtime.exceptionThrown') {
console.error('PAGE EXCEPTION', JSON.stringify(msg.params?.exceptionDetails?.exception?.description ?? msg.params?.exceptionDetails?.text));
}
if (msg.id && pending.has(msg.id)) { pending.get(msg.id)(msg); pending.delete(msg.id); }
});
const send = (method, params = {}, sessionId) => new Promise(resolve => {
const callId = ++id;
pending.set(callId, resolve);
ws.send(JSON.stringify({ id: callId, method, params, ...(sessionId ? { sessionId } : {}) }));
});
const created = await send('Target.createTarget', { url: 'about:blank' });
if (!created.result) throw new Error(`Target creation failed: ${JSON.stringify(created)}`);
const session = (await send('Target.attachToTarget', { targetId: created.result.targetId, flatten: true })).result.sessionId;
await send('Page.enable', {}, session);
await send('Runtime.enable', {}, session);
const navigation = await send('Page.navigate', { url: `http://127.0.0.1:${port}/` }, session);
if (navigation.error || navigation.result?.errorText) throw new Error(JSON.stringify(navigation));
const evaluate = async expression => {
const response = await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }, session);
if (response.error) throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
return response.result?.result?.value;
};
// A cold bundle can need longer than a fixed delay before Flutter publishes
// its semantics placeholder; wait for it instead of sampling a blank page.
for (let i = 0; i < 120; i++) {
if (await evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))")) break;
await delay(100);
}
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
await delay(1800);
const labels = async () =>
evaluate("[...document.querySelectorAll('flt-semantics')].map(x => x.getAttribute('aria-label') || x.innerText || '').filter(Boolean)");
const semanticText = async () => (await labels()).join('\n');
const assertSemantics = async (text, what) => {
const read = await semanticText();
if (!read.toLowerCase().replace(/\s+/g, ' ').includes(text.toLowerCase())) {
throw new Error(`${what}: semantics lacks ${JSON.stringify(text)}\nbody: ${(await evaluate('document.body?.innerText ?? \"\"') || '').slice(0, 800)}\n${read.slice(0, 2000)}`);
}
};
const clickLabel = async (label, { contains = false } = {}) => {
const point = await evaluate(`(() => {
const needle = ${JSON.stringify(label)};
const nodes = [...document.querySelectorAll('flt-semantics')];
const e = nodes.find(x => {
const text = x.getAttribute('aria-label') || x.innerText || '';
return ${contains ? 'text.includes(needle)' : 'text === needle'};
});
if (!e) return null;
const r = e.getBoundingClientRect();
return { x: r.x + r.width / 2, y: r.y + r.height / 2 };
})()`);
if (!point) return false;
await send('Input.dispatchMouseEvent', { type: 'mousePressed', x: point.x, y: point.y, button: 'left', clickCount: 1 }, session);
await send('Input.dispatchMouseEvent', { type: 'mouseReleased', x: point.x, y: point.y, button: 'left', clickCount: 1 }, session);
return true;
};
const focusField = async () => {
const focused = await evaluate(`(() => {
const needle = 'Message the agent';
const nodes = [...document.querySelectorAll('textarea, input')];
const node = nodes.find(n => (n.getAttribute('aria-label') || n.placeholder || '').includes(needle));
if (!node) return null;
node.focus();
const r = node.getBoundingClientRect();
return { tag: node.tagName, label: node.getAttribute('aria-label'), rect: { x: r.x, y: r.y, w: r.width, h: r.height } };
})()`);
if (!focused) {
const read = await evaluate("[...document.querySelectorAll('textarea, input')].map(x => ({tag:x.tagName,label:x.getAttribute('aria-label'),placeholder:x.placeholder,value:x.value}))");
throw new Error(`Composer field was not exposed to semantics: ${JSON.stringify(read)}`);
}
await delay(500);
const editing = await editingValue();
if (!editing) throw new Error('The composer did not expose an editing element');
return { ...focused, ...editing };
};
const editingValue = async () => evaluate(`(() => {
const match = n => (n.getAttribute('aria-label') || n.getAttribute('placeholder') || '').includes('Message the agent');
const node = document.querySelector('input.flt-text-editing, textarea.flt-text-editing')
|| [...document.querySelectorAll('textarea, input')].find(match)
|| null;
return node ? { tag: node.tagName, value: node.value } : null;
})()`);
const type = async text => { await send('Input.insertText', { text }, session); await delay(250); };
const key = async (type, params) => send('Input.dispatchKeyEvent', { type, ...params }, session);
const pressEnter = async modifiers => {
if (modifiers === 0) {
await key('keyDown', {
key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
text: '\r', unmodifiedText: '\r',
});
} else {
const modifier = modifiers === 4
? { key: 'Meta', code: 'MetaLeft', windowsVirtualKeyCode: 91, nativeVirtualKeyCode: 55 }
: { key: 'Control', code: 'ControlLeft', windowsVirtualKeyCode: 17, nativeVirtualKeyCode: 59 };
await key('rawKeyDown', { ...modifier, modifiers });
await key('rawKeyDown', {
modifiers, key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
});
await key('keyUp', {
modifiers, key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
});
await key('keyUp', { ...modifier, modifiers: 0 });
await delay(500);
return;
}
await key('keyUp', {
modifiers, key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
});
await delay(500);
};
const setSize = async (width, height, scale = 1) => {
await send('Emulation.setDeviceMetricsOverride', {
width, height, deviceScaleFactor: scale, mobile: width < 600,
}, session);
await delay(400);
};
const captures = [];
const capture = async name => {
const data = (await send('Page.captureScreenshot', { format: 'png', captureBeyondViewport: false }, session)).result.data;
const file = `${variant}-${name}.png`;
await writeFile(`${artifactDir}/${file}`, Buffer.from(data, 'base64'));
captures.push(file);
return file;
};
const ocr = async file =>
(await promisify(execFile)('tesseract', [`${artifactDir}/${file}`, 'stdout'])).stdout;
const assertOcr = async (file, text) => {
const read = await ocr(file);
if (!read.toLowerCase().replace(/\s+/g, ' ').includes(text.toLowerCase())) {
throw new Error(`OCR of ${file} lacks ${JSON.stringify(text)}:\n${read.slice(0, 2000)}`);
}
};
const checks = [];
await setSize(390, 844);
await assertSemantics('Composer fixture', 'The fixture thread');
const initial = await editingValue();
if (initial?.value !== '') throw new Error(`The composer started with ${JSON.stringify(initial)}`);
await capture('composer-initial');
const focused = await focusField();
checks.push({ check: 'field exposed as an editing element', focused });
await type('First line');
await pressEnter(0);
const afterEnter = await editingValue();
if (!afterEnter?.value?.includes('\n')) {
throw new Error(`Enter did not insert a newline: ${JSON.stringify(afterEnter)}`);
}
await type('Second line');
const multiline = await editingValue();
if (multiline?.value !== 'First line\nSecond line') {
throw new Error(`Multiline draft is ${JSON.stringify(multiline)}`);
}
const multilineCapture = await capture('composer-multiline');
await assertOcr(multilineCapture, 'Second');
checks.push({ check: 'Enter inserted a newline without sending', value: multiline.value });
const usersBefore = (await labels()).filter(label => label === 'User message').length;
await pressEnter(variant === 'cupertino' ? 4 : 2);
await waitFor(async () => (await semanticText()).includes('Stop'), 'Stop after send');
const usersAfter = (await labels()).filter(label => label === 'User message').length;
if (usersAfter <= usersBefore) {
throw new Error(`No optimistic user row appeared (${usersBefore} -> ${usersAfter})`);
}
const cleared = await editingValue();
if (cleared?.value !== '') throw new Error(`Send left the field as ${JSON.stringify(cleared)}`);
const runningCapture = await capture('composer-running');
await assertOcr(runningCapture, 'First');
checks.push({ check: 'Ctrl/Cmd+Enter sent and the optimistic turn is visible', cleared });
// The reply that carries `chat_queued` lands after the running frame.
await delay(700);
await focusField();
await type('Queued follow-up');
const queuedDraft = await editingValue();
if (process.env.PROBE_DEBUG) console.log('queued draft', JSON.stringify(queuedDraft));
await pressEnter(variant === 'cupertino' ? 4 : 2);
await delay(900);
const queuedCapture = await capture('composer-queued');
const queuedRead = await ocr(queuedCapture);
if (process.env.PROBE_DEBUG) console.log('queued OCR', JSON.stringify(queuedRead.slice(0, 500)));
if (!queuedRead.toLowerCase().replace(/\s+/g, ' ').includes('current turn finishes')) {
throw new Error(`The queued notice did not render:\n${queuedRead.slice(0, 2000)}`);
}
checks.push({ check: 'the queued reply shows its polite notice while Stop stays' });
if (!await clickLabel('Stop')) throw new Error('Could not press Stop');
await waitFor(async () => !(await semanticText()).includes('Stop'), 'the stopped turn');
await capture('composer-stopped');
checks.push({ check: 'Stop interrupted the turn and Send returned' });
await focusField();
await type('refuse me');
await pressEnter(variant === 'cupertino' ? 4 : 2);
await waitFor(
async () => (await semanticText()).includes('The fixture refused this prompt.'),
'the refusal line',
);
const restored = await editingValue();
if (restored?.value !== 'refuse me') {
throw new Error(`Refused draft was not restored: ${JSON.stringify(restored)}`);
}
await capture('composer-refused');
await assertOcr(`${variant}-composer-refused.png`, 'refuse me');
checks.push({ check: 'the refusal restored the exact draft and reported it', restored });
await setSize(320, 640, 2);
await delay(500);
const wrapped = await semanticText();
if (!wrapped.includes('Composer fixture') || !wrapped.includes('Send')) {
throw new Error(`The 320px/2x layout lost its content:\n${wrapped.slice(0, 2000)}`);
}
await capture('composer-320-2x');
checks.push({ check: 'the composer still renders at 320 logical px with 2x pixels' });
console.log(JSON.stringify({ variant, checks, captures }, null, 2));
} finally {
ws?.close();
chrome.kill('SIGTERM');
await Promise.race([once(chrome, 'exit'), delay(3000)]);
server.close();
await rm(profile, { recursive: true, force: true });
}