309 lines
14 KiB
JavaScript
309 lines
14 KiB
JavaScript
import { createServer } from 'node:http';
|
|
import { readFile, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
import { spawn, execFile } from 'node:child_process';
|
|
import { promisify } from 'node:util';
|
|
import { once } from 'node:events';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { join, extname, resolve, sep } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
|
|
// Real Chrome acceptance for the chat composer strip.
|
|
//
|
|
// Usage (from app/):
|
|
// flutter build web -t tool/chat_composer_probe_entry.dart \
|
|
// --dart-define=CHAT_COMPOSER_PROBE_STYLE=material -o build/composer_probe_material
|
|
// flutter build web -t tool/chat_composer_probe_entry.dart \
|
|
// --dart-define=CHAT_COMPOSER_PROBE_STYLE=cupertino -o build/composer_probe_cupertino
|
|
// OPENCODE_ARTIFACT_DIR=<dir> PROBE_VARIANT=material \
|
|
// node tool/chat_composer_probe.mjs build/composer_probe_material
|
|
// OPENCODE_ARTIFACT_DIR=<dir> PROBE_VARIANT=cupertino \
|
|
// node tool/chat_composer_probe.mjs build/composer_probe_cupertino
|
|
//
|
|
// The fixture is served over loopback only. Flutter paints text on a canvas, so
|
|
// assertions read the semantics tree (aria-label/innerText) and the focused
|
|
// editing element; captures are PNGs for visual inspection.
|
|
|
|
const variant = process.env.PROBE_VARIANT ?? 'material';
|
|
const artifactDir = process.env.OPENCODE_ARTIFACT_DIR;
|
|
if (!artifactDir) throw new Error('Set OPENCODE_ARTIFACT_DIR for the captures');
|
|
await mkdir(artifactDir, { recursive: true });
|
|
const root = resolve(process.argv[2] ?? join(process.cwd(), 'build/composer_probe_material'));
|
|
|
|
const mime = {
|
|
'.html': 'text/html',
|
|
'.js': 'text/javascript',
|
|
'.json': 'application/json',
|
|
'.wasm': 'application/wasm',
|
|
'.png': 'image/png',
|
|
'.ico': 'image/x-icon',
|
|
'.ttf': 'font/ttf',
|
|
'.otf': 'font/otf',
|
|
'.woff': 'font/woff',
|
|
'.woff2': 'font/woff2',
|
|
};
|
|
const server = createServer(async (req, res) => {
|
|
try {
|
|
const path = resolve(root, '.' + decodeURIComponent(new URL(req.url, 'http://localhost').pathname));
|
|
if (path !== root && !path.startsWith(root + sep)) return res.writeHead(403).end();
|
|
const filePath = path === root || path.endsWith('/') ? join(path, 'index.html') : path;
|
|
const file = await readFile(filePath);
|
|
res.writeHead(200, { 'content-type': mime[extname(filePath)] ?? 'application/octet-stream' }).end(file);
|
|
} catch {
|
|
res.writeHead(404).end('not found');
|
|
}
|
|
});
|
|
server.listen(0, '127.0.0.1');
|
|
await once(server, 'listening');
|
|
const port = server.address().port;
|
|
const profile = await mkdtemp(join(tmpdir(), 'chat-composer-chrome-'));
|
|
const chrome = spawn('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', [
|
|
'--headless=new', '--no-first-run', '--no-default-browser-check', '--disable-gpu',
|
|
'--remote-debugging-port=0', `--user-data-dir=${profile}`, 'about:blank',
|
|
], { stdio: ['ignore', 'ignore', 'pipe'] });
|
|
let stderr = '';
|
|
chrome.stderr.on('data', data => { stderr += data; });
|
|
|
|
let ws;
|
|
try {
|
|
const waitFor = async (fn, what) => {
|
|
for (let i = 0; i < 120; i++) { const value = await fn(); if (value) return value; await delay(100); }
|
|
throw new Error(`Timed out waiting for ${what}`);
|
|
};
|
|
const endpoint = await waitFor(
|
|
() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1],
|
|
'Chrome DevTools',
|
|
);
|
|
ws = new WebSocket(endpoint);
|
|
await once(ws, 'open');
|
|
let id = 0;
|
|
const pending = new Map();
|
|
ws.addEventListener('message', event => {
|
|
const msg = JSON.parse(event.data);
|
|
if (msg.method?.includes('detached') || msg.method?.includes('crashed')) {
|
|
console.error('CDP event', JSON.stringify(msg));
|
|
}
|
|
if (msg.method === 'Runtime.exceptionThrown') {
|
|
console.error('PAGE EXCEPTION', JSON.stringify(msg.params?.exceptionDetails?.exception?.description ?? msg.params?.exceptionDetails?.text));
|
|
}
|
|
if (msg.id && pending.has(msg.id)) { pending.get(msg.id)(msg); pending.delete(msg.id); }
|
|
});
|
|
const send = (method, params = {}, sessionId) => new Promise(resolve => {
|
|
const callId = ++id;
|
|
pending.set(callId, resolve);
|
|
ws.send(JSON.stringify({ id: callId, method, params, ...(sessionId ? { sessionId } : {}) }));
|
|
});
|
|
const created = await send('Target.createTarget', { url: 'about:blank' });
|
|
if (!created.result) throw new Error(`Target creation failed: ${JSON.stringify(created)}`);
|
|
const session = (await send('Target.attachToTarget', { targetId: created.result.targetId, flatten: true })).result.sessionId;
|
|
await send('Page.enable', {}, session);
|
|
await send('Runtime.enable', {}, session);
|
|
const navigation = await send('Page.navigate', { url: `http://127.0.0.1:${port}/` }, session);
|
|
if (navigation.error || navigation.result?.errorText) throw new Error(JSON.stringify(navigation));
|
|
const evaluate = async expression => {
|
|
const response = await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }, session);
|
|
if (response.error) throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
|
|
return response.result?.result?.value;
|
|
};
|
|
// A cold bundle can need longer than a fixed delay before Flutter publishes
|
|
// its semantics placeholder; wait for it instead of sampling a blank page.
|
|
for (let i = 0; i < 120; i++) {
|
|
if (await evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))")) break;
|
|
await delay(100);
|
|
}
|
|
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
|
|
await delay(1800);
|
|
|
|
const labels = async () =>
|
|
evaluate("[...document.querySelectorAll('flt-semantics')].map(x => x.getAttribute('aria-label') || x.innerText || '').filter(Boolean)");
|
|
const semanticText = async () => (await labels()).join('\n');
|
|
const assertSemantics = async (text, what) => {
|
|
const read = await semanticText();
|
|
if (!read.toLowerCase().replace(/\s+/g, ' ').includes(text.toLowerCase())) {
|
|
throw new Error(`${what}: semantics lacks ${JSON.stringify(text)}\nbody: ${(await evaluate('document.body?.innerText ?? \"\"') || '').slice(0, 800)}\n${read.slice(0, 2000)}`);
|
|
}
|
|
};
|
|
const clickLabel = async (label, { contains = false } = {}) => {
|
|
const point = await evaluate(`(() => {
|
|
const needle = ${JSON.stringify(label)};
|
|
const nodes = [...document.querySelectorAll('flt-semantics')];
|
|
const e = nodes.find(x => {
|
|
const text = x.getAttribute('aria-label') || x.innerText || '';
|
|
return ${contains ? 'text.includes(needle)' : 'text === needle'};
|
|
});
|
|
if (!e) return null;
|
|
const r = e.getBoundingClientRect();
|
|
return { x: r.x + r.width / 2, y: r.y + r.height / 2 };
|
|
})()`);
|
|
if (!point) return false;
|
|
await send('Input.dispatchMouseEvent', { type: 'mousePressed', x: point.x, y: point.y, button: 'left', clickCount: 1 }, session);
|
|
await send('Input.dispatchMouseEvent', { type: 'mouseReleased', x: point.x, y: point.y, button: 'left', clickCount: 1 }, session);
|
|
return true;
|
|
};
|
|
const focusField = async () => {
|
|
const focused = await evaluate(`(() => {
|
|
const needle = 'Message the agent';
|
|
const nodes = [...document.querySelectorAll('textarea, input')];
|
|
const node = nodes.find(n => (n.getAttribute('aria-label') || n.placeholder || '').includes(needle));
|
|
if (!node) return null;
|
|
node.focus();
|
|
const r = node.getBoundingClientRect();
|
|
return { tag: node.tagName, label: node.getAttribute('aria-label'), rect: { x: r.x, y: r.y, w: r.width, h: r.height } };
|
|
})()`);
|
|
if (!focused) {
|
|
const read = await evaluate("[...document.querySelectorAll('textarea, input')].map(x => ({tag:x.tagName,label:x.getAttribute('aria-label'),placeholder:x.placeholder,value:x.value}))");
|
|
throw new Error(`Composer field was not exposed to semantics: ${JSON.stringify(read)}`);
|
|
}
|
|
await delay(500);
|
|
const editing = await editingValue();
|
|
if (!editing) throw new Error('The composer did not expose an editing element');
|
|
return { ...focused, ...editing };
|
|
};
|
|
const editingValue = async () => evaluate(`(() => {
|
|
const match = n => (n.getAttribute('aria-label') || n.getAttribute('placeholder') || '').includes('Message the agent');
|
|
const node = document.querySelector('input.flt-text-editing, textarea.flt-text-editing')
|
|
|| [...document.querySelectorAll('textarea, input')].find(match)
|
|
|| null;
|
|
return node ? { tag: node.tagName, value: node.value } : null;
|
|
})()`);
|
|
const type = async text => { await send('Input.insertText', { text }, session); await delay(250); };
|
|
const key = async (type, params) => send('Input.dispatchKeyEvent', { type, ...params }, session);
|
|
const pressEnter = async modifiers => {
|
|
if (modifiers === 0) {
|
|
await key('keyDown', {
|
|
key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
text: '\r', unmodifiedText: '\r',
|
|
});
|
|
} else {
|
|
const modifier = modifiers === 4
|
|
? { key: 'Meta', code: 'MetaLeft', windowsVirtualKeyCode: 91, nativeVirtualKeyCode: 55 }
|
|
: { key: 'Control', code: 'ControlLeft', windowsVirtualKeyCode: 17, nativeVirtualKeyCode: 59 };
|
|
await key('rawKeyDown', { ...modifier, modifiers });
|
|
await key('rawKeyDown', {
|
|
modifiers, key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
});
|
|
await key('keyUp', {
|
|
modifiers, key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
});
|
|
await key('keyUp', { ...modifier, modifiers: 0 });
|
|
await delay(500);
|
|
return;
|
|
}
|
|
await key('keyUp', {
|
|
modifiers, key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
});
|
|
await delay(500);
|
|
};
|
|
const setSize = async (width, height, scale = 1) => {
|
|
await send('Emulation.setDeviceMetricsOverride', {
|
|
width, height, deviceScaleFactor: scale, mobile: width < 600,
|
|
}, session);
|
|
await delay(400);
|
|
};
|
|
const captures = [];
|
|
const capture = async name => {
|
|
const data = (await send('Page.captureScreenshot', { format: 'png', captureBeyondViewport: false }, session)).result.data;
|
|
const file = `${variant}-${name}.png`;
|
|
await writeFile(`${artifactDir}/${file}`, Buffer.from(data, 'base64'));
|
|
captures.push(file);
|
|
return file;
|
|
};
|
|
const ocr = async file =>
|
|
(await promisify(execFile)('tesseract', [`${artifactDir}/${file}`, 'stdout'])).stdout;
|
|
const assertOcr = async (file, text) => {
|
|
const read = await ocr(file);
|
|
if (!read.toLowerCase().replace(/\s+/g, ' ').includes(text.toLowerCase())) {
|
|
throw new Error(`OCR of ${file} lacks ${JSON.stringify(text)}:\n${read.slice(0, 2000)}`);
|
|
}
|
|
};
|
|
|
|
const checks = [];
|
|
await setSize(390, 844);
|
|
await assertSemantics('Composer fixture', 'The fixture thread');
|
|
const initial = await editingValue();
|
|
if (initial?.value !== '') throw new Error(`The composer started with ${JSON.stringify(initial)}`);
|
|
await capture('composer-initial');
|
|
const focused = await focusField();
|
|
checks.push({ check: 'field exposed as an editing element', focused });
|
|
|
|
await type('First line');
|
|
await pressEnter(0);
|
|
const afterEnter = await editingValue();
|
|
if (!afterEnter?.value?.includes('\n')) {
|
|
throw new Error(`Enter did not insert a newline: ${JSON.stringify(afterEnter)}`);
|
|
}
|
|
await type('Second line');
|
|
const multiline = await editingValue();
|
|
if (multiline?.value !== 'First line\nSecond line') {
|
|
throw new Error(`Multiline draft is ${JSON.stringify(multiline)}`);
|
|
}
|
|
const multilineCapture = await capture('composer-multiline');
|
|
await assertOcr(multilineCapture, 'Second');
|
|
checks.push({ check: 'Enter inserted a newline without sending', value: multiline.value });
|
|
|
|
const usersBefore = (await labels()).filter(label => label === 'User message').length;
|
|
await pressEnter(variant === 'cupertino' ? 4 : 2);
|
|
await waitFor(async () => (await semanticText()).includes('Stop'), 'Stop after send');
|
|
const usersAfter = (await labels()).filter(label => label === 'User message').length;
|
|
if (usersAfter <= usersBefore) {
|
|
throw new Error(`No optimistic user row appeared (${usersBefore} -> ${usersAfter})`);
|
|
}
|
|
const cleared = await editingValue();
|
|
if (cleared?.value !== '') throw new Error(`Send left the field as ${JSON.stringify(cleared)}`);
|
|
const runningCapture = await capture('composer-running');
|
|
await assertOcr(runningCapture, 'First');
|
|
checks.push({ check: 'Ctrl/Cmd+Enter sent and the optimistic turn is visible', cleared });
|
|
|
|
// The reply that carries `chat_queued` lands after the running frame.
|
|
await delay(700);
|
|
await focusField();
|
|
await type('Queued follow-up');
|
|
const queuedDraft = await editingValue();
|
|
if (process.env.PROBE_DEBUG) console.log('queued draft', JSON.stringify(queuedDraft));
|
|
await pressEnter(variant === 'cupertino' ? 4 : 2);
|
|
await delay(900);
|
|
const queuedCapture = await capture('composer-queued');
|
|
const queuedRead = await ocr(queuedCapture);
|
|
if (process.env.PROBE_DEBUG) console.log('queued OCR', JSON.stringify(queuedRead.slice(0, 500)));
|
|
if (!queuedRead.toLowerCase().replace(/\s+/g, ' ').includes('current turn finishes')) {
|
|
throw new Error(`The queued notice did not render:\n${queuedRead.slice(0, 2000)}`);
|
|
}
|
|
checks.push({ check: 'the queued reply shows its polite notice while Stop stays' });
|
|
|
|
if (!await clickLabel('Stop')) throw new Error('Could not press Stop');
|
|
await waitFor(async () => !(await semanticText()).includes('Stop'), 'the stopped turn');
|
|
await capture('composer-stopped');
|
|
checks.push({ check: 'Stop interrupted the turn and Send returned' });
|
|
|
|
await focusField();
|
|
await type('refuse me');
|
|
await pressEnter(variant === 'cupertino' ? 4 : 2);
|
|
await waitFor(
|
|
async () => (await semanticText()).includes('The fixture refused this prompt.'),
|
|
'the refusal line',
|
|
);
|
|
const restored = await editingValue();
|
|
if (restored?.value !== 'refuse me') {
|
|
throw new Error(`Refused draft was not restored: ${JSON.stringify(restored)}`);
|
|
}
|
|
await capture('composer-refused');
|
|
await assertOcr(`${variant}-composer-refused.png`, 'refuse me');
|
|
checks.push({ check: 'the refusal restored the exact draft and reported it', restored });
|
|
|
|
await setSize(320, 640, 2);
|
|
await delay(500);
|
|
const wrapped = await semanticText();
|
|
if (!wrapped.includes('Composer fixture') || !wrapped.includes('Send')) {
|
|
throw new Error(`The 320px/2x layout lost its content:\n${wrapped.slice(0, 2000)}`);
|
|
}
|
|
await capture('composer-320-2x');
|
|
checks.push({ check: 'the composer still renders at 320 logical px with 2x pixels' });
|
|
|
|
console.log(JSON.stringify({ variant, checks, captures }, null, 2));
|
|
} finally {
|
|
ws?.close();
|
|
chrome.kill('SIGTERM');
|
|
await Promise.race([once(chrome, 'exit'), delay(3000)]);
|
|
server.close();
|
|
await rm(profile, { recursive: true, force: true });
|
|
}
|