b60ad6d98f
Keep the probe suite on the Material composition so browser fixtures and capture drivers no longer render or select Cupertino hosts. Record the final residual legacy arguments and implementation checks for the delivery gate.
309 lines
14 KiB
JavaScript
309 lines
14 KiB
JavaScript
import { createServer } from 'node:http';
|
|
import { readFile, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
import { spawn, execFile } from 'node:child_process';
|
|
import { promisify } from 'node:util';
|
|
import { once } from 'node:events';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { join, extname, resolve, sep } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
|
|
// Real Chrome acceptance for the chat composer strip.
|
|
//
|
|
// Usage (from app/):
|
|
// flutter build web -t tool/chat_composer_probe_entry.dart \
|
|
// --dart-define=CHAT_COMPOSER_PROBE_STYLE=material -o build/composer_probe_material
|
|
// flutter build web -t tool/chat_composer_probe_entry.dart \
|
|
// --dart-define=CHAT_COMPOSER_PROBE_STYLE=material -o build/composer_probe_material
|
|
// OPENCODE_ARTIFACT_DIR=<dir> \
|
|
// node tool/chat_composer_probe.mjs build/composer_probe_material
|
|
// OPENCODE_ARTIFACT_DIR=<dir> \
|
|
// node tool/chat_composer_probe.mjs build/composer_probe_material
|
|
//
|
|
// The fixture is served over loopback only. Flutter paints text on a canvas, so
|
|
// assertions read the semantics tree (aria-label/innerText) and the focused
|
|
// editing element; captures are PNGs for visual inspection.
|
|
|
|
const variant = 'material';
|
|
const artifactDir = process.env.OPENCODE_ARTIFACT_DIR;
|
|
if (!artifactDir) throw new Error('Set OPENCODE_ARTIFACT_DIR for the captures');
|
|
await mkdir(artifactDir, { recursive: true });
|
|
const root = resolve(process.argv[2] ?? join(process.cwd(), 'build/composer_probe_material'));
|
|
|
|
const mime = {
|
|
'.html': 'text/html',
|
|
'.js': 'text/javascript',
|
|
'.json': 'application/json',
|
|
'.wasm': 'application/wasm',
|
|
'.png': 'image/png',
|
|
'.ico': 'image/x-icon',
|
|
'.ttf': 'font/ttf',
|
|
'.otf': 'font/otf',
|
|
'.woff': 'font/woff',
|
|
'.woff2': 'font/woff2',
|
|
};
|
|
const server = createServer(async (req, res) => {
|
|
try {
|
|
const path = resolve(root, '.' + decodeURIComponent(new URL(req.url, 'http://localhost').pathname));
|
|
if (path !== root && !path.startsWith(root + sep)) return res.writeHead(403).end();
|
|
const filePath = path === root || path.endsWith('/') ? join(path, 'index.html') : path;
|
|
const file = await readFile(filePath);
|
|
res.writeHead(200, { 'content-type': mime[extname(filePath)] ?? 'application/octet-stream' }).end(file);
|
|
} catch {
|
|
res.writeHead(404).end('not found');
|
|
}
|
|
});
|
|
server.listen(0, '127.0.0.1');
|
|
await once(server, 'listening');
|
|
const port = server.address().port;
|
|
const profile = await mkdtemp(join(tmpdir(), 'chat-composer-chrome-'));
|
|
const chrome = spawn('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', [
|
|
'--headless=new', '--no-first-run', '--no-default-browser-check', '--disable-gpu',
|
|
'--remote-debugging-port=0', `--user-data-dir=${profile}`, 'about:blank',
|
|
], { stdio: ['ignore', 'ignore', 'pipe'] });
|
|
let stderr = '';
|
|
chrome.stderr.on('data', data => { stderr += data; });
|
|
|
|
let ws;
|
|
try {
|
|
const waitFor = async (fn, what) => {
|
|
for (let i = 0; i < 120; i++) { const value = await fn(); if (value) return value; await delay(100); }
|
|
throw new Error(`Timed out waiting for ${what}`);
|
|
};
|
|
const endpoint = await waitFor(
|
|
() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1],
|
|
'Chrome DevTools',
|
|
);
|
|
ws = new WebSocket(endpoint);
|
|
await once(ws, 'open');
|
|
let id = 0;
|
|
const pending = new Map();
|
|
ws.addEventListener('message', event => {
|
|
const msg = JSON.parse(event.data);
|
|
if (msg.method?.includes('detached') || msg.method?.includes('crashed')) {
|
|
console.error('CDP event', JSON.stringify(msg));
|
|
}
|
|
if (msg.method === 'Runtime.exceptionThrown') {
|
|
console.error('PAGE EXCEPTION', JSON.stringify(msg.params?.exceptionDetails?.exception?.description ?? msg.params?.exceptionDetails?.text));
|
|
}
|
|
if (msg.id && pending.has(msg.id)) { pending.get(msg.id)(msg); pending.delete(msg.id); }
|
|
});
|
|
const send = (method, params = {}, sessionId) => new Promise(resolve => {
|
|
const callId = ++id;
|
|
pending.set(callId, resolve);
|
|
ws.send(JSON.stringify({ id: callId, method, params, ...(sessionId ? { sessionId } : {}) }));
|
|
});
|
|
const created = await send('Target.createTarget', { url: 'about:blank' });
|
|
if (!created.result) throw new Error(`Target creation failed: ${JSON.stringify(created)}`);
|
|
const session = (await send('Target.attachToTarget', { targetId: created.result.targetId, flatten: true })).result.sessionId;
|
|
await send('Page.enable', {}, session);
|
|
await send('Runtime.enable', {}, session);
|
|
const navigation = await send('Page.navigate', { url: `http://127.0.0.1:${port}/` }, session);
|
|
if (navigation.error || navigation.result?.errorText) throw new Error(JSON.stringify(navigation));
|
|
const evaluate = async expression => {
|
|
const response = await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }, session);
|
|
if (response.error) throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
|
|
return response.result?.result?.value;
|
|
};
|
|
// A cold bundle can need longer than a fixed delay before Flutter publishes
|
|
// its semantics placeholder; wait for it instead of sampling a blank page.
|
|
for (let i = 0; i < 120; i++) {
|
|
if (await evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))")) break;
|
|
await delay(100);
|
|
}
|
|
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
|
|
await delay(1800);
|
|
|
|
const labels = async () =>
|
|
evaluate("[...document.querySelectorAll('flt-semantics')].map(x => x.getAttribute('aria-label') || x.innerText || '').filter(Boolean)");
|
|
const semanticText = async () => (await labels()).join('\n');
|
|
const assertSemantics = async (text, what) => {
|
|
const read = await semanticText();
|
|
if (!read.toLowerCase().replace(/\s+/g, ' ').includes(text.toLowerCase())) {
|
|
throw new Error(`${what}: semantics lacks ${JSON.stringify(text)}\nbody: ${(await evaluate('document.body?.innerText ?? \"\"') || '').slice(0, 800)}\n${read.slice(0, 2000)}`);
|
|
}
|
|
};
|
|
const clickLabel = async (label, { contains = false } = {}) => {
|
|
const point = await evaluate(`(() => {
|
|
const needle = ${JSON.stringify(label)};
|
|
const nodes = [...document.querySelectorAll('flt-semantics')];
|
|
const e = nodes.find(x => {
|
|
const text = x.getAttribute('aria-label') || x.innerText || '';
|
|
return ${contains ? 'text.includes(needle)' : 'text === needle'};
|
|
});
|
|
if (!e) return null;
|
|
const r = e.getBoundingClientRect();
|
|
return { x: r.x + r.width / 2, y: r.y + r.height / 2 };
|
|
})()`);
|
|
if (!point) return false;
|
|
await send('Input.dispatchMouseEvent', { type: 'mousePressed', x: point.x, y: point.y, button: 'left', clickCount: 1 }, session);
|
|
await send('Input.dispatchMouseEvent', { type: 'mouseReleased', x: point.x, y: point.y, button: 'left', clickCount: 1 }, session);
|
|
return true;
|
|
};
|
|
const focusField = async () => {
|
|
const focused = await evaluate(`(() => {
|
|
const needle = 'Message the agent';
|
|
const nodes = [...document.querySelectorAll('textarea, input')];
|
|
const node = nodes.find(n => (n.getAttribute('aria-label') || n.placeholder || '').includes(needle));
|
|
if (!node) return null;
|
|
node.focus();
|
|
const r = node.getBoundingClientRect();
|
|
return { tag: node.tagName, label: node.getAttribute('aria-label'), rect: { x: r.x, y: r.y, w: r.width, h: r.height } };
|
|
})()`);
|
|
if (!focused) {
|
|
const read = await evaluate("[...document.querySelectorAll('textarea, input')].map(x => ({tag:x.tagName,label:x.getAttribute('aria-label'),placeholder:x.placeholder,value:x.value}))");
|
|
throw new Error(`Composer field was not exposed to semantics: ${JSON.stringify(read)}`);
|
|
}
|
|
await delay(500);
|
|
const editing = await editingValue();
|
|
if (!editing) throw new Error('The composer did not expose an editing element');
|
|
return { ...focused, ...editing };
|
|
};
|
|
const editingValue = async () => evaluate(`(() => {
|
|
const match = n => (n.getAttribute('aria-label') || n.getAttribute('placeholder') || '').includes('Message the agent');
|
|
const node = document.querySelector('input.flt-text-editing, textarea.flt-text-editing')
|
|
|| [...document.querySelectorAll('textarea, input')].find(match)
|
|
|| null;
|
|
return node ? { tag: node.tagName, value: node.value } : null;
|
|
})()`);
|
|
const type = async text => { await send('Input.insertText', { text }, session); await delay(250); };
|
|
const key = async (type, params) => send('Input.dispatchKeyEvent', { type, ...params }, session);
|
|
const pressEnter = async modifiers => {
|
|
if (modifiers === 0) {
|
|
await key('keyDown', {
|
|
key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
text: '\r', unmodifiedText: '\r',
|
|
});
|
|
} else {
|
|
const modifier = modifiers === 4
|
|
? { key: 'Meta', code: 'MetaLeft', windowsVirtualKeyCode: 91, nativeVirtualKeyCode: 55 }
|
|
: { key: 'Control', code: 'ControlLeft', windowsVirtualKeyCode: 17, nativeVirtualKeyCode: 59 };
|
|
await key('rawKeyDown', { ...modifier, modifiers });
|
|
await key('rawKeyDown', {
|
|
modifiers, key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
});
|
|
await key('keyUp', {
|
|
modifiers, key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
});
|
|
await key('keyUp', { ...modifier, modifiers: 0 });
|
|
await delay(500);
|
|
return;
|
|
}
|
|
await key('keyUp', {
|
|
modifiers, key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13,
|
|
});
|
|
await delay(500);
|
|
};
|
|
const setSize = async (width, height, scale = 1) => {
|
|
await send('Emulation.setDeviceMetricsOverride', {
|
|
width, height, deviceScaleFactor: scale, mobile: width < 600,
|
|
}, session);
|
|
await delay(400);
|
|
};
|
|
const captures = [];
|
|
const capture = async name => {
|
|
const data = (await send('Page.captureScreenshot', { format: 'png', captureBeyondViewport: false }, session)).result.data;
|
|
const file = `${variant}-${name}.png`;
|
|
await writeFile(`${artifactDir}/${file}`, Buffer.from(data, 'base64'));
|
|
captures.push(file);
|
|
return file;
|
|
};
|
|
const ocr = async file =>
|
|
(await promisify(execFile)('tesseract', [`${artifactDir}/${file}`, 'stdout'])).stdout;
|
|
const assertOcr = async (file, text) => {
|
|
const read = await ocr(file);
|
|
if (!read.toLowerCase().replace(/\s+/g, ' ').includes(text.toLowerCase())) {
|
|
throw new Error(`OCR of ${file} lacks ${JSON.stringify(text)}:\n${read.slice(0, 2000)}`);
|
|
}
|
|
};
|
|
|
|
const checks = [];
|
|
await setSize(390, 844);
|
|
await assertSemantics('Composer fixture', 'The fixture thread');
|
|
const initial = await editingValue();
|
|
if (initial?.value !== '') throw new Error(`The composer started with ${JSON.stringify(initial)}`);
|
|
await capture('composer-initial');
|
|
const focused = await focusField();
|
|
checks.push({ check: 'field exposed as an editing element', focused });
|
|
|
|
await type('First line');
|
|
await pressEnter(0);
|
|
const afterEnter = await editingValue();
|
|
if (!afterEnter?.value?.includes('\n')) {
|
|
throw new Error(`Enter did not insert a newline: ${JSON.stringify(afterEnter)}`);
|
|
}
|
|
await type('Second line');
|
|
const multiline = await editingValue();
|
|
if (multiline?.value !== 'First line\nSecond line') {
|
|
throw new Error(`Multiline draft is ${JSON.stringify(multiline)}`);
|
|
}
|
|
const multilineCapture = await capture('composer-multiline');
|
|
await assertOcr(multilineCapture, 'Second');
|
|
checks.push({ check: 'Enter inserted a newline without sending', value: multiline.value });
|
|
|
|
const usersBefore = (await labels()).filter(label => label === 'User message').length;
|
|
await pressEnter(variant === 'material' ? 4 : 2);
|
|
await waitFor(async () => (await semanticText()).includes('Stop'), 'Stop after send');
|
|
const usersAfter = (await labels()).filter(label => label === 'User message').length;
|
|
if (usersAfter <= usersBefore) {
|
|
throw new Error(`No optimistic user row appeared (${usersBefore} -> ${usersAfter})`);
|
|
}
|
|
const cleared = await editingValue();
|
|
if (cleared?.value !== '') throw new Error(`Send left the field as ${JSON.stringify(cleared)}`);
|
|
const runningCapture = await capture('composer-running');
|
|
await assertOcr(runningCapture, 'First');
|
|
checks.push({ check: 'Ctrl/Cmd+Enter sent and the optimistic turn is visible', cleared });
|
|
|
|
// The reply that carries `chat_queued` lands after the running frame.
|
|
await delay(700);
|
|
await focusField();
|
|
await type('Queued follow-up');
|
|
const queuedDraft = await editingValue();
|
|
if (process.env.PROBE_DEBUG) console.log('queued draft', JSON.stringify(queuedDraft));
|
|
await pressEnter(variant === 'material' ? 4 : 2);
|
|
await delay(900);
|
|
const queuedCapture = await capture('composer-queued');
|
|
const queuedRead = await ocr(queuedCapture);
|
|
if (process.env.PROBE_DEBUG) console.log('queued OCR', JSON.stringify(queuedRead.slice(0, 500)));
|
|
if (!queuedRead.toLowerCase().replace(/\s+/g, ' ').includes('current turn finishes')) {
|
|
throw new Error(`The queued notice did not render:\n${queuedRead.slice(0, 2000)}`);
|
|
}
|
|
checks.push({ check: 'the queued reply shows its polite notice while Stop stays' });
|
|
|
|
if (!await clickLabel('Stop')) throw new Error('Could not press Stop');
|
|
await waitFor(async () => !(await semanticText()).includes('Stop'), 'the stopped turn');
|
|
await capture('composer-stopped');
|
|
checks.push({ check: 'Stop interrupted the turn and Send returned' });
|
|
|
|
await focusField();
|
|
await type('refuse me');
|
|
await pressEnter(variant === 'material' ? 4 : 2);
|
|
await waitFor(
|
|
async () => (await semanticText()).includes('The fixture refused this prompt.'),
|
|
'the refusal line',
|
|
);
|
|
const restored = await editingValue();
|
|
if (restored?.value !== 'refuse me') {
|
|
throw new Error(`Refused draft was not restored: ${JSON.stringify(restored)}`);
|
|
}
|
|
await capture('composer-refused');
|
|
await assertOcr(`${variant}-composer-refused.png`, 'refuse me');
|
|
checks.push({ check: 'the refusal restored the exact draft and reported it', restored });
|
|
|
|
await setSize(320, 640, 2);
|
|
await delay(500);
|
|
const wrapped = await semanticText();
|
|
if (!wrapped.includes('Composer fixture') || !wrapped.includes('Send')) {
|
|
throw new Error(`The 320px/2x layout lost its content:\n${wrapped.slice(0, 2000)}`);
|
|
}
|
|
await capture('composer-320-2x');
|
|
checks.push({ check: 'the composer still renders at 320 logical px with 2x pixels' });
|
|
|
|
console.log(JSON.stringify({ variant, checks, captures }, null, 2));
|
|
} finally {
|
|
ws?.close();
|
|
chrome.kill('SIGTERM');
|
|
await Promise.race([once(chrome, 'exit'), delay(3000)]);
|
|
server.close();
|
|
await rm(profile, { recursive: true, force: true });
|
|
}
|