b60ad6d98f
Keep the probe suite on the Material composition so browser fixtures and capture drivers no longer render or select Cupertino hosts. Record the final residual legacy arguments and implementation checks for the delivery gate.
109 lines
6.7 KiB
JavaScript
109 lines
6.7 KiB
JavaScript
// Real-browser Send -> pending bubble -> tools -> answer, against loopback data.
|
|
import assert from 'node:assert/strict';
|
|
import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
|
|
import { once } from 'node:events';
|
|
import { tmpdir } from 'node:os';
|
|
import { join, resolve } from 'node:path';
|
|
import { startFixture } from './runs_probe_server.mjs';
|
|
import { launchChrome, waitForPort, waitForPage, connect, sleep } from './pipelines_probe_cdp_lib.mjs';
|
|
import { axNodes, enableSemantics, waitFor, capture, click, clickAt } from './pipelines_probe_cdp_input.mjs';
|
|
|
|
const platform = 'material';
|
|
const enterSends = process.env.PROBE_ENTER === '1';
|
|
const output = resolve(process.argv[3] ?? `build/chat-response-pending-${platform}`);
|
|
await mkdir(output, { recursive: true });
|
|
const fixture = await startFixture({ root: resolve(process.argv[2]), chatMessages: [] });
|
|
const profile = await mkdtemp(join(tmpdir(), 'chat-response-pending-'));
|
|
const chrome = launchChrome('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', profile, 'about:blank');
|
|
const log = { platform, enterSends, checks: [], captures: [] };
|
|
let c;
|
|
const names = async () => (await axNodes(c)).map(n => n.name?.value ?? '');
|
|
const pending = async () => (await axNodes(c)).find(n => n.name?.value === 'Assistant is responding' && n.backendDOMNodeId);
|
|
const emit = async body => {
|
|
const response = await fetch(fixture.url + '/fixture/chat-event', { method: 'POST', body: JSON.stringify(body) });
|
|
assert.ok(response.ok);
|
|
};
|
|
try {
|
|
c = await connect((await waitForPage(await waitForPort(join(profile, 'DevToolsActivePort')))).webSocketDebuggerUrl);
|
|
await c.send('Page.enable');
|
|
if (platform === 'material') await c.send('Emulation.setUserAgentOverride', {
|
|
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36', platform: 'Win32',
|
|
});
|
|
await c.send('Emulation.setDeviceMetricsOverride', { width: 1280, height: 800, deviceScaleFactor: 1, mobile: false });
|
|
await c.send('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-reduced-motion', value: 'no-preference' }] });
|
|
await c.send('Page.navigate', { url: fixture.url + '/chat/ses_probechat' });
|
|
assert.ok(await waitFor(() => c.evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))"), 30000));
|
|
await enableSemantics(c);
|
|
const field = await waitFor(async () => (await axNodes(c)).find(n => n.role?.value === 'textbox' && n.backendDOMNodeId), 30000);
|
|
assert.ok(field, 'composer field');
|
|
const { model } = await c.send('DOM.getBoxModel', { backendNodeId: field.backendDOMNodeId });
|
|
const [x1, y1, , , x3, y3] = model.content;
|
|
await clickAt(c, (x1 + x3) / 2, (y1 + y3) / 2);
|
|
await c.send('Input.insertText', { text: 'Fixture prompt before any response tokens.' });
|
|
await sleep(150);
|
|
if (enterSends) {
|
|
await c.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'Shift', code: 'ShiftLeft', windowsVirtualKeyCode: 16, modifiers: 8 });
|
|
await c.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, modifiers: 8, text: '\r' });
|
|
await c.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, modifiers: 8 });
|
|
await c.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'Shift', code: 'ShiftLeft', windowsVirtualKeyCode: 16 });
|
|
await sleep(150);
|
|
const beforeSend = await (await fetch(fixture.url + '/fixture/state')).json();
|
|
assert.equal(beforeSend.chatPrompts.length, 0, 'Shift+Enter does not submit');
|
|
assert.equal(await pending(), undefined);
|
|
await c.send('Input.insertText', { text: 'Second line.' });
|
|
await c.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, text: '\r' });
|
|
await c.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13 });
|
|
log.checks.push('real Shift+Enter inserts a newline; plain Enter submits');
|
|
} else {
|
|
await click(c, 'Send');
|
|
}
|
|
assert.ok(await waitFor(pending, 10000), 'assistant bubble before a server response frame');
|
|
await sleep(500); // The accepted prompt still has no assistant frames.
|
|
if (enterSends) {
|
|
const state = await (await fetch(fixture.url + '/fixture/state')).json();
|
|
assert.equal(state.chatPrompts.length, 1, 'Enter submits once');
|
|
assert.equal(state.chatPrompts[0].text, 'Fixture prompt before any response tokens.\nSecond line.');
|
|
log.checks.push('exact multiline prompt submitted once without Control or Command');
|
|
}
|
|
await capture(c, output, log, '01-before-first-frame');
|
|
log.checks.push('normal assistant response bubble before any frames or tokens');
|
|
|
|
const node = await pending();
|
|
const { model: dots } = await c.send('DOM.getBoxModel', { backendNodeId: node.backendDOMNodeId });
|
|
const [dx1, dy1, , , dx3, dy3] = dots.content;
|
|
const clip = { x: dx1, y: dy1, width: dx3 - dx1, height: dy3 - dy1, scale: 1 };
|
|
const first = await c.send('Page.captureScreenshot', { format: 'png', clip });
|
|
await sleep(250);
|
|
const second = await c.send('Page.captureScreenshot', { format: 'png', clip });
|
|
assert.notEqual(first.data, second.data, 'ellipsis pixels animate without response events');
|
|
log.checks.push('ellipsis visibly animates independently of the transcript');
|
|
|
|
await emit({ kind: 'step', step: { status: 'running', turn_id: 'fixture-turn' } });
|
|
await emit({ kind: 'tool', message_id: 'answer', part: {
|
|
type: 'tool', id: 'tool', name: 'read', state: { status: 'completed' },
|
|
} });
|
|
assert.ok(await waitFor(async () => (await names()).some(name => name.startsWith('1 tool call')), 10000));
|
|
assert.ok(await pending(), 'response bubble remains above tool-only activity');
|
|
await capture(c, output, log, '02-tool-only-response');
|
|
log.checks.push('tool-only response retains its pending answer bubble');
|
|
|
|
await emit({ kind: 'text', message_id: 'answer', part: { type: 'text', id: 'text' }, delta: 'The first answer tokens replace the dots.' });
|
|
await emit({ kind: 'step', step: { status: 'ended', turn_id: 'fixture-turn' } });
|
|
assert.ok(await waitFor(async () => (await names()).includes('The first answer tokens replace the dots.'), 10000));
|
|
assert.equal(await pending(), undefined);
|
|
await capture(c, output, log, '03-answer-text');
|
|
log.checks.push('answer text replaces the dots; finished turn has no pending animation');
|
|
console.log(JSON.stringify(log, null, 2));
|
|
} catch (error) {
|
|
if (c) {
|
|
await capture(c, output, log, 'failure');
|
|
await writeFile(join(output, 'failure-ax.json'), JSON.stringify(await axNodes(c), null, 2));
|
|
}
|
|
throw error;
|
|
} finally {
|
|
await writeFile(join(output, 'result.json'), JSON.stringify(log, null, 2));
|
|
c?.close();
|
|
const exited = once(chrome, 'exit'); chrome.kill(); await exited;
|
|
await fixture.close();
|
|
}
|