109 lines
6.8 KiB
JavaScript
109 lines
6.8 KiB
JavaScript
// Real-browser Send -> pending bubble -> tools -> answer, against loopback data.
|
|
import assert from 'node:assert/strict';
|
|
import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
|
|
import { once } from 'node:events';
|
|
import { tmpdir } from 'node:os';
|
|
import { join, resolve } from 'node:path';
|
|
import { startFixture } from './runs_probe_server.mjs';
|
|
import { launchChrome, waitForPort, waitForPage, connect, sleep } from './pipelines_probe_cdp_lib.mjs';
|
|
import { axNodes, enableSemantics, waitFor, capture, click, clickAt } from './pipelines_probe_cdp_input.mjs';
|
|
|
|
const platform = process.env.PROBE_PLATFORM ?? 'material';
|
|
const enterSends = process.env.PROBE_ENTER === '1';
|
|
const output = resolve(process.argv[3] ?? `build/chat-response-pending-${platform}`);
|
|
await mkdir(output, { recursive: true });
|
|
const fixture = await startFixture({ root: resolve(process.argv[2]), chatMessages: [] });
|
|
const profile = await mkdtemp(join(tmpdir(), 'chat-response-pending-'));
|
|
const chrome = launchChrome('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', profile, 'about:blank');
|
|
const log = { platform, enterSends, checks: [], captures: [] };
|
|
let c;
|
|
const names = async () => (await axNodes(c)).map(n => n.name?.value ?? '');
|
|
const pending = async () => (await axNodes(c)).find(n => n.name?.value === 'Assistant is responding' && n.backendDOMNodeId);
|
|
const emit = async body => {
|
|
const response = await fetch(fixture.url + '/fixture/chat-event', { method: 'POST', body: JSON.stringify(body) });
|
|
assert.ok(response.ok);
|
|
};
|
|
try {
|
|
c = await connect((await waitForPage(await waitForPort(join(profile, 'DevToolsActivePort')))).webSocketDebuggerUrl);
|
|
await c.send('Page.enable');
|
|
if (platform === 'material') await c.send('Emulation.setUserAgentOverride', {
|
|
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36', platform: 'Win32',
|
|
});
|
|
await c.send('Emulation.setDeviceMetricsOverride', { width: 1280, height: 800, deviceScaleFactor: 1, mobile: false });
|
|
await c.send('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-reduced-motion', value: 'no-preference' }] });
|
|
await c.send('Page.navigate', { url: fixture.url + '/chat/ses_probechat' });
|
|
assert.ok(await waitFor(() => c.evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))"), 30000));
|
|
await enableSemantics(c);
|
|
const field = await waitFor(async () => (await axNodes(c)).find(n => n.role?.value === 'textbox' && n.backendDOMNodeId), 30000);
|
|
assert.ok(field, 'composer field');
|
|
const { model } = await c.send('DOM.getBoxModel', { backendNodeId: field.backendDOMNodeId });
|
|
const [x1, y1, , , x3, y3] = model.content;
|
|
await clickAt(c, (x1 + x3) / 2, (y1 + y3) / 2);
|
|
await c.send('Input.insertText', { text: 'Fixture prompt before any response tokens.' });
|
|
await sleep(150);
|
|
if (enterSends) {
|
|
await c.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'Shift', code: 'ShiftLeft', windowsVirtualKeyCode: 16, modifiers: 8 });
|
|
await c.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, modifiers: 8, text: '\r' });
|
|
await c.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, modifiers: 8 });
|
|
await c.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'Shift', code: 'ShiftLeft', windowsVirtualKeyCode: 16 });
|
|
await sleep(150);
|
|
const beforeSend = await (await fetch(fixture.url + '/fixture/state')).json();
|
|
assert.equal(beforeSend.chatPrompts.length, 0, 'Shift+Enter does not submit');
|
|
assert.equal(await pending(), undefined);
|
|
await c.send('Input.insertText', { text: 'Second line.' });
|
|
await c.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, text: '\r' });
|
|
await c.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13 });
|
|
log.checks.push('real Shift+Enter inserts a newline; plain Enter submits');
|
|
} else {
|
|
await click(c, 'Send');
|
|
}
|
|
assert.ok(await waitFor(pending, 10000), 'assistant bubble before a server response frame');
|
|
await sleep(500); // The accepted prompt still has no assistant frames.
|
|
if (enterSends) {
|
|
const state = await (await fetch(fixture.url + '/fixture/state')).json();
|
|
assert.equal(state.chatPrompts.length, 1, 'Enter submits once');
|
|
assert.equal(state.chatPrompts[0].text, 'Fixture prompt before any response tokens.\nSecond line.');
|
|
log.checks.push('exact multiline prompt submitted once without Control or Command');
|
|
}
|
|
await capture(c, output, log, '01-before-first-frame');
|
|
log.checks.push('normal assistant response bubble before any frames or tokens');
|
|
|
|
const node = await pending();
|
|
const { model: dots } = await c.send('DOM.getBoxModel', { backendNodeId: node.backendDOMNodeId });
|
|
const [dx1, dy1, , , dx3, dy3] = dots.content;
|
|
const clip = { x: dx1, y: dy1, width: dx3 - dx1, height: dy3 - dy1, scale: 1 };
|
|
const first = await c.send('Page.captureScreenshot', { format: 'png', clip });
|
|
await sleep(250);
|
|
const second = await c.send('Page.captureScreenshot', { format: 'png', clip });
|
|
assert.notEqual(first.data, second.data, 'ellipsis pixels animate without response events');
|
|
log.checks.push('ellipsis visibly animates independently of the transcript');
|
|
|
|
await emit({ kind: 'step', step: { status: 'running', turn_id: 'fixture-turn' } });
|
|
await emit({ kind: 'tool', message_id: 'answer', part: {
|
|
type: 'tool', id: 'tool', name: 'read', state: { status: 'completed' },
|
|
} });
|
|
assert.ok(await waitFor(async () => (await names()).some(name => name.startsWith('1 tool call')), 10000));
|
|
assert.ok(await pending(), 'response bubble remains above tool-only activity');
|
|
await capture(c, output, log, '02-tool-only-response');
|
|
log.checks.push('tool-only response retains its pending answer bubble');
|
|
|
|
await emit({ kind: 'text', message_id: 'answer', part: { type: 'text', id: 'text' }, delta: 'The first answer tokens replace the dots.' });
|
|
await emit({ kind: 'step', step: { status: 'ended', turn_id: 'fixture-turn' } });
|
|
assert.ok(await waitFor(async () => (await names()).includes('The first answer tokens replace the dots.'), 10000));
|
|
assert.equal(await pending(), undefined);
|
|
await capture(c, output, log, '03-answer-text');
|
|
log.checks.push('answer text replaces the dots; finished turn has no pending animation');
|
|
console.log(JSON.stringify(log, null, 2));
|
|
} catch (error) {
|
|
if (c) {
|
|
await capture(c, output, log, 'failure');
|
|
await writeFile(join(output, 'failure-ax.json'), JSON.stringify(await axNodes(c), null, 2));
|
|
}
|
|
throw error;
|
|
} finally {
|
|
await writeFile(join(output, 'result.json'), JSON.stringify(log, null, 2));
|
|
c?.close();
|
|
const exited = once(chrome, 'exit'); chrome.kill(); await exited;
|
|
await fixture.close();
|
|
}
|