Files
pipelines/app/tool/chat_response_pending_probe.mjs
T

109 lines
6.8 KiB
JavaScript

// Real-browser Send -> pending bubble -> tools -> answer, against loopback data.
import assert from 'node:assert/strict';
import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
import { once } from 'node:events';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { startFixture } from './runs_probe_server.mjs';
import { launchChrome, waitForPort, waitForPage, connect, sleep } from './pipelines_probe_cdp_lib.mjs';
import { axNodes, enableSemantics, waitFor, capture, click, clickAt } from './pipelines_probe_cdp_input.mjs';
const platform = process.env.PROBE_PLATFORM ?? 'material';
const enterSends = process.env.PROBE_ENTER === '1';
const output = resolve(process.argv[3] ?? `build/chat-response-pending-${platform}`);
await mkdir(output, { recursive: true });
const fixture = await startFixture({ root: resolve(process.argv[2]), chatMessages: [] });
const profile = await mkdtemp(join(tmpdir(), 'chat-response-pending-'));
const chrome = launchChrome('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', profile, 'about:blank');
const log = { platform, enterSends, checks: [], captures: [] };
let c;
const names = async () => (await axNodes(c)).map(n => n.name?.value ?? '');
const pending = async () => (await axNodes(c)).find(n => n.name?.value === 'Assistant is responding' && n.backendDOMNodeId);
const emit = async body => {
const response = await fetch(fixture.url + '/fixture/chat-event', { method: 'POST', body: JSON.stringify(body) });
assert.ok(response.ok);
};
try {
c = await connect((await waitForPage(await waitForPort(join(profile, 'DevToolsActivePort')))).webSocketDebuggerUrl);
await c.send('Page.enable');
if (platform === 'material') await c.send('Emulation.setUserAgentOverride', {
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36', platform: 'Win32',
});
await c.send('Emulation.setDeviceMetricsOverride', { width: 1280, height: 800, deviceScaleFactor: 1, mobile: false });
await c.send('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-reduced-motion', value: 'no-preference' }] });
await c.send('Page.navigate', { url: fixture.url + '/chat/ses_probechat' });
assert.ok(await waitFor(() => c.evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))"), 30000));
await enableSemantics(c);
const field = await waitFor(async () => (await axNodes(c)).find(n => n.role?.value === 'textbox' && n.backendDOMNodeId), 30000);
assert.ok(field, 'composer field');
const { model } = await c.send('DOM.getBoxModel', { backendNodeId: field.backendDOMNodeId });
const [x1, y1, , , x3, y3] = model.content;
await clickAt(c, (x1 + x3) / 2, (y1 + y3) / 2);
await c.send('Input.insertText', { text: 'Fixture prompt before any response tokens.' });
await sleep(150);
if (enterSends) {
await c.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'Shift', code: 'ShiftLeft', windowsVirtualKeyCode: 16, modifiers: 8 });
await c.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, modifiers: 8, text: '\r' });
await c.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, modifiers: 8 });
await c.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'Shift', code: 'ShiftLeft', windowsVirtualKeyCode: 16 });
await sleep(150);
const beforeSend = await (await fetch(fixture.url + '/fixture/state')).json();
assert.equal(beforeSend.chatPrompts.length, 0, 'Shift+Enter does not submit');
assert.equal(await pending(), undefined);
await c.send('Input.insertText', { text: 'Second line.' });
await c.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, text: '\r' });
await c.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13 });
log.checks.push('real Shift+Enter inserts a newline; plain Enter submits');
} else {
await click(c, 'Send');
}
assert.ok(await waitFor(pending, 10000), 'assistant bubble before a server response frame');
await sleep(500); // The accepted prompt still has no assistant frames.
if (enterSends) {
const state = await (await fetch(fixture.url + '/fixture/state')).json();
assert.equal(state.chatPrompts.length, 1, 'Enter submits once');
assert.equal(state.chatPrompts[0].text, 'Fixture prompt before any response tokens.\nSecond line.');
log.checks.push('exact multiline prompt submitted once without Control or Command');
}
await capture(c, output, log, '01-before-first-frame');
log.checks.push('normal assistant response bubble before any frames or tokens');
const node = await pending();
const { model: dots } = await c.send('DOM.getBoxModel', { backendNodeId: node.backendDOMNodeId });
const [dx1, dy1, , , dx3, dy3] = dots.content;
const clip = { x: dx1, y: dy1, width: dx3 - dx1, height: dy3 - dy1, scale: 1 };
const first = await c.send('Page.captureScreenshot', { format: 'png', clip });
await sleep(250);
const second = await c.send('Page.captureScreenshot', { format: 'png', clip });
assert.notEqual(first.data, second.data, 'ellipsis pixels animate without response events');
log.checks.push('ellipsis visibly animates independently of the transcript');
await emit({ kind: 'step', step: { status: 'running', turn_id: 'fixture-turn' } });
await emit({ kind: 'tool', message_id: 'answer', part: {
type: 'tool', id: 'tool', name: 'read', state: { status: 'completed' },
} });
assert.ok(await waitFor(async () => (await names()).some(name => name.startsWith('1 tool call')), 10000));
assert.ok(await pending(), 'response bubble remains above tool-only activity');
await capture(c, output, log, '02-tool-only-response');
log.checks.push('tool-only response retains its pending answer bubble');
await emit({ kind: 'text', message_id: 'answer', part: { type: 'text', id: 'text' }, delta: 'The first answer tokens replace the dots.' });
await emit({ kind: 'step', step: { status: 'ended', turn_id: 'fixture-turn' } });
assert.ok(await waitFor(async () => (await names()).includes('The first answer tokens replace the dots.'), 10000));
assert.equal(await pending(), undefined);
await capture(c, output, log, '03-answer-text');
log.checks.push('answer text replaces the dots; finished turn has no pending animation');
console.log(JSON.stringify(log, null, 2));
} catch (error) {
if (c) {
await capture(c, output, log, 'failure');
await writeFile(join(output, 'failure-ax.json'), JSON.stringify(await axNodes(c), null, 2));
}
throw error;
} finally {
await writeFile(join(output, 'result.json'), JSON.stringify(log, null, 2));
c?.close();
const exited = once(chrome, 'exit'); chrome.kill(); await exited;
await fixture.close();
}