Files
pipelines/app/tool/chat_stable_activity_probe.mjs
T

106 lines
6.6 KiB
JavaScript

// Production chat turn, driven only by isolated loopback tool/reasoning frames.
import assert from 'node:assert/strict';
import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
import { once } from 'node:events';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { startFixture } from './runs_probe_server.mjs';
import { launchChrome, waitForPort, waitForPage, connect, sleep } from './pipelines_probe_cdp_lib.mjs';
import { axNodes, enableSemantics, waitFor, capture, click, clickAt } from './pipelines_probe_cdp_input.mjs';
const platform = process.env.PROBE_PLATFORM ?? 'material';
const width = Number(process.env.PROBE_WIDTH ?? 1280);
const output = resolve(process.argv[3] ?? `build/stable-activity-${platform}`);
await mkdir(output, { recursive: true });
const fixture = await startFixture({ root: resolve(process.argv[2]), chatMessages: [] });
const profile = await mkdtemp(join(tmpdir(), 'stable-activity-'));
const chrome = launchChrome('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', profile, 'about:blank');
const log = { platform, width, checks: [], captures: [] };
let c;
const nodes = () => axNodes(c);
const names = async () => (await nodes()).flatMap(n => [n.name?.value ?? '', n.value?.value ?? '']);
const emit = async event => {
const response = await fetch(fixture.url + '/fixture/chat-event', { method: 'POST', body: JSON.stringify(event) });
assert.ok(response.ok);
};
async function status(title) {
return waitFor(async () => (await nodes()).find(n => n.role?.value === 'button' &&
n.name?.value?.includes(title) && n.backendDOMNodeId), 10000);
}
async function box(node) {
assert.ok(node, 'activity disclosure is accessible');
const { model } = await c.send('DOM.getBoxModel', { backendNodeId: node.backendDOMNodeId });
const [x,y,,,right,bottom] = model.content;
return { x, y, width:right-x, height:bottom-y };
}
async function tap(node) {
const b = await box(node); await clickAt(c, b.x+b.width/2, b.y+b.height/2);
}
const call = (id, description, state = 'running') => ({ kind:'tool', message_id:'fixture-answer', part:{
type:'tool', id, name:'bash', time:{created:Date.now()-9000, ran:Date.now()-9000,
...(state === 'completed' ? {completed:Date.now()} : {})},
state:{status:state, input:{description, command:'echo isolated-fixture'},
...(state === 'completed' ? {content:[{type:'text',text:'Fixture command output '+id}]} : {})},
} });
try {
c = await connect((await waitForPage(await waitForPort(join(profile,'DevToolsActivePort')))).webSocketDebuggerUrl);
await c.send('Page.enable');
if (platform === 'material') await c.send('Emulation.setUserAgentOverride', {
userAgent:'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36',platform:'Win32',
});
else await c.send('Emulation.setUserAgentOverride', {
userAgent:'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36',platform:'MacIntel',
});
await c.send('Emulation.setDeviceMetricsOverride',{width,height:900,deviceScaleFactor:1,mobile:false});
await c.send('Page.navigate',{url:fixture.url+'/chat/ses_probechat'});
assert.ok(await waitFor(()=>c.evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))"),30000));
await enableSemantics(c);
const composer = await waitFor(async()=>(await nodes()).find(n=>n.role?.value==='textbox'&&n.backendDOMNodeId),15000);
await tap(composer);
await c.send('Input.insertText',{text:'Run the isolated activity fixture.'});
await click(c,'Send');
await emit({kind:'step',step:{status:'running',turn_id:'stable-turn'}});
await emit({kind:'reasoning',message_id:'fixture-answer',part:{type:'reasoning',id:'r'},delta:'Private fixture reasoning retained in details.'});
await emit(call('one','Checking fixture configuration'));
const first = await status('Checking fixture configuration');
const before = await box(first);
assert.ok(!(await names()).some(n=>n==='Private fixture reasoning retained in details.'),'reasoning hidden until disclosure');
await capture(c,output,log,'01-one-line-running');
await sleep(1200);
const timed = await status('Checking fixture configuration');
assert.ok(/\d+s/.test(timed.name.value),'elapsed seconds visible');
assert.equal((await box(timed)).height,before.height,'timer does not resize the row');
await emit(call('one','Checking fixture configuration','completed'));
await emit(call('two','Running fixture verification'));
const second = await status('Running fixture verification');
const after = await box(second);
assert.equal(after.height,before.height,'new actions retain one-row height');
assert.equal(after.y,before.y,'tool transitions do not move the activity row');
log.checks.push('one stable line with recorded current action and localized elapsed clock; reasoning hidden');
await tap(second);
assert.ok(await waitFor(async()=>(await names()).some(n=>n.includes('Checking fixture configuration')),10000));
await capture(c,output,log,'02-live-details');
await emit(call('two','Running fixture verification','completed'));
await emit(call('three','Checking fixture dependencies'));
assert.ok(await waitFor(async()=>(await names()).some(n=>n.includes('Checking fixture dependencies')),10000),'open details update live');
await c.send('Input.dispatchKeyEvent',{type:'keyDown',key:'Escape',code:'Escape',windowsVirtualKeyCode:27});
await c.send('Input.dispatchKeyEvent',{type:'keyUp',key:'Escape',code:'Escape',windowsVirtualKeyCode:27});
await sleep(300);
log.checks.push('disclosure opens recorded details and follows new tools while open');
await emit(call('three','Checking fixture dependencies','completed'));
await emit(call('four','Final fixture command','completed'));
await emit({kind:'text',message_id:'fixture-answer',part:{type:'text',id:'final'},delta:'The verified fixture answer stays readable.'});
await emit({kind:'step',step:{status:'ended',turn_id:'stable-turn'}});
assert.ok(await status('Ran 4 commands'),'finished compact command count');
assert.ok((await names()).includes('The verified fixture answer stays readable.'));
await capture(c,output,log,'03-finished-summary');
log.checks.push('finished turn stops loading and retains Ran 4 commands plus its answer');
console.log(JSON.stringify(log,null,2));
} catch(error) {
if(c){await capture(c,output,log,'failure');await writeFile(join(output,'failure-ax.json'),JSON.stringify(await nodes(),null,2));}
throw error;
} finally {
await writeFile(join(output,'result.json'),JSON.stringify(log,null,2));
c?.close();const exited=once(chrome,'exit');chrome.kill();await exited;await fixture.close();
}