Files
pipelines/app/tool/ui_followups_probe.mjs
alan b60ad6d98f Remove alternate hosts from tooling probes
Keep the probe suite on the Material composition so browser fixtures and capture drivers no longer render or select Cupertino hosts. Record the final residual legacy arguments and implementation checks for the delivery gate.
2026-11-11 21:38:30 +00:00

95 lines
5.6 KiB
JavaScript

// Production app UI, isolated create-prompt and recorded-stage fixtures only.
import assert from 'node:assert/strict';
import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
import { once } from 'node:events';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { startFixture } from './runs_probe_server.mjs';
import { launchChrome, waitForPort, waitForPage, connect, sleep } from './pipelines_probe_cdp_lib.mjs';
import { axNodes, enableSemantics, waitFor, capture, click, clickAt } from './pipelines_probe_cdp_input.mjs';
const output = resolve(process.argv[3] ?? 'build/ui-followups-material');
await mkdir(output,{recursive:true});
const stage = 'verify_release';
const runDetail = { id:1,title:'Recorded release',status:'failed',graph:{stages:[{key:stage,type:'model'}]},steps:[
{stage,sequence:1,status:'completed',outcome:'success',provider:'openai',model:'fixture-model',
started_at:'2026-10-25T12:00:00Z',finished_at:'2026-10-25T12:02:05Z',output:{result:'First visit'}},
{stage,sequence:2,status:'failed',outcome:'error',diagnostic:'Recorded verification failure',
started_at:'2026-10-25T12:03:00Z',finished_at:'2026-10-25T12:03:10Z',output:{result:'Second visit'}},
]};
const fixture=await startFixture({root:resolve(process.argv[2]),runDetail,chatMessages:[]});
const profile=await mkdtemp(join(tmpdir(),'ui-followups-'));
const chrome=launchChrome('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',profile,'about:blank');
const log={checks:[],captures:[]};
let c;
const names=async()=>(await axNodes(c)).flatMap(n=>[n.name?.value??'',n.value?.value??'']);
async function point(node) {
assert.ok(node,'expected accessible control');
const {model}=await c.send('DOM.getBoxModel',{backendNodeId:node.backendDOMNodeId});
const [x1,y1,,,x3,y3]=model.content;await clickAt(c,(x1+x3)/2,(y1+y3)/2);
}
async function navigate(path) {
await c.send('Page.navigate',{url:fixture.url+path});
assert.ok(await waitFor(()=>c.evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))"),30000));
await enableSemantics(c);
}
async function createButton() {
return waitFor(async () => {
const button=(await axNodes(c)).filter(n=>n.role?.value==='button'&&n.name?.value==='New chat'&&n.backendDOMNodeId).at(-1);
return button && !button.properties?.some(p=>p.name==='disabled'&&p.value?.value===true) ? button : null;
},10000);
}
async function promptSwitch() {
return (await axNodes(c)).find(n=>['switch','checkbox'].includes(n.role?.value)&&n.backendDOMNodeId);
}
try {
c=await connect((await waitForPage(await waitForPort(join(profile,'DevToolsActivePort')))).webSocketDebuggerUrl);
await c.send('Page.enable');
await c.send('Emulation.setUserAgentOverride',{
userAgent:'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36', platform:'Win32',
});
await c.send('Emulation.setDeviceMetricsOverride',{width:1280,height:1000,deviceScaleFactor:1,mobile:false});
await navigate('/chat');
await click(c,'New chat');
let toggle=await waitFor(promptSwitch,15000);assert.ok(toggle);
assert.equal(String(toggle.properties.find(p=>p.name==='checked')?.value?.value),'false','prompt is off by default');
await capture(c,output,log,'01-prompt-off');
await point(await createButton());
assert.ok(await waitFor(()=>c.evaluate("location.pathname.includes('fixture_created_1')"),10000));
let state=await(await fetch(fixture.url+'/fixture/state')).json();
assert.equal(state.chatCreateRequests.length,1);
assert.equal(Object.hasOwn(state.chatCreateRequests[0],'instructions'),false);
log.checks.push('default-off chat creation omits instructions');
await navigate('/chat');await click(c,'New chat');
await point(await waitFor(promptSwitch,10000));
await sleep(300);
await click(c,'Custom');await click(c,'Careful coding');await sleep(300);
await capture(c,output,log,'02-preset-editable');
await click(c,'Careful coding');await click(c,'Custom');await sleep(200);
const editor=(await axNodes(c)).filter(n=>n.role?.value==='textbox'&&n.backendDOMNodeId).at(-1);
await point(editor);await c.send('Input.insertText',{text:'Custom prompt from the isolated fixture.'});
await sleep(200);await point(await createButton());
assert.ok(await waitFor(()=>c.evaluate("location.pathname.includes('fixture_created_2')"),10000));
state=await(await fetch(fixture.url+'/fixture/state')).json();
assert.equal(state.chatCreateRequests[1].instructions,'Custom prompt from the isolated fixture.');
log.checks.push('real preset/custom picker and exact custom create payload');
for(const [index,sequence] of [[0,1],[1,2]]) {
await navigate(`/runs/1/stage/${stage}?visit=${index}`);
assert.ok(await waitFor(()=>c.evaluate("Boolean(document.querySelector('[flt-semantics-identifier=run-stage-heading]'))"),15000));
assert.ok((await names()).includes(stage));
assert.ok((await names()).includes('Visit '+sequence));
if(index===1) assert.ok((await names()).some(name=>name.includes('Recorded verification failure')));
await capture(c,output,log,`03-stage-visit-${sequence}`);
}
log.checks.push('prominent stage heading and correct separate coordinates for repeated deep-linked visits');
console.log(JSON.stringify(log,null,2));
} catch(error) {
if(c){await capture(c,output,log,'failure');await writeFile(join(output,'failure-ax.json'),JSON.stringify(await axNodes(c),null,2));}
throw error;
} finally {
await writeFile(join(output,'result.json'),JSON.stringify(log,null,2));
c?.close();const exited=once(chrome,'exit');chrome.kill();await exited;await fixture.close();
}