Files
pipelines/web/test/chat-hardening-probe.mjs
T

168 lines
13 KiB
JavaScript

import assert from "node:assert/strict";
import { spawn } from "node:child_process";
import { mkdir } from "node:fs/promises";
import { join } from "node:path";
import net from "node:net";
import { fileURLToPath } from "node:url";
import { eventually, launchChrome } from "./browser.mjs";
const [target, sessionId] = process.argv.slice(2);
assert.match(target ?? "", /^127\.0\.0\.1:\d+$/);
const reserve = net.createServer();
await new Promise((resolve) => reserve.listen(0, "127.0.0.1", resolve));
const port = reserve.address().port;
await new Promise((resolve) => reserve.close(resolve));
const serve = spawn(process.execPath, ["serve.mjs", "--port", String(port), "--pipelines", target], {
cwd: fileURLToPath(new URL("..", import.meta.url)),
env: { ...process.env, PIPELINES_WEB_AUTH: "disabled" }, stdio: ["ignore", "pipe", "pipe"],
});
let logs = "";
serve.stdout.on("data", (data) => { logs += data; });
serve.stderr.on("data", (data) => { logs += data; });
let browser;
try {
await eventually(async () => {
if (serve.exitCode != null) throw Error(logs);
return fetch(`http://127.0.0.1:${port}/`).then((r) => r.ok).catch(() => false);
});
browser = await launchChrome({ width: 390, height: 844 });
const page = await browser.openPage("about:blank");
await page.addInitScript(`(() => {
window.__errors = [];
window.addEventListener('error', (event) => window.__errors.push(event.message));
window.addEventListener('unhandledrejection', (event) => window.__errors.push(String(event.reason)));
const Native = window.WebSocket;
window.WebSocket = class extends Native { constructor(...args) { super(...args); window.__socket = this; } };
// Controlled geometry, not a claim that desktop Chrome has an iOS keyboard.
const viewport = Object.assign(new EventTarget(), {height:844,width:390,scale:1,offsetTop:0});
Object.defineProperty(window, 'visualViewport', {value:viewport,configurable:true});
})()`);
await page.goto(`http://127.0.0.1:${port}/chat/${sessionId}`);
await page.waitForSelector("#chatTranscript .chat-reasoning-toggle");
await page.click(".chat-model-button");
await page.waitForSelector("#chatModelSearch");
await page.fill("#chatModelSearch", "beta");
assert.equal(await page.evaluate("document.querySelectorAll('.chat-model-option').length"), 1);
assert.equal(await page.evaluate("document.querySelector('.chat-model-option-name').textContent"), "Model Beta");
assert.equal(await page.evaluate("document.querySelector('.app-root').inert"), true);
if (process.env.PIPELINES_CAPTURE_DIR) {
await mkdir(process.env.PIPELINES_CAPTURE_DIR, { recursive: true });
await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR, "managed-model-search.png"));
}
await page.evaluate("document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true, cancelable: true }))");
await eventually(() => page.evaluate("!document.querySelector('.confirm-chat-model')"));
const emit = (frame) => page.evaluate(`window.__socket.onmessage({data: JSON.stringify({type:'chat_event', chat_event: {session_id:${JSON.stringify(sessionId)}, ...${JSON.stringify(frame)}}})})`);
await emit({ message_id:"msg_answer",kind:"step",step:{status:"started"} });
await emit({ message_id:"msg_answer",kind:"step",step:{status:"ended",intermediate:true} });
await eventually(() => page.evaluate("!!document.querySelector('#chatStop') && !!document.querySelector('.chat-activity-spinner')"));
await page.click("#chatTranscript .chat-reasoning-toggle");
await page.waitForSelector(".chat-reasoning-viewer");
assert.equal(await page.evaluate("document.querySelector('.chat-reasoning-modal-body').textContent"), "Controlled reasoning evidence");
if (process.env.PIPELINES_CAPTURE_DIR) {
await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR, "stable-reasoning-popup.png"));
}
await page.evaluate("document.querySelector('.chat-reasoning-viewer .chat-tool-close').click()");
await emit({ message_id:"msg_answer",kind:"step",step:{status:"ended"} });
await eventually(() => page.evaluate("!document.querySelector('#chatStop') && !document.querySelector('.chat-activity-spinner')"));
await page.fill("#chatInput", "Keep this draft and caret");
await page.evaluate(`(() => {
const input = document.querySelector('#chatInput');
input.setSelectionRange(5, 9);
window.__composer = input;
window.visualViewport.height = 430;
window.visualViewport.dispatchEvent(new Event('resize'));
})()`);
await eventually(() => page.evaluate("document.documentElement.classList.contains('keyboard-open')"));
assert.equal(await page.evaluate("document.querySelector('#chatInput') === window.__composer && document.activeElement === window.__composer"), true);
assert.equal(await page.evaluate("document.querySelector('#chatInput').selectionStart"), 5);
assert.equal(await page.evaluate("getComputedStyle(document.querySelector('#chatInput')).fontSize"), "16px");
assert.equal(await page.evaluate("document.querySelector('#chatComposer').getBoundingClientRect().bottom <= 431"), true);
if (process.env.PIPELINES_CAPTURE_DIR) {
await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR, "keyboard-composer-visible.png"));
}
await page.evaluate("window.visualViewport.height = 844; window.visualViewport.dispatchEvent(new Event('resize'))");
await eventually(() => page.evaluate("!document.documentElement.classList.contains('keyboard-open')"));
await page.click(".chat-back");
await page.waitForSelector("#chatRows");
await page.evaluate("document.querySelector('#chatRows').scrollTop = 280");
const scroll = await page.evaluate("document.querySelector('#chatRows').scrollTop");
assert.ok(scroll > 0);
await page.evaluate("document.querySelector('#chatRows .chat-session-button').click()");
await page.waitForSelector(".chat-back");
await page.click(".chat-back");
await page.waitForSelector("#chatRows");
assert.equal(await page.evaluate("document.querySelector('#chatRows').scrollTop"), scroll);
// Activity changes color in place; it must not move the harness mark or add a
// Streaming label to the row.
const iconRect = await page.evaluate(`(() => { const icon=document.querySelector('#chatRows .chat-row[data-session="${sessionId}"] .chat-backend-logo'); window.__icon=icon; const r=icon.getBoundingClientRect(); return [r.x,r.y,r.width,r.height]; })()`);
await emit({ message_id:"msg_answer",kind:"step",step:{status:"started"} });
assert.equal(await page.evaluate("window.__icon.dataset.activity"), "streaming");
assert.equal(await page.evaluate("document.querySelectorAll('#chatRows .chat-activity-streaming').length"), 0);
assert.deepEqual(await page.evaluate("(() => { const r=window.__icon.getBoundingClientRect(); return [r.x,r.y,r.width,r.height]; })()"), iconRect);
await emit({ message_id:"msg_answer",kind:"step",step:{status:"ended"} });
await page.evaluate(`document.querySelector('#chatRows .chat-row[data-session="${sessionId}"] .chat-session-button').click()`);
await page.waitForSelector("#chatInput");
await page.waitForSelector("#chatTranscript");
await page.evaluate("(() => { const t=document.querySelector('#chatTranscript'); t.dispatchEvent(new WheelEvent('wheel',{deltaY:-240,bubbles:true})); t.scrollTop=0; t.dispatchEvent(new Event('scroll')); })()");
await page.fill("#chatInput", "New message must scroll into view");
await page.click("#chatSend");
await eventually(() => page.evaluate("(() => { const t=document.querySelector('#chatTranscript'); return t.textContent.includes('New message must scroll into view') && t.scrollHeight-t.scrollTop-t.clientHeight<3; })()"));
// These requests originate at the broker snapshot, not an injected browser
// frame. A reload must recover all of them, and each control reaches Go.
assert.equal((await fetch(`http://${target}/fixture-attention`,{method:'POST'})).status,204);
await eventually(() => page.evaluate("document.querySelectorAll('.chat-approval-pending').length===2 && document.querySelectorAll('.chat-question').length===2"));
await page.goto(`http://127.0.0.1:${port}/chat/${sessionId}`);
try {
await eventually(() => page.evaluate("document.querySelectorAll('.chat-approval-pending').length===2 && document.querySelectorAll('.chat-question').length===2"));
} catch(error) {
console.log(await page.evaluate("({approvals:document.querySelectorAll('.chat-approval-pending').length,questions:document.querySelectorAll('.chat-question').length,errors:window.__errors,text:document.body.innerText.slice(-2000)})"));
throw error;
}
assert.equal(await page.evaluate("!!document.querySelector('.chat-approval-jump')"),true);
await page.click('.chat-approval-jump');
if(process.env.PIPELINES_CAPTURE_DIR) await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR,'mobile-pending-interactions.png'));
await page.evaluate("document.querySelector('[data-request-id=per_first] .chat-approval-control').click()");
await eventually(() => page.evaluate("document.querySelectorAll('.chat-approval-pending').length===1"));
await page.evaluate("document.querySelector('[data-request-id=per_second] .chat-approval-control:last-child').click()");
await eventually(() => page.evaluate("document.querySelectorAll('.chat-approval-pending').length===0"));
for(const id of ['form_first','form_second']) {
await page.fill(`[data-form-id=${id}] .chat-question-input`,'Browser answer');
if(id==='form_second') {
await page.evaluate("(()=>{const s=document.querySelector('[data-form-id=form_second] select');s.value='true';s.dispatchEvent(new Event('change',{bubbles:true}));})()");
await page.fill('[data-form-id=form_second] input[aria-label="Only when enabled"]','must not leak');
await page.evaluate("(()=>{const s=document.querySelector('[data-form-id=form_second] select');s.value='false';s.dispatchEvent(new Event('change',{bubbles:true}));})()");
await page.fill('[data-form-id=form_second] input[type=number]','0');
assert.equal(await page.evaluate("document.querySelector('[data-form-id=form_second] input[aria-label=\"Only when enabled\"]').closest('.chat-question-field').getBoundingClientRect().height"),0);
assert.equal(await page.evaluate("document.querySelector('[data-form-id=form_second] input[aria-label=Secret]').type"),'password');
if(process.env.PIPELINES_CAPTURE_DIR) await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR,'typed-question-controls.png'));
}
await emit({message_id:'msg_answer',kind:'text',part:{id:'draft-check',type:'text',text:'unrelated frame'}});
await eventually(() => page.evaluate(`document.querySelector('[data-form-id=${id}] .chat-question-input')?.value==='Browser answer'`));
await page.evaluate(`document.querySelector('[data-form-id=${id}] .chat-question-submit').click()`);
await eventually(() => page.evaluate(`!document.querySelector('[data-form-id=${id}]')`));
}
await eventually(() => page.evaluate("!document.querySelector('.chat-approval-jump')"));
assert.equal((await fetch(`http://${target}/fixture-cancel`,{method:'POST'})).status,204);
await page.waitForSelector('[data-form-id=form_cancel]');
await page.evaluate("[...document.querySelectorAll('[data-form-id=form_cancel] button')].find(button=>button.textContent==='Cancel question').click()");
await eventually(()=>page.evaluate("!document.querySelector('[data-form-id=form_cancel]')"));
assert.equal((await fetch(`http://${target}/fixture-shell`,{method:'POST'})).status,204);
await page.goto(`http://127.0.0.1:${port}/chat/${sessionId}`);
await page.waitForSelector('.chat-bg-task');
await page.click('.chat-bg-task');
await eventually(()=>page.evaluate("document.querySelector('.chat-bg-task-output')?.textContent==='Controlled background output'"));
if(process.env.PIPELINES_CAPTURE_DIR) await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR,'harness-background-output.png'));
await page.fill('#chatInput','Keep this draft through readback');
await page.evaluate("window.__resyncComposer=document.querySelector('#chatInput')");
assert.equal((await fetch(`http://${target}/fixture-resync`,{method:'POST'})).status,204);
await emit({kind:'resync'});
await eventually(()=>page.evaluate("document.querySelector('#chatTranscript').textContent.includes('Authoritative resync evidence')"));
assert.equal(await page.evaluate("document.querySelector('#chatInput')===window.__resyncComposer && document.querySelector('#chatInput').value==='Keep this draft through readback'"),true);
assert.deepEqual(await page.evaluate("window.__errors"), []);
console.log("Model dialog, stable activity mark, keyboard/caret, Back restoration, send-to-bottom and all recovered approvals/questions passed.");
} finally {
await browser?.close();
serve.kill("SIGTERM");
await new Promise((resolve) => serve.exitCode != null ? resolve() : serve.once("exit", resolve));
}