test: exercise managed model dialogs and intermediate chat steps in Chrome

This commit is contained in:
2026-10-20 15:53:59 +01:00
parent b96b9d94a6
commit f0532e5ec4
3 changed files with 133 additions and 0 deletions
+62
View File
@@ -0,0 +1,62 @@
package server
import (
"context"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
type browserModelFixture struct{ *fakeChat }
func (b browserModelFixture) Models(ctx context.Context, id string) ([]protocol.ChatModel, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(250 * time.Millisecond):
}
return b.fakeChat.Models(ctx, id)
}
func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
if os.Getenv("PIPELINES_BROWSER_TEST") != "1" {
t.Skip("set PIPELINES_BROWSER_TEST=1 for the scratch Chrome check")
}
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
directory := t.TempDir()
session := protocol.ChatSession{ID: "ses_browser", Title: "Scratch chat", Model: "model-a", Directory: directory}
if err := store.RecordOperatorSession(ctx, session.ID, directory, session.Title, protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
fake := scriptedChat()
fake.block = true
fake.sessions = []protocol.ChatSession{session}
fake.models = []protocol.ChatModel{{ID: "model-a", Provider: "fixture", Name: "Model Alpha"}, {ID: "model-b", Provider: "fixture", Name: "Model Beta"}}
fake.messages = []protocol.ChatMessage{
{ID: "msg_user", Role: protocol.ChatRoleUser, Parts: []protocol.ChatPart{{Type: "text", Text: "Scratch prompt"}}},
{ID: "msg_answer", Role: protocol.ChatRoleAssistant, Parts: []protocol.ChatPart{{Type: "reasoning", Text: "Controlled reasoning evidence"}, {Type: "text", Text: "Controlled answer"}}},
}
handler := HandlerWithOptions(ctx, Options{Chat: browserModelFixture{fake}}, store)
defer func() { cancel(); WaitRuns(handler) }()
srv := httptest.NewServer(handler)
defer srv.Close()
cmd := exec.CommandContext(ctx, "node", "../../web/test/chat-hardening-probe.mjs", strings.TrimPrefix(srv.URL, "http://"), session.ID)
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("chat browser regression: %v\n%s", err, output)
} else {
t.Log(string(output))
}
}
+70
View File
@@ -0,0 +1,70 @@
import assert from "node:assert/strict";
import { spawn } from "node:child_process";
import { mkdir } from "node:fs/promises";
import { join } from "node:path";
import net from "node:net";
import { fileURLToPath } from "node:url";
import { eventually, launchChrome } from "./browser.mjs";
const [target, sessionId] = process.argv.slice(2);
assert.match(target ?? "", /^127\.0\.0\.1:\d+$/);
const reserve = net.createServer();
await new Promise((resolve) => reserve.listen(0, "127.0.0.1", resolve));
const port = reserve.address().port;
await new Promise((resolve) => reserve.close(resolve));
const serve = spawn(process.execPath, ["serve.mjs", "--port", String(port), "--pipelines", target], {
cwd: fileURLToPath(new URL("..", import.meta.url)),
env: { ...process.env, PIPELINES_WEB_AUTH: "disabled" }, stdio: ["ignore", "pipe", "pipe"],
});
let logs = "";
serve.stdout.on("data", (data) => { logs += data; });
serve.stderr.on("data", (data) => { logs += data; });
let browser;
try {
await eventually(async () => {
if (serve.exitCode != null) throw Error(logs);
return fetch(`http://127.0.0.1:${port}/`).then((r) => r.ok).catch(() => false);
});
browser = await launchChrome({ width: 390, height: 844 });
const page = await browser.openPage("about:blank");
await page.addInitScript(`(() => {
window.__errors = [];
window.addEventListener('error', (event) => window.__errors.push(event.message));
window.addEventListener('unhandledrejection', (event) => window.__errors.push(String(event.reason)));
const Native = window.WebSocket;
window.WebSocket = class extends Native { constructor(...args) { super(...args); window.__socket = this; } };
})()`);
await page.goto(`http://127.0.0.1:${port}/chat/${sessionId}`);
await page.waitForSelector("#chatTranscript .chat-reasoning-toggle");
await page.click(".chat-model-button");
await page.waitForSelector("#chatModelSearch");
await page.fill("#chatModelSearch", "beta");
assert.equal(await page.evaluate("document.querySelectorAll('.chat-model-option').length"), 1);
assert.equal(await page.evaluate("document.querySelector('.chat-model-option-name').textContent"), "Model Beta");
assert.equal(await page.evaluate("document.querySelector('.app-root').inert"), true);
if (process.env.PIPELINES_CAPTURE_DIR) {
await mkdir(process.env.PIPELINES_CAPTURE_DIR, { recursive: true });
await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR, "managed-model-search.png"));
}
await page.evaluate("document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true, cancelable: true }))");
await eventually(() => page.evaluate("!document.querySelector('.confirm-chat-model')"));
const emit = (frame) => page.evaluate(`window.__socket.onmessage({data: JSON.stringify({type:'chat_event', chat_event: {session_id:${JSON.stringify(sessionId)}, ...${JSON.stringify(frame)}}})})`);
await emit({ message_id:"msg_answer",kind:"step",step:{status:"started"} });
await emit({ message_id:"msg_answer",kind:"step",step:{status:"ended",intermediate:true} });
await eventually(() => page.evaluate("!!document.querySelector('#chatStop') && !!document.querySelector('.chat-activity-spinner')"));
await page.click("#chatTranscript .chat-reasoning-toggle");
await page.waitForSelector(".chat-reasoning-viewer");
assert.equal(await page.evaluate("document.querySelector('.chat-reasoning-modal-body').textContent"), "Controlled reasoning evidence");
if (process.env.PIPELINES_CAPTURE_DIR) {
await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR, "stable-reasoning-popup.png"));
}
await page.evaluate("document.querySelector('.chat-reasoning-viewer .chat-tool-close').click()");
await emit({ message_id:"msg_answer",kind:"step",step:{status:"ended"} });
await eventually(() => page.evaluate("!document.querySelector('#chatStop') && !document.querySelector('.chat-activity-spinner')"));
assert.deepEqual(await page.evaluate("window.__errors"), []);
console.log("Delayed model dialog loads, filters and traps focus; intermediate step retains Stop until terminal turn.");
} finally {
await browser?.close();
serve.kill("SIGTERM");
await new Promise((resolve) => serve.exitCode != null ? resolve() : serve.once("exit", resolve));
}
+1
View File
@@ -82,6 +82,7 @@ try {
assert.equal(await page.evaluate("document.querySelector('#runPrompt').disabled"), true);
if (process.env.PIPELINES_CAPTURE_DIR) {
await mkdir(process.env.PIPELINES_CAPTURE_DIR, { recursive: true });
await eventually(() => page.evaluate("document.querySelector('.run-start-dialog').getAnimations({subtree:true}).every(a => a.playState !== 'running')"));
await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR, "uncertain-run-recovered.png"));
}
await page.click(".run-start-confirm");