From f0532e5ec490df502a58da94d3f7b3fcbee6f1d3 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Tue, 20 Oct 2026 15:53:59 +0100 Subject: [PATCH] test: exercise managed model dialogs and intermediate chat steps in Chrome --- internal/server/chat_browser_test.go | 62 ++++++++++++++++++++++++ web/test/chat-hardening-probe.mjs | 70 ++++++++++++++++++++++++++++ web/test/run-admission-probe.mjs | 1 + 3 files changed, 133 insertions(+) create mode 100644 internal/server/chat_browser_test.go create mode 100644 web/test/chat-hardening-probe.mjs diff --git a/internal/server/chat_browser_test.go b/internal/server/chat_browser_test.go new file mode 100644 index 00000000..839b0cc0 --- /dev/null +++ b/internal/server/chat_browser_test.go @@ -0,0 +1,62 @@ +package server + +import ( + "context" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +type browserModelFixture struct{ *fakeChat } + +func (b browserModelFixture) Models(ctx context.Context, id string) ([]protocol.ChatModel, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(250 * time.Millisecond): + } + return b.fakeChat.Models(ctx, id) +} + +func TestChatBrowserLifecycleAndModelDialog(t *testing.T) { + if os.Getenv("PIPELINES_BROWSER_TEST") != "1" { + t.Skip("set PIPELINES_BROWSER_TEST=1 for the scratch Chrome check") + } + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + directory := t.TempDir() + session := protocol.ChatSession{ID: "ses_browser", Title: "Scratch chat", Model: "model-a", Directory: directory} + if err := store.RecordOperatorSession(ctx, session.ID, directory, session.Title, protocol.ChatSessionTagOperator); err != nil { + t.Fatal(err) + } + fake := scriptedChat() + fake.block = true + fake.sessions = []protocol.ChatSession{session} + fake.models = []protocol.ChatModel{{ID: "model-a", Provider: "fixture", Name: "Model Alpha"}, {ID: "model-b", Provider: "fixture", Name: "Model Beta"}} + fake.messages = []protocol.ChatMessage{ + {ID: "msg_user", Role: protocol.ChatRoleUser, Parts: []protocol.ChatPart{{Type: "text", Text: "Scratch prompt"}}}, + {ID: "msg_answer", Role: protocol.ChatRoleAssistant, Parts: []protocol.ChatPart{{Type: "reasoning", Text: "Controlled reasoning evidence"}, {Type: "text", Text: "Controlled answer"}}}, + } + handler := HandlerWithOptions(ctx, Options{Chat: browserModelFixture{fake}}, store) + defer func() { cancel(); WaitRuns(handler) }() + srv := httptest.NewServer(handler) + defer srv.Close() + cmd := exec.CommandContext(ctx, "node", "../../web/test/chat-hardening-probe.mjs", strings.TrimPrefix(srv.URL, "http://"), session.ID) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("chat browser regression: %v\n%s", err, output) + } else { + t.Log(string(output)) + } +} diff --git a/web/test/chat-hardening-probe.mjs b/web/test/chat-hardening-probe.mjs new file mode 100644 index 00000000..6a34108d --- /dev/null +++ b/web/test/chat-hardening-probe.mjs @@ -0,0 +1,70 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import net from "node:net"; +import { fileURLToPath } from "node:url"; +import { eventually, launchChrome } from "./browser.mjs"; + +const [target, sessionId] = process.argv.slice(2); +assert.match(target ?? "", /^127\.0\.0\.1:\d+$/); +const reserve = net.createServer(); +await new Promise((resolve) => reserve.listen(0, "127.0.0.1", resolve)); +const port = reserve.address().port; +await new Promise((resolve) => reserve.close(resolve)); +const serve = spawn(process.execPath, ["serve.mjs", "--port", String(port), "--pipelines", target], { + cwd: fileURLToPath(new URL("..", import.meta.url)), + env: { ...process.env, PIPELINES_WEB_AUTH: "disabled" }, stdio: ["ignore", "pipe", "pipe"], +}); +let logs = ""; +serve.stdout.on("data", (data) => { logs += data; }); +serve.stderr.on("data", (data) => { logs += data; }); +let browser; +try { + await eventually(async () => { + if (serve.exitCode != null) throw Error(logs); + return fetch(`http://127.0.0.1:${port}/`).then((r) => r.ok).catch(() => false); + }); + browser = await launchChrome({ width: 390, height: 844 }); + const page = await browser.openPage("about:blank"); + await page.addInitScript(`(() => { + window.__errors = []; + window.addEventListener('error', (event) => window.__errors.push(event.message)); + window.addEventListener('unhandledrejection', (event) => window.__errors.push(String(event.reason))); + const Native = window.WebSocket; + window.WebSocket = class extends Native { constructor(...args) { super(...args); window.__socket = this; } }; + })()`); + await page.goto(`http://127.0.0.1:${port}/chat/${sessionId}`); + await page.waitForSelector("#chatTranscript .chat-reasoning-toggle"); + await page.click(".chat-model-button"); + await page.waitForSelector("#chatModelSearch"); + await page.fill("#chatModelSearch", "beta"); + assert.equal(await page.evaluate("document.querySelectorAll('.chat-model-option').length"), 1); + assert.equal(await page.evaluate("document.querySelector('.chat-model-option-name').textContent"), "Model Beta"); + assert.equal(await page.evaluate("document.querySelector('.app-root').inert"), true); + if (process.env.PIPELINES_CAPTURE_DIR) { + await mkdir(process.env.PIPELINES_CAPTURE_DIR, { recursive: true }); + await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR, "managed-model-search.png")); + } + await page.evaluate("document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true, cancelable: true }))"); + await eventually(() => page.evaluate("!document.querySelector('.confirm-chat-model')")); + const emit = (frame) => page.evaluate(`window.__socket.onmessage({data: JSON.stringify({type:'chat_event', chat_event: {session_id:${JSON.stringify(sessionId)}, ...${JSON.stringify(frame)}}})})`); + await emit({ message_id:"msg_answer",kind:"step",step:{status:"started"} }); + await emit({ message_id:"msg_answer",kind:"step",step:{status:"ended",intermediate:true} }); + await eventually(() => page.evaluate("!!document.querySelector('#chatStop') && !!document.querySelector('.chat-activity-spinner')")); + await page.click("#chatTranscript .chat-reasoning-toggle"); + await page.waitForSelector(".chat-reasoning-viewer"); + assert.equal(await page.evaluate("document.querySelector('.chat-reasoning-modal-body').textContent"), "Controlled reasoning evidence"); + if (process.env.PIPELINES_CAPTURE_DIR) { + await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR, "stable-reasoning-popup.png")); + } + await page.evaluate("document.querySelector('.chat-reasoning-viewer .chat-tool-close').click()"); + await emit({ message_id:"msg_answer",kind:"step",step:{status:"ended"} }); + await eventually(() => page.evaluate("!document.querySelector('#chatStop') && !document.querySelector('.chat-activity-spinner')")); + assert.deepEqual(await page.evaluate("window.__errors"), []); + console.log("Delayed model dialog loads, filters and traps focus; intermediate step retains Stop until terminal turn."); +} finally { + await browser?.close(); + serve.kill("SIGTERM"); + await new Promise((resolve) => serve.exitCode != null ? resolve() : serve.once("exit", resolve)); +} diff --git a/web/test/run-admission-probe.mjs b/web/test/run-admission-probe.mjs index 0bf3fe08..9351d5b4 100644 --- a/web/test/run-admission-probe.mjs +++ b/web/test/run-admission-probe.mjs @@ -82,6 +82,7 @@ try { assert.equal(await page.evaluate("document.querySelector('#runPrompt').disabled"), true); if (process.env.PIPELINES_CAPTURE_DIR) { await mkdir(process.env.PIPELINES_CAPTURE_DIR, { recursive: true }); + await eventually(() => page.evaluate("document.querySelector('.run-start-dialog').getAnimations({subtree:true}).every(a => a.playState !== 'running')")); await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR, "uncertain-run-recovered.png")); } await page.click(".run-start-confirm");