test: exercise managed model dialogs and intermediate chat steps in Chrome
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
type browserModelFixture struct{ *fakeChat }
|
||||
|
||||
func (b browserModelFixture) Models(ctx context.Context, id string) ([]protocol.ChatModel, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(250 * time.Millisecond):
|
||||
}
|
||||
return b.fakeChat.Models(ctx, id)
|
||||
}
|
||||
|
||||
func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
|
||||
if os.Getenv("PIPELINES_BROWSER_TEST") != "1" {
|
||||
t.Skip("set PIPELINES_BROWSER_TEST=1 for the scratch Chrome check")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
directory := t.TempDir()
|
||||
session := protocol.ChatSession{ID: "ses_browser", Title: "Scratch chat", Model: "model-a", Directory: directory}
|
||||
if err := store.RecordOperatorSession(ctx, session.ID, directory, session.Title, protocol.ChatSessionTagOperator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fake := scriptedChat()
|
||||
fake.block = true
|
||||
fake.sessions = []protocol.ChatSession{session}
|
||||
fake.models = []protocol.ChatModel{{ID: "model-a", Provider: "fixture", Name: "Model Alpha"}, {ID: "model-b", Provider: "fixture", Name: "Model Beta"}}
|
||||
fake.messages = []protocol.ChatMessage{
|
||||
{ID: "msg_user", Role: protocol.ChatRoleUser, Parts: []protocol.ChatPart{{Type: "text", Text: "Scratch prompt"}}},
|
||||
{ID: "msg_answer", Role: protocol.ChatRoleAssistant, Parts: []protocol.ChatPart{{Type: "reasoning", Text: "Controlled reasoning evidence"}, {Type: "text", Text: "Controlled answer"}}},
|
||||
}
|
||||
handler := HandlerWithOptions(ctx, Options{Chat: browserModelFixture{fake}}, store)
|
||||
defer func() { cancel(); WaitRuns(handler) }()
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
cmd := exec.CommandContext(ctx, "node", "../../web/test/chat-hardening-probe.mjs", strings.TrimPrefix(srv.URL, "http://"), session.ID)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("chat browser regression: %v\n%s", err, output)
|
||||
} else {
|
||||
t.Log(string(output))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import net from "node:net";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { eventually, launchChrome } from "./browser.mjs";
|
||||
|
||||
const [target, sessionId] = process.argv.slice(2);
|
||||
assert.match(target ?? "", /^127\.0\.0\.1:\d+$/);
|
||||
const reserve = net.createServer();
|
||||
await new Promise((resolve) => reserve.listen(0, "127.0.0.1", resolve));
|
||||
const port = reserve.address().port;
|
||||
await new Promise((resolve) => reserve.close(resolve));
|
||||
const serve = spawn(process.execPath, ["serve.mjs", "--port", String(port), "--pipelines", target], {
|
||||
cwd: fileURLToPath(new URL("..", import.meta.url)),
|
||||
env: { ...process.env, PIPELINES_WEB_AUTH: "disabled" }, stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
let logs = "";
|
||||
serve.stdout.on("data", (data) => { logs += data; });
|
||||
serve.stderr.on("data", (data) => { logs += data; });
|
||||
let browser;
|
||||
try {
|
||||
await eventually(async () => {
|
||||
if (serve.exitCode != null) throw Error(logs);
|
||||
return fetch(`http://127.0.0.1:${port}/`).then((r) => r.ok).catch(() => false);
|
||||
});
|
||||
browser = await launchChrome({ width: 390, height: 844 });
|
||||
const page = await browser.openPage("about:blank");
|
||||
await page.addInitScript(`(() => {
|
||||
window.__errors = [];
|
||||
window.addEventListener('error', (event) => window.__errors.push(event.message));
|
||||
window.addEventListener('unhandledrejection', (event) => window.__errors.push(String(event.reason)));
|
||||
const Native = window.WebSocket;
|
||||
window.WebSocket = class extends Native { constructor(...args) { super(...args); window.__socket = this; } };
|
||||
})()`);
|
||||
await page.goto(`http://127.0.0.1:${port}/chat/${sessionId}`);
|
||||
await page.waitForSelector("#chatTranscript .chat-reasoning-toggle");
|
||||
await page.click(".chat-model-button");
|
||||
await page.waitForSelector("#chatModelSearch");
|
||||
await page.fill("#chatModelSearch", "beta");
|
||||
assert.equal(await page.evaluate("document.querySelectorAll('.chat-model-option').length"), 1);
|
||||
assert.equal(await page.evaluate("document.querySelector('.chat-model-option-name').textContent"), "Model Beta");
|
||||
assert.equal(await page.evaluate("document.querySelector('.app-root').inert"), true);
|
||||
if (process.env.PIPELINES_CAPTURE_DIR) {
|
||||
await mkdir(process.env.PIPELINES_CAPTURE_DIR, { recursive: true });
|
||||
await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR, "managed-model-search.png"));
|
||||
}
|
||||
await page.evaluate("document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true, cancelable: true }))");
|
||||
await eventually(() => page.evaluate("!document.querySelector('.confirm-chat-model')"));
|
||||
const emit = (frame) => page.evaluate(`window.__socket.onmessage({data: JSON.stringify({type:'chat_event', chat_event: {session_id:${JSON.stringify(sessionId)}, ...${JSON.stringify(frame)}}})})`);
|
||||
await emit({ message_id:"msg_answer",kind:"step",step:{status:"started"} });
|
||||
await emit({ message_id:"msg_answer",kind:"step",step:{status:"ended",intermediate:true} });
|
||||
await eventually(() => page.evaluate("!!document.querySelector('#chatStop') && !!document.querySelector('.chat-activity-spinner')"));
|
||||
await page.click("#chatTranscript .chat-reasoning-toggle");
|
||||
await page.waitForSelector(".chat-reasoning-viewer");
|
||||
assert.equal(await page.evaluate("document.querySelector('.chat-reasoning-modal-body').textContent"), "Controlled reasoning evidence");
|
||||
if (process.env.PIPELINES_CAPTURE_DIR) {
|
||||
await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR, "stable-reasoning-popup.png"));
|
||||
}
|
||||
await page.evaluate("document.querySelector('.chat-reasoning-viewer .chat-tool-close').click()");
|
||||
await emit({ message_id:"msg_answer",kind:"step",step:{status:"ended"} });
|
||||
await eventually(() => page.evaluate("!document.querySelector('#chatStop') && !document.querySelector('.chat-activity-spinner')"));
|
||||
assert.deepEqual(await page.evaluate("window.__errors"), []);
|
||||
console.log("Delayed model dialog loads, filters and traps focus; intermediate step retains Stop until terminal turn.");
|
||||
} finally {
|
||||
await browser?.close();
|
||||
serve.kill("SIGTERM");
|
||||
await new Promise((resolve) => serve.exitCode != null ? resolve() : serve.once("exit", resolve));
|
||||
}
|
||||
@@ -82,6 +82,7 @@ try {
|
||||
assert.equal(await page.evaluate("document.querySelector('#runPrompt').disabled"), true);
|
||||
if (process.env.PIPELINES_CAPTURE_DIR) {
|
||||
await mkdir(process.env.PIPELINES_CAPTURE_DIR, { recursive: true });
|
||||
await eventually(() => page.evaluate("document.querySelector('.run-start-dialog').getAnimations({subtree:true}).every(a => a.playState !== 'running')"));
|
||||
await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR, "uncertain-run-recovered.png"));
|
||||
}
|
||||
await page.click(".run-start-confirm");
|
||||
|
||||
Reference in New Issue
Block a user