From 855de19a351aec6e5e4f69cb9dee4006a271dc2a Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Tue, 20 Oct 2026 18:12:59 +0100 Subject: [PATCH] test(chat): require interactive tool parity across advertised harnesses --- docs/chat-backends.md | 26 +++++++++++++++ internal/server/chat_browser_test.go | 48 +++++++++++++++++++++++---- internal/server/chat_contract_test.go | 31 +++++++++++++++++ web/PARITY.md | 10 ++++++ web/test/chat-hardening-probe.mjs | 6 ++++ 5 files changed, 115 insertions(+), 6 deletions(-) create mode 100644 internal/server/chat_contract_test.go diff --git a/docs/chat-backends.md b/docs/chat-backends.md index cb4c5a74..0a950346 100644 --- a/docs/chat-backends.md +++ b/docs/chat-backends.md @@ -32,3 +32,29 @@ not provide durable callback recovery; a disconnected transport is reported as an error rather than claiming old requests are still actionable. Claude is not advertised as a backend until its sidecar and login flow are wired. + +## Web interaction contract + +Both advertised chat adapters (OpenCode V2 and Codex) implement the same server +contract for uploads, models, pending approvals, questions, form replies, +interrupts, command readback and bounded output. Compile-time assertions and a +browser regression run against both backend identities protect this contract; +support is not satisfied by text streaming alone. Pi's pipeline executor is not +advertised as a Pi web-chat adapter, and an unavailable Claude adapter is not +presented as a working choice. + +Codex uploads use native `localImage` for verified supported images. Other files +are retained intact with an explicit server-local reference; the model's own +tools inspect them rather than pretending their contents were already loaded. +The saved next-turn model is also respected with uploads. Model sandbox and +approval policy remain in force. + +Codex's paged history retains tool and reasoning items, including structured MCP +errors and Code Mode outputs, instead of discarding everything except text. +Unknown item kinds remain inspectable. Tool detail is explicitly truncated at +64 KiB, not silently represented as complete. Running commands and their output +use real command items plus read-only thread history; no unsupported terminal +endpoint, process control, resumed turn or experimental flag is invented. +Output reads are bounded and verify thread ownership. The installed Codex +version exposes no API for controlling an agent-owned terminal through the +separate, client-owned `command/exec` process IDs; those IDs are never conflated. diff --git a/internal/server/chat_browser_test.go b/internal/server/chat_browser_test.go index 3771701c..b8d04266 100644 --- a/internal/server/chat_browser_test.go +++ b/internal/server/chat_browser_test.go @@ -23,10 +23,11 @@ type browserModelFixture struct { } type browserForms struct { - mu sync.Mutex - pending []protocol.ChatForm - answered int - canceled int + mu sync.Mutex + pending []protocol.ChatForm + answered int + canceled int + outputReads int } func (b browserModelFixture) Forms(context.Context, string) ([]protocol.ChatForm, error) { @@ -78,6 +79,20 @@ func (b browserModelFixture) Reply(ctx context.Context, answer protocol.ChatRepl return nil } +func (b browserModelFixture) ShellOutput(_ context.Context, q protocol.ChatShellOutputQuery) (protocol.ChatShellOutputPage, error) { + b.fakeChat.mu.Lock() + defer b.fakeChat.mu.Unlock() + for _, shell := range b.fakeChat.shells { + if shell.ID == q.ShellID && q.SessionID == b.fakeChat.sessions[0].ID { + b.forms.mu.Lock() + b.forms.outputReads++ + b.forms.mu.Unlock() + return protocol.ChatShellOutputPage{Shell: shell, Output: "Controlled background output", Size: 28}, nil + } + } + return protocol.ChatShellOutputPage{}, fmt.Errorf("unknown task") +} + func (b browserModelFixture) Models(ctx context.Context, id string) ([]protocol.ChatModel, error) { select { case <-ctx.Done(): @@ -91,6 +106,12 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) { if os.Getenv("PIPELINES_BROWSER_TEST") != "1" { t.Skip("set PIPELINES_BROWSER_TEST=1 for the scratch Chrome check") } + for _, backend := range []string{"opencode", "codex"} { + t.Run(backend, func(t *testing.T) { checkChatBrowserLifecycle(t, backend) }) + } +} + +func checkChatBrowserLifecycle(t *testing.T, backend string) { ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) defer cancel() store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db")) @@ -100,6 +121,10 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) { defer store.Close() directory := t.TempDir() session := protocol.ChatSession{ID: "ses_browser", Title: "Scratch chat", Model: "model-a", Directory: directory} + session.Backend = backend + if backend == "codex" { + session.ID = "01900000-0000-7000-8000-000000000001" + } if err := store.RecordOperatorSession(ctx, session.ID, directory, session.Title, protocol.ChatSessionTagOperator); err != nil { t.Fatal(err) } @@ -122,6 +147,13 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) { handler := HandlerWithOptions(ctx, Options{Chat: browserModelFixture{fake, forms}}, store) defer func() { cancel(); WaitRuns(handler) }() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path == "/fixture-shell" && req.Method == http.MethodPost { + fake.mu.Lock() + fake.shells = []protocol.ChatShell{{ID: "sh_" + backend + "_fixture", Status: protocol.ChatShellRunning, Command: "printf controlled"}} + fake.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + return + } if req.URL.Path == "/fixture-cancel" && req.Method == http.MethodPost { forms.mu.Lock() forms.pending = append(forms.pending, protocol.ChatForm{ID: "form_cancel", SessionID: session.ID, Title: "Cancel this question", Actions: []string{"accept", "cancel"}, Fields: []protocol.ChatFormField{{Key: "answer", Type: "string", Required: true}}}) @@ -157,6 +189,9 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) { })) defer srv.Close() cmd := exec.CommandContext(ctx, "node", "../../web/test/chat-hardening-probe.mjs", strings.TrimPrefix(srv.URL, "http://"), session.ID) + if captures := os.Getenv("PIPELINES_CAPTURE_DIR"); captures != "" { + cmd.Env = append(os.Environ(), "PIPELINES_CAPTURE_DIR="+filepath.Join(captures, backend)) + } if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("chat browser regression: %v\n%s", err, output) } else { @@ -168,8 +203,9 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) { forms.mu.Lock() answers := forms.answered canceled := forms.canceled + outputReads := forms.outputReads forms.mu.Unlock() - if replies != 2 || answers != 2 || canceled != 1 { - t.Fatalf("browser did not deliver all interactions: approvals=%d answers=%d canceled=%d", replies, answers, canceled) + if replies != 2 || answers != 2 || canceled != 1 || outputReads != 1 { + t.Fatalf("browser did not deliver all interactions: approvals=%d answers=%d canceled=%d output=%d", replies, answers, canceled, outputReads) } } diff --git a/internal/server/chat_contract_test.go b/internal/server/chat_contract_test.go new file mode 100644 index 00000000..de9b6251 --- /dev/null +++ b/internal/server/chat_contract_test.go @@ -0,0 +1,31 @@ +package server + +import ( + "testing" + + "git.a-wan.net/alan/pipelines/internal/codex" + "git.a-wan.net/alan/pipelines/internal/opencode" +) + +// Advertising either real harness must not regress to a broker that can stream +// text but cannot deliver the interactive tools the web needs to finish a turn. +type interactiveWebChat interface { + chatBroker + chatPromptWithFiles + chatAnswerer + chatFormReader + chatModeler + chatShellOutputReader +} + +var _ interactiveWebChat = (*opencode.Chat)(nil) +var _ interactiveWebChat = (*codex.Chat)(nil) + +func TestAdvertisedChatHarnessesExposeInteractiveWebContract(t *testing.T) { + m := &chatManager{opencode: opencode.NewChat(&opencode.Connection{}), codex: &codex.Chat{}} + for _, backend := range m.Backends() { + if _, ok := m.adapter(backend).(interactiveWebChat); !ok { + t.Errorf("advertised harness %s lacks web tool controls", backend) + } + } +} diff --git a/web/PARITY.md b/web/PARITY.md index 5a472880..84b5e558 100644 --- a/web/PARITY.md +++ b/web/PARITY.md @@ -254,6 +254,16 @@ to it, but the server never lists it and a reload drops it. ## Web interaction recovery (2026-10-20) +Cross-harness contract follow-up: the same browser interaction regression now +runs with OpenCode and Codex identities. Codex additionally accepts prepared +uploads without losing the selected model, retains all native tool/history +items (including structured MCP failures and Code Mode outputs), pages history +with opaque cursors, and exposes running-command/output readback through the +shared background-task UI. Command ownership/output bounds and exact installed +protocol shapes are tested. Read-only live Codex validation passed history, +command listing and model listing without starting/resuming a turn or changing +settings. No real Codex command or permission grant was used for testing. + Typed-tool follow-up: number/integer/boolean/default/conditional/secret and external-link fields use native controls, preserve JSON types and validate before sending. `typed-question-controls.png` and the scratch browser regression cover diff --git a/web/test/chat-hardening-probe.mjs b/web/test/chat-hardening-probe.mjs index b1ee3363..262c4e46 100644 --- a/web/test/chat-hardening-probe.mjs +++ b/web/test/chat-hardening-probe.mjs @@ -146,6 +146,12 @@ try { await page.waitForSelector('[data-form-id=form_cancel]'); await page.evaluate("[...document.querySelectorAll('[data-form-id=form_cancel] button')].find(button=>button.textContent==='Cancel question').click()"); await eventually(()=>page.evaluate("!document.querySelector('[data-form-id=form_cancel]')")); + assert.equal((await fetch(`http://${target}/fixture-shell`,{method:'POST'})).status,204); + await page.goto(`http://127.0.0.1:${port}/chat/${sessionId}`); + await page.waitForSelector('.chat-bg-task'); + await page.click('.chat-bg-task'); + await eventually(()=>page.evaluate("document.querySelector('.chat-bg-task-output')?.textContent==='Controlled background output'")); + if(process.env.PIPELINES_CAPTURE_DIR) await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR,'harness-background-output.png')); assert.deepEqual(await page.evaluate("window.__errors"), []); console.log("Model dialog, stable activity mark, keyboard/caret, Back restoration, send-to-bottom and all recovered approvals/questions passed."); } finally {