test(chat): require interactive tool parity across advertised harnesses

This commit is contained in:
2026-10-20 18:12:59 +01:00
parent 68761652cf
commit 855de19a35
5 changed files with 115 additions and 6 deletions
+26
View File
@@ -32,3 +32,29 @@ not provide durable callback recovery; a disconnected transport is reported as
an error rather than claiming old requests are still actionable.
Claude is not advertised as a backend until its sidecar and login flow are wired.
## Web interaction contract
Both advertised chat adapters (OpenCode V2 and Codex) implement the same server
contract for uploads, models, pending approvals, questions, form replies,
interrupts, command readback and bounded output. Compile-time assertions and a
browser regression run against both backend identities protect this contract;
support is not satisfied by text streaming alone. Pi's pipeline executor is not
advertised as a Pi web-chat adapter, and an unavailable Claude adapter is not
presented as a working choice.
Codex uploads use native `localImage` for verified supported images. Other files
are retained intact with an explicit server-local reference; the model's own
tools inspect them rather than pretending their contents were already loaded.
The saved next-turn model is also respected with uploads. Model sandbox and
approval policy remain in force.
Codex's paged history retains tool and reasoning items, including structured MCP
errors and Code Mode outputs, instead of discarding everything except text.
Unknown item kinds remain inspectable. Tool detail is explicitly truncated at
64 KiB, not silently represented as complete. Running commands and their output
use real command items plus read-only thread history; no unsupported terminal
endpoint, process control, resumed turn or experimental flag is invented.
Output reads are bounded and verify thread ownership. The installed Codex
version exposes no API for controlling an agent-owned terminal through the
separate, client-owned `command/exec` process IDs; those IDs are never conflated.
+42 -6
View File
@@ -23,10 +23,11 @@ type browserModelFixture struct {
}
type browserForms struct {
mu sync.Mutex
pending []protocol.ChatForm
answered int
canceled int
mu sync.Mutex
pending []protocol.ChatForm
answered int
canceled int
outputReads int
}
func (b browserModelFixture) Forms(context.Context, string) ([]protocol.ChatForm, error) {
@@ -78,6 +79,20 @@ func (b browserModelFixture) Reply(ctx context.Context, answer protocol.ChatRepl
return nil
}
func (b browserModelFixture) ShellOutput(_ context.Context, q protocol.ChatShellOutputQuery) (protocol.ChatShellOutputPage, error) {
b.fakeChat.mu.Lock()
defer b.fakeChat.mu.Unlock()
for _, shell := range b.fakeChat.shells {
if shell.ID == q.ShellID && q.SessionID == b.fakeChat.sessions[0].ID {
b.forms.mu.Lock()
b.forms.outputReads++
b.forms.mu.Unlock()
return protocol.ChatShellOutputPage{Shell: shell, Output: "Controlled background output", Size: 28}, nil
}
}
return protocol.ChatShellOutputPage{}, fmt.Errorf("unknown task")
}
func (b browserModelFixture) Models(ctx context.Context, id string) ([]protocol.ChatModel, error) {
select {
case <-ctx.Done():
@@ -91,6 +106,12 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
if os.Getenv("PIPELINES_BROWSER_TEST") != "1" {
t.Skip("set PIPELINES_BROWSER_TEST=1 for the scratch Chrome check")
}
for _, backend := range []string{"opencode", "codex"} {
t.Run(backend, func(t *testing.T) { checkChatBrowserLifecycle(t, backend) })
}
}
func checkChatBrowserLifecycle(t *testing.T, backend string) {
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db"))
@@ -100,6 +121,10 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
defer store.Close()
directory := t.TempDir()
session := protocol.ChatSession{ID: "ses_browser", Title: "Scratch chat", Model: "model-a", Directory: directory}
session.Backend = backend
if backend == "codex" {
session.ID = "01900000-0000-7000-8000-000000000001"
}
if err := store.RecordOperatorSession(ctx, session.ID, directory, session.Title, protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
@@ -122,6 +147,13 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
handler := HandlerWithOptions(ctx, Options{Chat: browserModelFixture{fake, forms}}, store)
defer func() { cancel(); WaitRuns(handler) }()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if req.URL.Path == "/fixture-shell" && req.Method == http.MethodPost {
fake.mu.Lock()
fake.shells = []protocol.ChatShell{{ID: "sh_" + backend + "_fixture", Status: protocol.ChatShellRunning, Command: "printf controlled"}}
fake.mu.Unlock()
w.WriteHeader(http.StatusNoContent)
return
}
if req.URL.Path == "/fixture-cancel" && req.Method == http.MethodPost {
forms.mu.Lock()
forms.pending = append(forms.pending, protocol.ChatForm{ID: "form_cancel", SessionID: session.ID, Title: "Cancel this question", Actions: []string{"accept", "cancel"}, Fields: []protocol.ChatFormField{{Key: "answer", Type: "string", Required: true}}})
@@ -157,6 +189,9 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
}))
defer srv.Close()
cmd := exec.CommandContext(ctx, "node", "../../web/test/chat-hardening-probe.mjs", strings.TrimPrefix(srv.URL, "http://"), session.ID)
if captures := os.Getenv("PIPELINES_CAPTURE_DIR"); captures != "" {
cmd.Env = append(os.Environ(), "PIPELINES_CAPTURE_DIR="+filepath.Join(captures, backend))
}
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("chat browser regression: %v\n%s", err, output)
} else {
@@ -168,8 +203,9 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
forms.mu.Lock()
answers := forms.answered
canceled := forms.canceled
outputReads := forms.outputReads
forms.mu.Unlock()
if replies != 2 || answers != 2 || canceled != 1 {
t.Fatalf("browser did not deliver all interactions: approvals=%d answers=%d canceled=%d", replies, answers, canceled)
if replies != 2 || answers != 2 || canceled != 1 || outputReads != 1 {
t.Fatalf("browser did not deliver all interactions: approvals=%d answers=%d canceled=%d output=%d", replies, answers, canceled, outputReads)
}
}
+31
View File
@@ -0,0 +1,31 @@
package server
import (
"testing"
"git.a-wan.net/alan/pipelines/internal/codex"
"git.a-wan.net/alan/pipelines/internal/opencode"
)
// Advertising either real harness must not regress to a broker that can stream
// text but cannot deliver the interactive tools the web needs to finish a turn.
type interactiveWebChat interface {
chatBroker
chatPromptWithFiles
chatAnswerer
chatFormReader
chatModeler
chatShellOutputReader
}
var _ interactiveWebChat = (*opencode.Chat)(nil)
var _ interactiveWebChat = (*codex.Chat)(nil)
func TestAdvertisedChatHarnessesExposeInteractiveWebContract(t *testing.T) {
m := &chatManager{opencode: opencode.NewChat(&opencode.Connection{}), codex: &codex.Chat{}}
for _, backend := range m.Backends() {
if _, ok := m.adapter(backend).(interactiveWebChat); !ok {
t.Errorf("advertised harness %s lacks web tool controls", backend)
}
}
}
+10
View File
@@ -254,6 +254,16 @@ to it, but the server never lists it and a reload drops it.
## Web interaction recovery (2026-10-20)
Cross-harness contract follow-up: the same browser interaction regression now
runs with OpenCode and Codex identities. Codex additionally accepts prepared
uploads without losing the selected model, retains all native tool/history
items (including structured MCP failures and Code Mode outputs), pages history
with opaque cursors, and exposes running-command/output readback through the
shared background-task UI. Command ownership/output bounds and exact installed
protocol shapes are tested. Read-only live Codex validation passed history,
command listing and model listing without starting/resuming a turn or changing
settings. No real Codex command or permission grant was used for testing.
Typed-tool follow-up: number/integer/boolean/default/conditional/secret and
external-link fields use native controls, preserve JSON types and validate before
sending. `typed-question-controls.png` and the scratch browser regression cover
+6
View File
@@ -146,6 +146,12 @@ try {
await page.waitForSelector('[data-form-id=form_cancel]');
await page.evaluate("[...document.querySelectorAll('[data-form-id=form_cancel] button')].find(button=>button.textContent==='Cancel question').click()");
await eventually(()=>page.evaluate("!document.querySelector('[data-form-id=form_cancel]')"));
assert.equal((await fetch(`http://${target}/fixture-shell`,{method:'POST'})).status,204);
await page.goto(`http://127.0.0.1:${port}/chat/${sessionId}`);
await page.waitForSelector('.chat-bg-task');
await page.click('.chat-bg-task');
await eventually(()=>page.evaluate("document.querySelector('.chat-bg-task-output')?.textContent==='Controlled background output'"));
if(process.env.PIPELINES_CAPTURE_DIR) await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR,'harness-background-output.png'));
assert.deepEqual(await page.evaluate("window.__errors"), []);
console.log("Model dialog, stable activity mark, keyboard/caret, Back restoration, send-to-bottom and all recovered approvals/questions passed.");
} finally {