test(chat): require interactive tool parity across advertised harnesses

This commit is contained in:
2026-10-20 18:12:59 +01:00
parent 68761652cf
commit 855de19a35
5 changed files with 115 additions and 6 deletions
+42 -6
View File
@@ -23,10 +23,11 @@ type browserModelFixture struct {
}
type browserForms struct {
mu sync.Mutex
pending []protocol.ChatForm
answered int
canceled int
mu sync.Mutex
pending []protocol.ChatForm
answered int
canceled int
outputReads int
}
func (b browserModelFixture) Forms(context.Context, string) ([]protocol.ChatForm, error) {
@@ -78,6 +79,20 @@ func (b browserModelFixture) Reply(ctx context.Context, answer protocol.ChatRepl
return nil
}
func (b browserModelFixture) ShellOutput(_ context.Context, q protocol.ChatShellOutputQuery) (protocol.ChatShellOutputPage, error) {
b.fakeChat.mu.Lock()
defer b.fakeChat.mu.Unlock()
for _, shell := range b.fakeChat.shells {
if shell.ID == q.ShellID && q.SessionID == b.fakeChat.sessions[0].ID {
b.forms.mu.Lock()
b.forms.outputReads++
b.forms.mu.Unlock()
return protocol.ChatShellOutputPage{Shell: shell, Output: "Controlled background output", Size: 28}, nil
}
}
return protocol.ChatShellOutputPage{}, fmt.Errorf("unknown task")
}
func (b browserModelFixture) Models(ctx context.Context, id string) ([]protocol.ChatModel, error) {
select {
case <-ctx.Done():
@@ -91,6 +106,12 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
if os.Getenv("PIPELINES_BROWSER_TEST") != "1" {
t.Skip("set PIPELINES_BROWSER_TEST=1 for the scratch Chrome check")
}
for _, backend := range []string{"opencode", "codex"} {
t.Run(backend, func(t *testing.T) { checkChatBrowserLifecycle(t, backend) })
}
}
func checkChatBrowserLifecycle(t *testing.T, backend string) {
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db"))
@@ -100,6 +121,10 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
defer store.Close()
directory := t.TempDir()
session := protocol.ChatSession{ID: "ses_browser", Title: "Scratch chat", Model: "model-a", Directory: directory}
session.Backend = backend
if backend == "codex" {
session.ID = "01900000-0000-7000-8000-000000000001"
}
if err := store.RecordOperatorSession(ctx, session.ID, directory, session.Title, protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
@@ -122,6 +147,13 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
handler := HandlerWithOptions(ctx, Options{Chat: browserModelFixture{fake, forms}}, store)
defer func() { cancel(); WaitRuns(handler) }()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if req.URL.Path == "/fixture-shell" && req.Method == http.MethodPost {
fake.mu.Lock()
fake.shells = []protocol.ChatShell{{ID: "sh_" + backend + "_fixture", Status: protocol.ChatShellRunning, Command: "printf controlled"}}
fake.mu.Unlock()
w.WriteHeader(http.StatusNoContent)
return
}
if req.URL.Path == "/fixture-cancel" && req.Method == http.MethodPost {
forms.mu.Lock()
forms.pending = append(forms.pending, protocol.ChatForm{ID: "form_cancel", SessionID: session.ID, Title: "Cancel this question", Actions: []string{"accept", "cancel"}, Fields: []protocol.ChatFormField{{Key: "answer", Type: "string", Required: true}}})
@@ -157,6 +189,9 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
}))
defer srv.Close()
cmd := exec.CommandContext(ctx, "node", "../../web/test/chat-hardening-probe.mjs", strings.TrimPrefix(srv.URL, "http://"), session.ID)
if captures := os.Getenv("PIPELINES_CAPTURE_DIR"); captures != "" {
cmd.Env = append(os.Environ(), "PIPELINES_CAPTURE_DIR="+filepath.Join(captures, backend))
}
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("chat browser regression: %v\n%s", err, output)
} else {
@@ -168,8 +203,9 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
forms.mu.Lock()
answers := forms.answered
canceled := forms.canceled
outputReads := forms.outputReads
forms.mu.Unlock()
if replies != 2 || answers != 2 || canceled != 1 {
t.Fatalf("browser did not deliver all interactions: approvals=%d answers=%d canceled=%d", replies, answers, canceled)
if replies != 2 || answers != 2 || canceled != 1 || outputReads != 1 {
t.Fatalf("browser did not deliver all interactions: approvals=%d answers=%d canceled=%d output=%d", replies, answers, canceled, outputReads)
}
}
+31
View File
@@ -0,0 +1,31 @@
package server
import (
"testing"
"git.a-wan.net/alan/pipelines/internal/codex"
"git.a-wan.net/alan/pipelines/internal/opencode"
)
// Advertising either real harness must not regress to a broker that can stream
// text but cannot deliver the interactive tools the web needs to finish a turn.
type interactiveWebChat interface {
chatBroker
chatPromptWithFiles
chatAnswerer
chatFormReader
chatModeler
chatShellOutputReader
}
var _ interactiveWebChat = (*opencode.Chat)(nil)
var _ interactiveWebChat = (*codex.Chat)(nil)
func TestAdvertisedChatHarnessesExposeInteractiveWebContract(t *testing.T) {
m := &chatManager{opencode: opencode.NewChat(&opencode.Connection{}), codex: &codex.Chat{}}
for _, backend := range m.Backends() {
if _, ok := m.adapter(backend).(interactiveWebChat); !ok {
t.Errorf("advertised harness %s lacks web tool controls", backend)
}
}
}