test(chat): require interactive tool parity across advertised harnesses
This commit is contained in:
@@ -32,3 +32,29 @@ not provide durable callback recovery; a disconnected transport is reported as
|
||||
an error rather than claiming old requests are still actionable.
|
||||
|
||||
Claude is not advertised as a backend until its sidecar and login flow are wired.
|
||||
|
||||
## Web interaction contract
|
||||
|
||||
Both advertised chat adapters (OpenCode V2 and Codex) implement the same server
|
||||
contract for uploads, models, pending approvals, questions, form replies,
|
||||
interrupts, command readback and bounded output. Compile-time assertions and a
|
||||
browser regression run against both backend identities protect this contract;
|
||||
support is not satisfied by text streaming alone. Pi's pipeline executor is not
|
||||
advertised as a Pi web-chat adapter, and an unavailable Claude adapter is not
|
||||
presented as a working choice.
|
||||
|
||||
Codex uploads use native `localImage` for verified supported images. Other files
|
||||
are retained intact with an explicit server-local reference; the model's own
|
||||
tools inspect them rather than pretending their contents were already loaded.
|
||||
The saved next-turn model is also respected with uploads. Model sandbox and
|
||||
approval policy remain in force.
|
||||
|
||||
Codex's paged history retains tool and reasoning items, including structured MCP
|
||||
errors and Code Mode outputs, instead of discarding everything except text.
|
||||
Unknown item kinds remain inspectable. Tool detail is explicitly truncated at
|
||||
64 KiB, not silently represented as complete. Running commands and their output
|
||||
use real command items plus read-only thread history; no unsupported terminal
|
||||
endpoint, process control, resumed turn or experimental flag is invented.
|
||||
Output reads are bounded and verify thread ownership. The installed Codex
|
||||
version exposes no API for controlling an agent-owned terminal through the
|
||||
separate, client-owned `command/exec` process IDs; those IDs are never conflated.
|
||||
|
||||
@@ -23,10 +23,11 @@ type browserModelFixture struct {
|
||||
}
|
||||
|
||||
type browserForms struct {
|
||||
mu sync.Mutex
|
||||
pending []protocol.ChatForm
|
||||
answered int
|
||||
canceled int
|
||||
mu sync.Mutex
|
||||
pending []protocol.ChatForm
|
||||
answered int
|
||||
canceled int
|
||||
outputReads int
|
||||
}
|
||||
|
||||
func (b browserModelFixture) Forms(context.Context, string) ([]protocol.ChatForm, error) {
|
||||
@@ -78,6 +79,20 @@ func (b browserModelFixture) Reply(ctx context.Context, answer protocol.ChatRepl
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b browserModelFixture) ShellOutput(_ context.Context, q protocol.ChatShellOutputQuery) (protocol.ChatShellOutputPage, error) {
|
||||
b.fakeChat.mu.Lock()
|
||||
defer b.fakeChat.mu.Unlock()
|
||||
for _, shell := range b.fakeChat.shells {
|
||||
if shell.ID == q.ShellID && q.SessionID == b.fakeChat.sessions[0].ID {
|
||||
b.forms.mu.Lock()
|
||||
b.forms.outputReads++
|
||||
b.forms.mu.Unlock()
|
||||
return protocol.ChatShellOutputPage{Shell: shell, Output: "Controlled background output", Size: 28}, nil
|
||||
}
|
||||
}
|
||||
return protocol.ChatShellOutputPage{}, fmt.Errorf("unknown task")
|
||||
}
|
||||
|
||||
func (b browserModelFixture) Models(ctx context.Context, id string) ([]protocol.ChatModel, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -91,6 +106,12 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
|
||||
if os.Getenv("PIPELINES_BROWSER_TEST") != "1" {
|
||||
t.Skip("set PIPELINES_BROWSER_TEST=1 for the scratch Chrome check")
|
||||
}
|
||||
for _, backend := range []string{"opencode", "codex"} {
|
||||
t.Run(backend, func(t *testing.T) { checkChatBrowserLifecycle(t, backend) })
|
||||
}
|
||||
}
|
||||
|
||||
func checkChatBrowserLifecycle(t *testing.T, backend string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db"))
|
||||
@@ -100,6 +121,10 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
|
||||
defer store.Close()
|
||||
directory := t.TempDir()
|
||||
session := protocol.ChatSession{ID: "ses_browser", Title: "Scratch chat", Model: "model-a", Directory: directory}
|
||||
session.Backend = backend
|
||||
if backend == "codex" {
|
||||
session.ID = "01900000-0000-7000-8000-000000000001"
|
||||
}
|
||||
if err := store.RecordOperatorSession(ctx, session.ID, directory, session.Title, protocol.ChatSessionTagOperator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -122,6 +147,13 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
|
||||
handler := HandlerWithOptions(ctx, Options{Chat: browserModelFixture{fake, forms}}, store)
|
||||
defer func() { cancel(); WaitRuns(handler) }()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
if req.URL.Path == "/fixture-shell" && req.Method == http.MethodPost {
|
||||
fake.mu.Lock()
|
||||
fake.shells = []protocol.ChatShell{{ID: "sh_" + backend + "_fixture", Status: protocol.ChatShellRunning, Command: "printf controlled"}}
|
||||
fake.mu.Unlock()
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
if req.URL.Path == "/fixture-cancel" && req.Method == http.MethodPost {
|
||||
forms.mu.Lock()
|
||||
forms.pending = append(forms.pending, protocol.ChatForm{ID: "form_cancel", SessionID: session.ID, Title: "Cancel this question", Actions: []string{"accept", "cancel"}, Fields: []protocol.ChatFormField{{Key: "answer", Type: "string", Required: true}}})
|
||||
@@ -157,6 +189,9 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
|
||||
}))
|
||||
defer srv.Close()
|
||||
cmd := exec.CommandContext(ctx, "node", "../../web/test/chat-hardening-probe.mjs", strings.TrimPrefix(srv.URL, "http://"), session.ID)
|
||||
if captures := os.Getenv("PIPELINES_CAPTURE_DIR"); captures != "" {
|
||||
cmd.Env = append(os.Environ(), "PIPELINES_CAPTURE_DIR="+filepath.Join(captures, backend))
|
||||
}
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("chat browser regression: %v\n%s", err, output)
|
||||
} else {
|
||||
@@ -168,8 +203,9 @@ func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
|
||||
forms.mu.Lock()
|
||||
answers := forms.answered
|
||||
canceled := forms.canceled
|
||||
outputReads := forms.outputReads
|
||||
forms.mu.Unlock()
|
||||
if replies != 2 || answers != 2 || canceled != 1 {
|
||||
t.Fatalf("browser did not deliver all interactions: approvals=%d answers=%d canceled=%d", replies, answers, canceled)
|
||||
if replies != 2 || answers != 2 || canceled != 1 || outputReads != 1 {
|
||||
t.Fatalf("browser did not deliver all interactions: approvals=%d answers=%d canceled=%d output=%d", replies, answers, canceled, outputReads)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/codex"
|
||||
"git.a-wan.net/alan/pipelines/internal/opencode"
|
||||
)
|
||||
|
||||
// Advertising either real harness must not regress to a broker that can stream
|
||||
// text but cannot deliver the interactive tools the web needs to finish a turn.
|
||||
type interactiveWebChat interface {
|
||||
chatBroker
|
||||
chatPromptWithFiles
|
||||
chatAnswerer
|
||||
chatFormReader
|
||||
chatModeler
|
||||
chatShellOutputReader
|
||||
}
|
||||
|
||||
var _ interactiveWebChat = (*opencode.Chat)(nil)
|
||||
var _ interactiveWebChat = (*codex.Chat)(nil)
|
||||
|
||||
func TestAdvertisedChatHarnessesExposeInteractiveWebContract(t *testing.T) {
|
||||
m := &chatManager{opencode: opencode.NewChat(&opencode.Connection{}), codex: &codex.Chat{}}
|
||||
for _, backend := range m.Backends() {
|
||||
if _, ok := m.adapter(backend).(interactiveWebChat); !ok {
|
||||
t.Errorf("advertised harness %s lacks web tool controls", backend)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -254,6 +254,16 @@ to it, but the server never lists it and a reload drops it.
|
||||
|
||||
## Web interaction recovery (2026-10-20)
|
||||
|
||||
Cross-harness contract follow-up: the same browser interaction regression now
|
||||
runs with OpenCode and Codex identities. Codex additionally accepts prepared
|
||||
uploads without losing the selected model, retains all native tool/history
|
||||
items (including structured MCP failures and Code Mode outputs), pages history
|
||||
with opaque cursors, and exposes running-command/output readback through the
|
||||
shared background-task UI. Command ownership/output bounds and exact installed
|
||||
protocol shapes are tested. Read-only live Codex validation passed history,
|
||||
command listing and model listing without starting/resuming a turn or changing
|
||||
settings. No real Codex command or permission grant was used for testing.
|
||||
|
||||
Typed-tool follow-up: number/integer/boolean/default/conditional/secret and
|
||||
external-link fields use native controls, preserve JSON types and validate before
|
||||
sending. `typed-question-controls.png` and the scratch browser regression cover
|
||||
|
||||
@@ -146,6 +146,12 @@ try {
|
||||
await page.waitForSelector('[data-form-id=form_cancel]');
|
||||
await page.evaluate("[...document.querySelectorAll('[data-form-id=form_cancel] button')].find(button=>button.textContent==='Cancel question').click()");
|
||||
await eventually(()=>page.evaluate("!document.querySelector('[data-form-id=form_cancel]')"));
|
||||
assert.equal((await fetch(`http://${target}/fixture-shell`,{method:'POST'})).status,204);
|
||||
await page.goto(`http://127.0.0.1:${port}/chat/${sessionId}`);
|
||||
await page.waitForSelector('.chat-bg-task');
|
||||
await page.click('.chat-bg-task');
|
||||
await eventually(()=>page.evaluate("document.querySelector('.chat-bg-task-output')?.textContent==='Controlled background output'"));
|
||||
if(process.env.PIPELINES_CAPTURE_DIR) await page.screenshot(join(process.env.PIPELINES_CAPTURE_DIR,'harness-background-output.png'));
|
||||
assert.deepEqual(await page.evaluate("window.__errors"), []);
|
||||
console.log("Model dialog, stable activity mark, keyboard/caret, Back restoration, send-to-bottom and all recovered approvals/questions passed.");
|
||||
} finally {
|
||||
|
||||
Reference in New Issue
Block a user