From cb57da5dfcc64fd4bdff3c8ad792bb77f972dfc6 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Tue, 20 Oct 2026 20:39:50 +0100 Subject: [PATCH] fix(stage): persist every V2 model generation in tracked sessions --- internal/cli/provider_check_test.go | 4 +- internal/cli/provider_dispatch_test.go | 12 +- internal/server/retry_test.go | 8 +- internal/stage/opencode.go | 40 +++- internal/stage/opencode_test.go | 2 +- internal/stage/tracked_budget_test.go | 6 +- internal/stage/tracked_model_test.go | 248 +++++++++++++++++++++++++ 7 files changed, 307 insertions(+), 13 deletions(-) create mode 100644 internal/stage/tracked_model_test.go diff --git a/internal/cli/provider_check_test.go b/internal/cli/provider_check_test.go index a289cf0c..50cb0a3d 100644 --- a/internal/cli/provider_check_test.go +++ b/internal/cli/provider_check_test.go @@ -19,8 +19,10 @@ func TestProviderCheckExecutesGenerationAndReportsFailures(t *testing.T) { script := `#!/bin/sh case "$*" in *--help*) echo 'api';; - *'/api/session/ses_check/generate'*) ` + map[bool]string{true: `echo '{"_tag":"UnauthorizedError","message":"expired provider credentials","token":"private-token"}'; exit 7`, false: `echo '{"data":{"text":"PIPELINES_PROVIDER_OK"}}'`}[fail] + `;; + *'/api/session/ses_check/prompt'*) ` + map[bool]string{true: `echo '{"_tag":"UnauthorizedError","message":"expired provider credentials","token":"private-token"}'; exit 7`, false: `echo '{}'`}[fail] + `;; + *'/api/session/ses_check/message?'*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"PIPELINES_PROVIDER_OK"}]}]}';; *'/api/session/ses_check/interrupt'*) echo '{}';; + *'/api/session/active'*) echo '{"data":{}}';; *'/api/session'*) echo '{"data":{"id":"ses_check"}}';; *) exit 9;; esac diff --git a/internal/cli/provider_dispatch_test.go b/internal/cli/provider_dispatch_test.go index 3526aa4b..fcef4416 100644 --- a/internal/cli/provider_dispatch_test.go +++ b/internal/cli/provider_dispatch_test.go @@ -119,7 +119,17 @@ func TestProviderCheckNativeProfileKeepsHostGenerator(t *testing.T) { opencode := filepath.Join(root, "opencode") // The native generator runs the host opencode; this fixture echoes a healthy // report so the test needs no provider. - script := "#!/bin/sh\ntouch " + marker + "\ncase \"$*\" in\n *--help*) echo api;;\n *'/api/session/ses_check/generate'*) echo '{\"data\":{\"text\":\"PIPELINES_PROVIDER_OK\"}}';;\n *'/api/session/ses_check/interrupt'*) echo '{}';;\n *'/api/session'*) echo '{\"data\":{\"id\":\"ses_check\"}}';;\n *) exit 9;;\nesac\n" + script := "#!/bin/sh\ntouch " + marker + ` +case "$*" in + *--help*) echo api;; + *'/api/session/ses_check/prompt'*) echo '{}';; + *'/api/session/ses_check/message?'*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"PIPELINES_PROVIDER_OK"}]}]}';; + *'/api/session/ses_check/interrupt'*) echo '{}';; + *'/api/session/active'*) echo '{"data":{}}';; + *'/api/session'*) echo '{"data":{"id":"ses_check"}}';; + *) exit 9;; +esac +` if err := os.WriteFile(opencode, []byte(script), 0700); err != nil { t.Fatal(err) } diff --git a/internal/server/retry_test.go b/internal/server/retry_test.go index 0d8bf387..928d2500 100644 --- a/internal/server/retry_test.go +++ b/internal/server/retry_test.go @@ -67,11 +67,13 @@ case "$3" in n=0; if [ -f %q ]; then n=$(cat %q); fi n=$((n+1)); echo "$n" > %q printf '{"data":{"id":"ses_fresh_%%s"}}' "$n" ;; - /api/session/ses_fresh_1/generate) echo '{"_tag":"InvalidRequestError"}'; exit 1 ;; + /api/session/ses_fresh_1/prompt) echo '{"_tag":"InvalidRequestError"}'; exit 1 ;; /api/session/ses_fresh_1/interrupt) echo '{}' ;; /api/session/ses_fresh_1) echo '{"data":{"id":"ses_fresh_1"}}' ;; /api/session/ses_fresh_1/message*) echo '{"data":[{"error":{"message":"provider rejected generation","token":"private-token"}}]}' ;; - /api/session/ses_fresh_2/generate) printf '%%s\n' '{"data":{"text":"# outcome\nsuccess\n"}}' ;; + /api/session/ses_fresh_2/prompt) echo '{}' ;; + /api/session/ses_fresh_2/message*) printf '%%s\n' '{"data":[{"finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess\n"}]}]}' ;; + /api/session/ses_fresh_2) echo '{"data":{"id":"ses_fresh_2"}}' ;; *) exit 99 ;; esac `, log, count, count, count) @@ -130,7 +132,7 @@ esac if err != nil { t.Fatal(err) } - if strings.Contains(string(calls), "delete") || strings.Count(string(calls), "post /api/session\n") != 2 || strings.Count(string(calls), "ses_fresh_1/generate") != 1 { + if strings.Contains(string(calls), "delete") || strings.Contains(string(calls), "/generate") || strings.Count(string(calls), "post /api/session\n") != 2 || strings.Count(string(calls), "ses_fresh_1/prompt") != 1 { t.Fatal("session deleted or reused", string(calls)) } for _, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") { diff --git a/internal/stage/opencode.go b/internal/stage/opencode.go index d58b0bf1..b375480d 100644 --- a/internal/stage/opencode.go +++ b/internal/stage/opencode.go @@ -266,6 +266,7 @@ func (c OpenCodeClient) sessionActive(ctx context.Context, sessionID string) (bo } func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) { + c.FormatAudit = "" if c.Operations != nil { c.Operations.Provider, c.Operations.Model = config.Provider, config.Model } @@ -287,7 +288,11 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp } } // Fail before creating a session if the generation payload is too large. - body := generationBody("prompt", prompt, config, c.Backend) + promptKey := "prompt" + if c.Backend == "v2" { + promptKey = "text" + } + body := generationBody(promptKey, prompt, config, c.Backend) encoded, err := json.Marshal(body) if err != nil { return "", err @@ -315,13 +320,28 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp return "", fmt.Errorf("OpenCode returned an invalid session ID") } var watch *budgetWatch + promptStarted := false defer func() { watch.stopWatch(ctx, &err) defer watch.finish(ctx, &err) cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() if err != nil { - if stopErr := c.InterruptAndConfirm(cleanup, session.Data.ID); stopErr != nil { + stopErr := c.InterruptAndConfirm(cleanup, session.Data.ID) + // Read after settlement so a late assistant turn remains available as + // failure evidence. Never replay a prompt to reconstruct missing turns. + if promptStarted && c.Backend == "v2" { + readCtx, readCancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second) + data, readErr := c.call(readCtx, "get", "/api/session/"+session.Data.ID+finalAssistantReadback, nil) + readCancel() + if readErr == nil && len(data) > 0 { + c.FormatAudit = RedactEvidence(string(data)) + } + if watch == nil { + c.settleV2Usage(ctx, config.Budget, session.Data.ID) + } + } + if stopErr != nil { err = fmt.Errorf("retained OpenCode session %s; interrupt failed: %w", session.Data.ID, errors.Join(err, stopErr, ErrStageCleanup)) return } @@ -351,10 +371,12 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp if err != nil { return "", err } - // V2 /generate returns text only and does not persist messages or update - // session usage. Budgeted calls must use tracked, deny-all prompting instead. - if c.Backend == "v2" && config.Budget.Enabled() { - if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", generationBody("text", prompt, config, c.Backend)); err != nil { + // V2 /generate does not persist messages or update session usage. Every V2 + // model call uses tracked prompting under the session's deny-all tool policy, + // including unbudgeted text-only stages, so its exact native turns survive. + if c.Backend == "v2" { + promptStarted = true + if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", body); err != nil { return "", err } if err = c.waitForSessionIdle(ctx, session.Data.ID); err != nil { @@ -368,6 +390,12 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp if err != nil { return "", err } + if !config.Budget.Enabled() { + // As with AgentClient, missing usage is not a generation failure when + // no limit is enabled (and a nil ledger needs no aggregate read). + c.settleV2Usage(ctx, config.Budget, session.Data.ID) + return text, nil + } // A completed tracked response must have accounted tokens; session-zero // placeholders are not proof of a free, zero-token provider request. usageCtx, usageCancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second) diff --git a/internal/stage/opencode_test.go b/internal/stage/opencode_test.go index 97b2328c..d4ee7c97 100644 --- a/internal/stage/opencode_test.go +++ b/internal/stage/opencode_test.go @@ -10,7 +10,7 @@ import ( func TestOpenCodeClientSelectsModelAndRetainsSession(t *testing.T) { var calls []string - client := OpenCodeClient{Directory: "/work/repo", command: func(ctx context.Context, directory string, args ...string) ([]byte, error) { + client := OpenCodeClient{Backend: "v1", Directory: "/work/repo", command: func(ctx context.Context, directory string, args ...string) ([]byte, error) { if directory != "/work/repo" || args[0] != "api" { t.Fatal("incorrect OpenCode invocation context") } diff --git a/internal/stage/tracked_budget_test.go b/internal/stage/tracked_budget_test.go index 809fa7f0..abe056c6 100644 --- a/internal/stage/tracked_budget_test.go +++ b/internal/stage/tracked_budget_test.go @@ -80,7 +80,11 @@ func TestBudgetedV2UsesTrackedDenyAllPrompt(t *testing.T) { } else if err != nil || !strings.Contains(fmt.Sprint(result.Output["markdown"]), "done") { t.Fatal(result, err) } - if !prompted || messages != 1 || !strings.Contains(client.FormatAudit, "done") { + wantMessages := 1 + if test.want != nil { + wantMessages++ // bounded late-evidence read after interrupt/settlement + } + if !prompted || messages != wantMessages || !strings.Contains(client.FormatAudit, "done") { t.Fatal("tracked evidence missing", prompted, messages, client.FormatAudit) } if budget.Snapshot().UsedTokens != test.tokens { diff --git a/internal/stage/tracked_model_test.go b/internal/stage/tracked_model_test.go new file mode 100644 index 00000000..5c89e3a7 --- /dev/null +++ b/internal/stage/tracked_model_test.go @@ -0,0 +1,248 @@ +package stage + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" +) + +func TestUnbudgetedV2PersistsTrackedTurnsBeforeReleasingOwnership(t *testing.T) { + for _, usage := range []string{"nil budget", "known", "missing", "malformed", "unavailable"} { + t.Run(usage, func(t *testing.T) { + var budget *Budget + if usage != "nil budget" { + budget = NewBudget(0, 0) + } + var events, turns []string + polls, prompts, usageReads := 0, 0, 0 + owned := false + const prompt = "exact pipeline instructions\nDo not change the definition or tool policy." + const final = "# outcome\nsuccess\n\n# Summary\n\nNative final result." + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := r.URL.RequestURI() + events = append(events, r.Method+" "+path) + switch path { + case "/api/session": + var body struct { + Permissions []map[string]string `json:"permissions"` + Model map[string]string `json:"model"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + } + if !reflect.DeepEqual(body.Permissions, []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}}) || body.Model["providerID"] != "fixture" || body.Model["id"] != "tracked" { + t.Errorf("model/tool policy changed: %+v", body) + } + io.WriteString(w, `{"data":{"id":"ses_native"}}`) + case "/api/session/ses_native/prompt": + prompts++ + var body map[string]string + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body["text"] != prompt || len(body) != 1 || !owned { + t.Errorf("prompt admitted before ownership or changed: %v, %v, owned=%v", body, err, owned) + } + // The tracked API fixture persists native turns; /generate is + // deliberately unsupported, so it cannot fabricate this history. + turns = append(turns, "user: "+body["text"]) + io.WriteString(w, `{}`) + case "/api/session/active": + polls++ + if polls == 1 { + io.WriteString(w, `{"data":{"ses_native":{"type":"running"}}}`) + } else { + turns = append(turns, "assistant: "+final) + io.WriteString(w, `{"data":{"ses_foreign":{"type":"running"}}}`) + } + case "/api/session/ses_native" + finalAssistantReadback: + if polls != 2 || !owned || len(turns) != 2 { + t.Error("final readback preceded exact-session settlement") + } + json.NewEncoder(w).Encode(map[string]any{"data": []any{map[string]any{"finish": "stop", "content": []any{map[string]string{"type": "text", "text": final}}}}}) + case "/api/session/ses_native": + usageReads++ + switch usage { + case "known": + w.Write(usageResponse("ses_native", .001, 3)) + case "missing": + io.WriteString(w, `{"data":{"id":"ses_native"}}`) + case "malformed": + io.WriteString(w, `{"data":`) + case "unavailable": + http.Error(w, "unavailable", http.StatusServiceUnavailable) + default: + t.Error("nil budget should not require usage") + } + default: + t.Errorf("unexpected native operation (replay, generate or interrupt): %s", path) + http.NotFound(w, r) + } + })) + defer server.Close() + client := OpenCodeClient{Backend: "v2", Directory: "/work", sessionPollInterval: time.Millisecond} + client.OnGenerationSession = func(id, backend, reference string, active bool) error { + if id != "ses_native" || backend != "v2" || reference != "" { + t.Fatalf("incorrect ownership: %s %s %s", id, backend, reference) + } + if active { + events = append(events, "own") + } else { + if polls != 2 || len(turns) != 2 || !strings.Contains(client.FormatAudit, "Native final result") { + t.Fatal("ownership released before settled final evidence") + } + events = append(events, "release") + } + owned = active + return nil + } + client.command = func(ctx context.Context, _ string, args ...string) ([]byte, error) { + var body io.Reader + if len(args) > 4 { + body = strings.NewReader(args[4]) + } + req, err := http.NewRequestWithContext(ctx, strings.ToUpper(args[1]), server.URL+args[2], body) + if err != nil { + return nil, err + } + res, err := server.Client().Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return nil, &openCodeError{status: res.StatusCode} + } + return io.ReadAll(res.Body) + } + text, err := client.Generate(context.Background(), ModelConfig{Provider: "fixture", Model: "tracked", Budget: budget}, prompt) + if err != nil || text != final || owned || prompts != 1 || !reflect.DeepEqual(turns, []string{"user: " + prompt, "assistant: " + final}) { + t.Fatalf("tracked generation: text=%q err=%v owned=%v prompts=%d turns=%v", text, err, owned, prompts, turns) + } + want := []string{"POST /api/session", "own", "POST /api/session/ses_native/prompt", "GET /api/session/active", "GET /api/session/active", "GET /api/session/ses_native" + finalAssistantReadback} + if budget != nil { + want = append(want, "GET /api/session/ses_native") + } + want = append(want, "release") + if !reflect.DeepEqual(events, want) || usageReads != map[bool]int{false: 0, true: 1}[budget != nil] { + t.Fatalf("lifecycle = %v; want %v; usage reads=%d", events, want, usageReads) + } + if usage == "known" && budget.Snapshot().UsedTokens != 3 { + t.Fatal("unbudgeted usage not retained", budget.Snapshot()) + } + }) + } +} + +func TestTrackedModelFailureSettlesAndRetainsLateEvidenceWithoutReplay(t *testing.T) { + for _, failure := range []string{"prompt", "wait", "readback", "cleanup", "canceled"} { + t.Run(failure, func(t *testing.T) { + var events []string + prompts, reads := 0, 0 + interrupted, owned := false, false + cause := errors.New("native transport failure") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if failure == "canceled" { + cause = context.Canceled + } + client := OpenCodeClient{Backend: "v2", sessionPollInterval: time.Millisecond} + client.OnGenerationSession = func(_ string, _, _ string, active bool) error { + owned = active + if active { + events = append(events, "own") + } else { + if !interrupted || reads == 0 { + t.Fatal("released before settlement and late evidence read") + } + events = append(events, "release") + } + return nil + } + client.command = func(ctx context.Context, _ string, args ...string) ([]byte, error) { + path := args[2] + events = append(events, path) + switch path { + case "/api/session": + return []byte(`{"data":{"id":"ses_failed"}}`), nil + case "/api/session/ses_failed/prompt": + prompts++ + if !owned { + t.Fatal("prompt before ownership") + } + if failure == "prompt" || failure == "canceled" { + if failure == "canceled" { + cancel() + } + return nil, cause + } + return []byte(`{}`), nil + case "/api/session/active": + if !interrupted && (failure == "wait" || failure == "cleanup") || interrupted && failure == "cleanup" { + return nil, cause + } + return []byte(`{"data":{}}`), nil + case "/api/session/ses_failed/interrupt": + if ctx.Err() != nil { + t.Fatal("cleanup inherited canceled context") + } + interrupted = true + return []byte(`{}`), nil + case "/api/session/ses_failed" + finalAssistantReadback: + reads++ + if !interrupted { + return nil, cause + } + return []byte(`{"data":[{"finish":"stop","content":[{"type":"text","text":"late failure evidence"}]}]}`), nil + default: + t.Fatalf("unexpected call: %s", path) + return nil, nil + } + } + text, err := client.Generate(ctx, ModelConfig{}, "unchanged prompt") + if !errors.Is(err, cause) || text != "" || prompts != 1 || !strings.Contains(client.FormatAudit, "late failure evidence") || owned != (failure == "cleanup") { + t.Fatalf("failure lost evidence/ownership or replayed: text=%q err=%v prompts=%d owned=%v audit=%s events=%v", text, err, prompts, owned, client.FormatAudit, events) + } + if failure == "cleanup" && !errors.Is(err, ErrStageCleanup) { + t.Fatal("unconfirmed cleanup must retain ownership", err) + } + }) + } +} + +func TestTrackedModelOwnershipFailureDoesNotAdmitPrompt(t *testing.T) { + cause := errors.New("ownership persistence failed") + var events []string + client := OpenCodeClient{Backend: "v2", FormatAudit: "old unrelated evidence"} + client.OnGenerationSession = func(_ string, _, _ string, active bool) error { + if active { + events = append(events, "own") + return cause + } + events = append(events, "release") + return nil + } + client.command = func(_ context.Context, _ string, args ...string) ([]byte, error) { + events = append(events, args[2]) + switch args[2] { + case "/api/session": + return []byte(`{"data":{"id":"ses_unowned"}}`), nil + case "/api/session/ses_unowned/interrupt": + return []byte(`{}`), nil + case "/api/session/active": + return []byte(`{"data":{}}`), nil + default: + t.Fatalf("admitted work without recorded ownership: %v", args) + return nil, nil + } + } + text, err := client.Generate(context.Background(), ModelConfig{}, "must not run") + want := []string{"/api/session", "own", "/api/session/ses_unowned/interrupt", "/api/session/active", "release"} + if !errors.Is(err, cause) || text != "" || client.FormatAudit != "" || !reflect.DeepEqual(events, want) { + t.Fatalf("ownership failure: text=%q err=%v audit=%q events=%v", text, err, client.FormatAudit, events) + } +}