fix(stage): persist every V2 model generation in tracked sessions

This commit is contained in:
2026-10-20 20:39:50 +01:00
parent 1b8d0f2a75
commit cb57da5dfc
7 changed files with 307 additions and 13 deletions
+3 -1
View File
@@ -19,8 +19,10 @@ func TestProviderCheckExecutesGenerationAndReportsFailures(t *testing.T) {
script := `#!/bin/sh
case "$*" in
*--help*) echo 'api';;
*'/api/session/ses_check/generate'*) ` + map[bool]string{true: `echo '{"_tag":"UnauthorizedError","message":"expired provider credentials","token":"private-token"}'; exit 7`, false: `echo '{"data":{"text":"PIPELINES_PROVIDER_OK"}}'`}[fail] + `;;
*'/api/session/ses_check/prompt'*) ` + map[bool]string{true: `echo '{"_tag":"UnauthorizedError","message":"expired provider credentials","token":"private-token"}'; exit 7`, false: `echo '{}'`}[fail] + `;;
*'/api/session/ses_check/message?'*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"PIPELINES_PROVIDER_OK"}]}]}';;
*'/api/session/ses_check/interrupt'*) echo '{}';;
*'/api/session/active'*) echo '{"data":{}}';;
*'/api/session'*) echo '{"data":{"id":"ses_check"}}';;
*) exit 9;;
esac
+11 -1
View File
@@ -119,7 +119,17 @@ func TestProviderCheckNativeProfileKeepsHostGenerator(t *testing.T) {
opencode := filepath.Join(root, "opencode")
// The native generator runs the host opencode; this fixture echoes a healthy
// report so the test needs no provider.
script := "#!/bin/sh\ntouch " + marker + "\ncase \"$*\" in\n *--help*) echo api;;\n *'/api/session/ses_check/generate'*) echo '{\"data\":{\"text\":\"PIPELINES_PROVIDER_OK\"}}';;\n *'/api/session/ses_check/interrupt'*) echo '{}';;\n *'/api/session'*) echo '{\"data\":{\"id\":\"ses_check\"}}';;\n *) exit 9;;\nesac\n"
script := "#!/bin/sh\ntouch " + marker + `
case "$*" in
*--help*) echo api;;
*'/api/session/ses_check/prompt'*) echo '{}';;
*'/api/session/ses_check/message?'*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"PIPELINES_PROVIDER_OK"}]}]}';;
*'/api/session/ses_check/interrupt'*) echo '{}';;
*'/api/session/active'*) echo '{"data":{}}';;
*'/api/session'*) echo '{"data":{"id":"ses_check"}}';;
*) exit 9;;
esac
`
if err := os.WriteFile(opencode, []byte(script), 0700); err != nil {
t.Fatal(err)
}
+5 -3
View File
@@ -67,11 +67,13 @@ case "$3" in
n=0; if [ -f %q ]; then n=$(cat %q); fi
n=$((n+1)); echo "$n" > %q
printf '{"data":{"id":"ses_fresh_%%s"}}' "$n" ;;
/api/session/ses_fresh_1/generate) echo '{"_tag":"InvalidRequestError"}'; exit 1 ;;
/api/session/ses_fresh_1/prompt) echo '{"_tag":"InvalidRequestError"}'; exit 1 ;;
/api/session/ses_fresh_1/interrupt) echo '{}' ;;
/api/session/ses_fresh_1) echo '{"data":{"id":"ses_fresh_1"}}' ;;
/api/session/ses_fresh_1/message*) echo '{"data":[{"error":{"message":"provider rejected generation","token":"private-token"}}]}' ;;
/api/session/ses_fresh_2/generate) printf '%%s\n' '{"data":{"text":"# outcome\nsuccess\n"}}' ;;
/api/session/ses_fresh_2/prompt) echo '{}' ;;
/api/session/ses_fresh_2/message*) printf '%%s\n' '{"data":[{"finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess\n"}]}]}' ;;
/api/session/ses_fresh_2) echo '{"data":{"id":"ses_fresh_2"}}' ;;
*) exit 99 ;;
esac
`, log, count, count, count)
@@ -130,7 +132,7 @@ esac
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(calls), "delete") || strings.Count(string(calls), "post /api/session\n") != 2 || strings.Count(string(calls), "ses_fresh_1/generate") != 1 {
if strings.Contains(string(calls), "delete") || strings.Contains(string(calls), "/generate") || strings.Count(string(calls), "post /api/session\n") != 2 || strings.Count(string(calls), "ses_fresh_1/prompt") != 1 {
t.Fatal("session deleted or reused", string(calls))
}
for _, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") {
+34 -6
View File
@@ -266,6 +266,7 @@ func (c OpenCodeClient) sessionActive(ctx context.Context, sessionID string) (bo
}
func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) {
c.FormatAudit = ""
if c.Operations != nil {
c.Operations.Provider, c.Operations.Model = config.Provider, config.Model
}
@@ -287,7 +288,11 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
}
}
// Fail before creating a session if the generation payload is too large.
body := generationBody("prompt", prompt, config, c.Backend)
promptKey := "prompt"
if c.Backend == "v2" {
promptKey = "text"
}
body := generationBody(promptKey, prompt, config, c.Backend)
encoded, err := json.Marshal(body)
if err != nil {
return "", err
@@ -315,13 +320,28 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
return "", fmt.Errorf("OpenCode returned an invalid session ID")
}
var watch *budgetWatch
promptStarted := false
defer func() {
watch.stopWatch(ctx, &err)
defer watch.finish(ctx, &err)
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if err != nil {
if stopErr := c.InterruptAndConfirm(cleanup, session.Data.ID); stopErr != nil {
stopErr := c.InterruptAndConfirm(cleanup, session.Data.ID)
// Read after settlement so a late assistant turn remains available as
// failure evidence. Never replay a prompt to reconstruct missing turns.
if promptStarted && c.Backend == "v2" {
readCtx, readCancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
data, readErr := c.call(readCtx, "get", "/api/session/"+session.Data.ID+finalAssistantReadback, nil)
readCancel()
if readErr == nil && len(data) > 0 {
c.FormatAudit = RedactEvidence(string(data))
}
if watch == nil {
c.settleV2Usage(ctx, config.Budget, session.Data.ID)
}
}
if stopErr != nil {
err = fmt.Errorf("retained OpenCode session %s; interrupt failed: %w", session.Data.ID, errors.Join(err, stopErr, ErrStageCleanup))
return
}
@@ -351,10 +371,12 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
if err != nil {
return "", err
}
// V2 /generate returns text only and does not persist messages or update
// session usage. Budgeted calls must use tracked, deny-all prompting instead.
if c.Backend == "v2" && config.Budget.Enabled() {
if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", generationBody("text", prompt, config, c.Backend)); err != nil {
// V2 /generate does not persist messages or update session usage. Every V2
// model call uses tracked prompting under the session's deny-all tool policy,
// including unbudgeted text-only stages, so its exact native turns survive.
if c.Backend == "v2" {
promptStarted = true
if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", body); err != nil {
return "", err
}
if err = c.waitForSessionIdle(ctx, session.Data.ID); err != nil {
@@ -368,6 +390,12 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
if err != nil {
return "", err
}
if !config.Budget.Enabled() {
// As with AgentClient, missing usage is not a generation failure when
// no limit is enabled (and a nil ledger needs no aggregate read).
c.settleV2Usage(ctx, config.Budget, session.Data.ID)
return text, nil
}
// A completed tracked response must have accounted tokens; session-zero
// placeholders are not proof of a free, zero-token provider request.
usageCtx, usageCancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
+1 -1
View File
@@ -10,7 +10,7 @@ import (
func TestOpenCodeClientSelectsModelAndRetainsSession(t *testing.T) {
var calls []string
client := OpenCodeClient{Directory: "/work/repo", command: func(ctx context.Context, directory string, args ...string) ([]byte, error) {
client := OpenCodeClient{Backend: "v1", Directory: "/work/repo", command: func(ctx context.Context, directory string, args ...string) ([]byte, error) {
if directory != "/work/repo" || args[0] != "api" {
t.Fatal("incorrect OpenCode invocation context")
}
+5 -1
View File
@@ -80,7 +80,11 @@ func TestBudgetedV2UsesTrackedDenyAllPrompt(t *testing.T) {
} else if err != nil || !strings.Contains(fmt.Sprint(result.Output["markdown"]), "done") {
t.Fatal(result, err)
}
if !prompted || messages != 1 || !strings.Contains(client.FormatAudit, "done") {
wantMessages := 1
if test.want != nil {
wantMessages++ // bounded late-evidence read after interrupt/settlement
}
if !prompted || messages != wantMessages || !strings.Contains(client.FormatAudit, "done") {
t.Fatal("tracked evidence missing", prompted, messages, client.FormatAudit)
}
if budget.Snapshot().UsedTokens != test.tokens {
+248
View File
@@ -0,0 +1,248 @@
package stage
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"testing"
"time"
)
func TestUnbudgetedV2PersistsTrackedTurnsBeforeReleasingOwnership(t *testing.T) {
for _, usage := range []string{"nil budget", "known", "missing", "malformed", "unavailable"} {
t.Run(usage, func(t *testing.T) {
var budget *Budget
if usage != "nil budget" {
budget = NewBudget(0, 0)
}
var events, turns []string
polls, prompts, usageReads := 0, 0, 0
owned := false
const prompt = "exact pipeline instructions\nDo not change the definition or tool policy."
const final = "# outcome\nsuccess\n\n# Summary\n\nNative final result."
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
path := r.URL.RequestURI()
events = append(events, r.Method+" "+path)
switch path {
case "/api/session":
var body struct {
Permissions []map[string]string `json:"permissions"`
Model map[string]string `json:"model"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(body.Permissions, []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}}) || body.Model["providerID"] != "fixture" || body.Model["id"] != "tracked" {
t.Errorf("model/tool policy changed: %+v", body)
}
io.WriteString(w, `{"data":{"id":"ses_native"}}`)
case "/api/session/ses_native/prompt":
prompts++
var body map[string]string
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body["text"] != prompt || len(body) != 1 || !owned {
t.Errorf("prompt admitted before ownership or changed: %v, %v, owned=%v", body, err, owned)
}
// The tracked API fixture persists native turns; /generate is
// deliberately unsupported, so it cannot fabricate this history.
turns = append(turns, "user: "+body["text"])
io.WriteString(w, `{}`)
case "/api/session/active":
polls++
if polls == 1 {
io.WriteString(w, `{"data":{"ses_native":{"type":"running"}}}`)
} else {
turns = append(turns, "assistant: "+final)
io.WriteString(w, `{"data":{"ses_foreign":{"type":"running"}}}`)
}
case "/api/session/ses_native" + finalAssistantReadback:
if polls != 2 || !owned || len(turns) != 2 {
t.Error("final readback preceded exact-session settlement")
}
json.NewEncoder(w).Encode(map[string]any{"data": []any{map[string]any{"finish": "stop", "content": []any{map[string]string{"type": "text", "text": final}}}}})
case "/api/session/ses_native":
usageReads++
switch usage {
case "known":
w.Write(usageResponse("ses_native", .001, 3))
case "missing":
io.WriteString(w, `{"data":{"id":"ses_native"}}`)
case "malformed":
io.WriteString(w, `{"data":`)
case "unavailable":
http.Error(w, "unavailable", http.StatusServiceUnavailable)
default:
t.Error("nil budget should not require usage")
}
default:
t.Errorf("unexpected native operation (replay, generate or interrupt): %s", path)
http.NotFound(w, r)
}
}))
defer server.Close()
client := OpenCodeClient{Backend: "v2", Directory: "/work", sessionPollInterval: time.Millisecond}
client.OnGenerationSession = func(id, backend, reference string, active bool) error {
if id != "ses_native" || backend != "v2" || reference != "" {
t.Fatalf("incorrect ownership: %s %s %s", id, backend, reference)
}
if active {
events = append(events, "own")
} else {
if polls != 2 || len(turns) != 2 || !strings.Contains(client.FormatAudit, "Native final result") {
t.Fatal("ownership released before settled final evidence")
}
events = append(events, "release")
}
owned = active
return nil
}
client.command = func(ctx context.Context, _ string, args ...string) ([]byte, error) {
var body io.Reader
if len(args) > 4 {
body = strings.NewReader(args[4])
}
req, err := http.NewRequestWithContext(ctx, strings.ToUpper(args[1]), server.URL+args[2], body)
if err != nil {
return nil, err
}
res, err := server.Client().Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, &openCodeError{status: res.StatusCode}
}
return io.ReadAll(res.Body)
}
text, err := client.Generate(context.Background(), ModelConfig{Provider: "fixture", Model: "tracked", Budget: budget}, prompt)
if err != nil || text != final || owned || prompts != 1 || !reflect.DeepEqual(turns, []string{"user: " + prompt, "assistant: " + final}) {
t.Fatalf("tracked generation: text=%q err=%v owned=%v prompts=%d turns=%v", text, err, owned, prompts, turns)
}
want := []string{"POST /api/session", "own", "POST /api/session/ses_native/prompt", "GET /api/session/active", "GET /api/session/active", "GET /api/session/ses_native" + finalAssistantReadback}
if budget != nil {
want = append(want, "GET /api/session/ses_native")
}
want = append(want, "release")
if !reflect.DeepEqual(events, want) || usageReads != map[bool]int{false: 0, true: 1}[budget != nil] {
t.Fatalf("lifecycle = %v; want %v; usage reads=%d", events, want, usageReads)
}
if usage == "known" && budget.Snapshot().UsedTokens != 3 {
t.Fatal("unbudgeted usage not retained", budget.Snapshot())
}
})
}
}
func TestTrackedModelFailureSettlesAndRetainsLateEvidenceWithoutReplay(t *testing.T) {
for _, failure := range []string{"prompt", "wait", "readback", "cleanup", "canceled"} {
t.Run(failure, func(t *testing.T) {
var events []string
prompts, reads := 0, 0
interrupted, owned := false, false
cause := errors.New("native transport failure")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if failure == "canceled" {
cause = context.Canceled
}
client := OpenCodeClient{Backend: "v2", sessionPollInterval: time.Millisecond}
client.OnGenerationSession = func(_ string, _, _ string, active bool) error {
owned = active
if active {
events = append(events, "own")
} else {
if !interrupted || reads == 0 {
t.Fatal("released before settlement and late evidence read")
}
events = append(events, "release")
}
return nil
}
client.command = func(ctx context.Context, _ string, args ...string) ([]byte, error) {
path := args[2]
events = append(events, path)
switch path {
case "/api/session":
return []byte(`{"data":{"id":"ses_failed"}}`), nil
case "/api/session/ses_failed/prompt":
prompts++
if !owned {
t.Fatal("prompt before ownership")
}
if failure == "prompt" || failure == "canceled" {
if failure == "canceled" {
cancel()
}
return nil, cause
}
return []byte(`{}`), nil
case "/api/session/active":
if !interrupted && (failure == "wait" || failure == "cleanup") || interrupted && failure == "cleanup" {
return nil, cause
}
return []byte(`{"data":{}}`), nil
case "/api/session/ses_failed/interrupt":
if ctx.Err() != nil {
t.Fatal("cleanup inherited canceled context")
}
interrupted = true
return []byte(`{}`), nil
case "/api/session/ses_failed" + finalAssistantReadback:
reads++
if !interrupted {
return nil, cause
}
return []byte(`{"data":[{"finish":"stop","content":[{"type":"text","text":"late failure evidence"}]}]}`), nil
default:
t.Fatalf("unexpected call: %s", path)
return nil, nil
}
}
text, err := client.Generate(ctx, ModelConfig{}, "unchanged prompt")
if !errors.Is(err, cause) || text != "" || prompts != 1 || !strings.Contains(client.FormatAudit, "late failure evidence") || owned != (failure == "cleanup") {
t.Fatalf("failure lost evidence/ownership or replayed: text=%q err=%v prompts=%d owned=%v audit=%s events=%v", text, err, prompts, owned, client.FormatAudit, events)
}
if failure == "cleanup" && !errors.Is(err, ErrStageCleanup) {
t.Fatal("unconfirmed cleanup must retain ownership", err)
}
})
}
}
func TestTrackedModelOwnershipFailureDoesNotAdmitPrompt(t *testing.T) {
cause := errors.New("ownership persistence failed")
var events []string
client := OpenCodeClient{Backend: "v2", FormatAudit: "old unrelated evidence"}
client.OnGenerationSession = func(_ string, _, _ string, active bool) error {
if active {
events = append(events, "own")
return cause
}
events = append(events, "release")
return nil
}
client.command = func(_ context.Context, _ string, args ...string) ([]byte, error) {
events = append(events, args[2])
switch args[2] {
case "/api/session":
return []byte(`{"data":{"id":"ses_unowned"}}`), nil
case "/api/session/ses_unowned/interrupt":
return []byte(`{}`), nil
case "/api/session/active":
return []byte(`{"data":{}}`), nil
default:
t.Fatalf("admitted work without recorded ownership: %v", args)
return nil, nil
}
}
text, err := client.Generate(context.Background(), ModelConfig{}, "must not run")
want := []string{"/api/session", "own", "/api/session/ses_unowned/interrupt", "/api/session/active", "release"}
if !errors.Is(err, cause) || text != "" || client.FormatAudit != "" || !reflect.DeepEqual(events, want) {
t.Fatalf("ownership failure: text=%q err=%v audit=%q events=%v", text, err, client.FormatAudit, events)
}
}