fix(stage): persist every V2 model generation in tracked sessions
This commit is contained in:
@@ -19,8 +19,10 @@ func TestProviderCheckExecutesGenerationAndReportsFailures(t *testing.T) {
|
||||
script := `#!/bin/sh
|
||||
case "$*" in
|
||||
*--help*) echo 'api';;
|
||||
*'/api/session/ses_check/generate'*) ` + map[bool]string{true: `echo '{"_tag":"UnauthorizedError","message":"expired provider credentials","token":"private-token"}'; exit 7`, false: `echo '{"data":{"text":"PIPELINES_PROVIDER_OK"}}'`}[fail] + `;;
|
||||
*'/api/session/ses_check/prompt'*) ` + map[bool]string{true: `echo '{"_tag":"UnauthorizedError","message":"expired provider credentials","token":"private-token"}'; exit 7`, false: `echo '{}'`}[fail] + `;;
|
||||
*'/api/session/ses_check/message?'*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"PIPELINES_PROVIDER_OK"}]}]}';;
|
||||
*'/api/session/ses_check/interrupt'*) echo '{}';;
|
||||
*'/api/session/active'*) echo '{"data":{}}';;
|
||||
*'/api/session'*) echo '{"data":{"id":"ses_check"}}';;
|
||||
*) exit 9;;
|
||||
esac
|
||||
|
||||
@@ -119,7 +119,17 @@ func TestProviderCheckNativeProfileKeepsHostGenerator(t *testing.T) {
|
||||
opencode := filepath.Join(root, "opencode")
|
||||
// The native generator runs the host opencode; this fixture echoes a healthy
|
||||
// report so the test needs no provider.
|
||||
script := "#!/bin/sh\ntouch " + marker + "\ncase \"$*\" in\n *--help*) echo api;;\n *'/api/session/ses_check/generate'*) echo '{\"data\":{\"text\":\"PIPELINES_PROVIDER_OK\"}}';;\n *'/api/session/ses_check/interrupt'*) echo '{}';;\n *'/api/session'*) echo '{\"data\":{\"id\":\"ses_check\"}}';;\n *) exit 9;;\nesac\n"
|
||||
script := "#!/bin/sh\ntouch " + marker + `
|
||||
case "$*" in
|
||||
*--help*) echo api;;
|
||||
*'/api/session/ses_check/prompt'*) echo '{}';;
|
||||
*'/api/session/ses_check/message?'*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"PIPELINES_PROVIDER_OK"}]}]}';;
|
||||
*'/api/session/ses_check/interrupt'*) echo '{}';;
|
||||
*'/api/session/active'*) echo '{"data":{}}';;
|
||||
*'/api/session'*) echo '{"data":{"id":"ses_check"}}';;
|
||||
*) exit 9;;
|
||||
esac
|
||||
`
|
||||
if err := os.WriteFile(opencode, []byte(script), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -67,11 +67,13 @@ case "$3" in
|
||||
n=0; if [ -f %q ]; then n=$(cat %q); fi
|
||||
n=$((n+1)); echo "$n" > %q
|
||||
printf '{"data":{"id":"ses_fresh_%%s"}}' "$n" ;;
|
||||
/api/session/ses_fresh_1/generate) echo '{"_tag":"InvalidRequestError"}'; exit 1 ;;
|
||||
/api/session/ses_fresh_1/prompt) echo '{"_tag":"InvalidRequestError"}'; exit 1 ;;
|
||||
/api/session/ses_fresh_1/interrupt) echo '{}' ;;
|
||||
/api/session/ses_fresh_1) echo '{"data":{"id":"ses_fresh_1"}}' ;;
|
||||
/api/session/ses_fresh_1/message*) echo '{"data":[{"error":{"message":"provider rejected generation","token":"private-token"}}]}' ;;
|
||||
/api/session/ses_fresh_2/generate) printf '%%s\n' '{"data":{"text":"# outcome\nsuccess\n"}}' ;;
|
||||
/api/session/ses_fresh_2/prompt) echo '{}' ;;
|
||||
/api/session/ses_fresh_2/message*) printf '%%s\n' '{"data":[{"finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess\n"}]}]}' ;;
|
||||
/api/session/ses_fresh_2) echo '{"data":{"id":"ses_fresh_2"}}' ;;
|
||||
*) exit 99 ;;
|
||||
esac
|
||||
`, log, count, count, count)
|
||||
@@ -130,7 +132,7 @@ esac
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(calls), "delete") || strings.Count(string(calls), "post /api/session\n") != 2 || strings.Count(string(calls), "ses_fresh_1/generate") != 1 {
|
||||
if strings.Contains(string(calls), "delete") || strings.Contains(string(calls), "/generate") || strings.Count(string(calls), "post /api/session\n") != 2 || strings.Count(string(calls), "ses_fresh_1/prompt") != 1 {
|
||||
t.Fatal("session deleted or reused", string(calls))
|
||||
}
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") {
|
||||
|
||||
@@ -266,6 +266,7 @@ func (c OpenCodeClient) sessionActive(ctx context.Context, sessionID string) (bo
|
||||
}
|
||||
|
||||
func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) {
|
||||
c.FormatAudit = ""
|
||||
if c.Operations != nil {
|
||||
c.Operations.Provider, c.Operations.Model = config.Provider, config.Model
|
||||
}
|
||||
@@ -287,7 +288,11 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
|
||||
}
|
||||
}
|
||||
// Fail before creating a session if the generation payload is too large.
|
||||
body := generationBody("prompt", prompt, config, c.Backend)
|
||||
promptKey := "prompt"
|
||||
if c.Backend == "v2" {
|
||||
promptKey = "text"
|
||||
}
|
||||
body := generationBody(promptKey, prompt, config, c.Backend)
|
||||
encoded, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -315,13 +320,28 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
|
||||
return "", fmt.Errorf("OpenCode returned an invalid session ID")
|
||||
}
|
||||
var watch *budgetWatch
|
||||
promptStarted := false
|
||||
defer func() {
|
||||
watch.stopWatch(ctx, &err)
|
||||
defer watch.finish(ctx, &err)
|
||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
if err != nil {
|
||||
if stopErr := c.InterruptAndConfirm(cleanup, session.Data.ID); stopErr != nil {
|
||||
stopErr := c.InterruptAndConfirm(cleanup, session.Data.ID)
|
||||
// Read after settlement so a late assistant turn remains available as
|
||||
// failure evidence. Never replay a prompt to reconstruct missing turns.
|
||||
if promptStarted && c.Backend == "v2" {
|
||||
readCtx, readCancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
|
||||
data, readErr := c.call(readCtx, "get", "/api/session/"+session.Data.ID+finalAssistantReadback, nil)
|
||||
readCancel()
|
||||
if readErr == nil && len(data) > 0 {
|
||||
c.FormatAudit = RedactEvidence(string(data))
|
||||
}
|
||||
if watch == nil {
|
||||
c.settleV2Usage(ctx, config.Budget, session.Data.ID)
|
||||
}
|
||||
}
|
||||
if stopErr != nil {
|
||||
err = fmt.Errorf("retained OpenCode session %s; interrupt failed: %w", session.Data.ID, errors.Join(err, stopErr, ErrStageCleanup))
|
||||
return
|
||||
}
|
||||
@@ -351,10 +371,12 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// V2 /generate returns text only and does not persist messages or update
|
||||
// session usage. Budgeted calls must use tracked, deny-all prompting instead.
|
||||
if c.Backend == "v2" && config.Budget.Enabled() {
|
||||
if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", generationBody("text", prompt, config, c.Backend)); err != nil {
|
||||
// V2 /generate does not persist messages or update session usage. Every V2
|
||||
// model call uses tracked prompting under the session's deny-all tool policy,
|
||||
// including unbudgeted text-only stages, so its exact native turns survive.
|
||||
if c.Backend == "v2" {
|
||||
promptStarted = true
|
||||
if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", body); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err = c.waitForSessionIdle(ctx, session.Data.ID); err != nil {
|
||||
@@ -368,6 +390,12 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !config.Budget.Enabled() {
|
||||
// As with AgentClient, missing usage is not a generation failure when
|
||||
// no limit is enabled (and a nil ledger needs no aggregate read).
|
||||
c.settleV2Usage(ctx, config.Budget, session.Data.ID)
|
||||
return text, nil
|
||||
}
|
||||
// A completed tracked response must have accounted tokens; session-zero
|
||||
// placeholders are not proof of a free, zero-token provider request.
|
||||
usageCtx, usageCancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second)
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
func TestOpenCodeClientSelectsModelAndRetainsSession(t *testing.T) {
|
||||
var calls []string
|
||||
client := OpenCodeClient{Directory: "/work/repo", command: func(ctx context.Context, directory string, args ...string) ([]byte, error) {
|
||||
client := OpenCodeClient{Backend: "v1", Directory: "/work/repo", command: func(ctx context.Context, directory string, args ...string) ([]byte, error) {
|
||||
if directory != "/work/repo" || args[0] != "api" {
|
||||
t.Fatal("incorrect OpenCode invocation context")
|
||||
}
|
||||
|
||||
@@ -80,7 +80,11 @@ func TestBudgetedV2UsesTrackedDenyAllPrompt(t *testing.T) {
|
||||
} else if err != nil || !strings.Contains(fmt.Sprint(result.Output["markdown"]), "done") {
|
||||
t.Fatal(result, err)
|
||||
}
|
||||
if !prompted || messages != 1 || !strings.Contains(client.FormatAudit, "done") {
|
||||
wantMessages := 1
|
||||
if test.want != nil {
|
||||
wantMessages++ // bounded late-evidence read after interrupt/settlement
|
||||
}
|
||||
if !prompted || messages != wantMessages || !strings.Contains(client.FormatAudit, "done") {
|
||||
t.Fatal("tracked evidence missing", prompted, messages, client.FormatAudit)
|
||||
}
|
||||
if budget.Snapshot().UsedTokens != test.tokens {
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestUnbudgetedV2PersistsTrackedTurnsBeforeReleasingOwnership(t *testing.T) {
|
||||
for _, usage := range []string{"nil budget", "known", "missing", "malformed", "unavailable"} {
|
||||
t.Run(usage, func(t *testing.T) {
|
||||
var budget *Budget
|
||||
if usage != "nil budget" {
|
||||
budget = NewBudget(0, 0)
|
||||
}
|
||||
var events, turns []string
|
||||
polls, prompts, usageReads := 0, 0, 0
|
||||
owned := false
|
||||
const prompt = "exact pipeline instructions\nDo not change the definition or tool policy."
|
||||
const final = "# outcome\nsuccess\n\n# Summary\n\nNative final result."
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path := r.URL.RequestURI()
|
||||
events = append(events, r.Method+" "+path)
|
||||
switch path {
|
||||
case "/api/session":
|
||||
var body struct {
|
||||
Permissions []map[string]string `json:"permissions"`
|
||||
Model map[string]string `json:"model"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if !reflect.DeepEqual(body.Permissions, []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}}) || body.Model["providerID"] != "fixture" || body.Model["id"] != "tracked" {
|
||||
t.Errorf("model/tool policy changed: %+v", body)
|
||||
}
|
||||
io.WriteString(w, `{"data":{"id":"ses_native"}}`)
|
||||
case "/api/session/ses_native/prompt":
|
||||
prompts++
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body["text"] != prompt || len(body) != 1 || !owned {
|
||||
t.Errorf("prompt admitted before ownership or changed: %v, %v, owned=%v", body, err, owned)
|
||||
}
|
||||
// The tracked API fixture persists native turns; /generate is
|
||||
// deliberately unsupported, so it cannot fabricate this history.
|
||||
turns = append(turns, "user: "+body["text"])
|
||||
io.WriteString(w, `{}`)
|
||||
case "/api/session/active":
|
||||
polls++
|
||||
if polls == 1 {
|
||||
io.WriteString(w, `{"data":{"ses_native":{"type":"running"}}}`)
|
||||
} else {
|
||||
turns = append(turns, "assistant: "+final)
|
||||
io.WriteString(w, `{"data":{"ses_foreign":{"type":"running"}}}`)
|
||||
}
|
||||
case "/api/session/ses_native" + finalAssistantReadback:
|
||||
if polls != 2 || !owned || len(turns) != 2 {
|
||||
t.Error("final readback preceded exact-session settlement")
|
||||
}
|
||||
json.NewEncoder(w).Encode(map[string]any{"data": []any{map[string]any{"finish": "stop", "content": []any{map[string]string{"type": "text", "text": final}}}}})
|
||||
case "/api/session/ses_native":
|
||||
usageReads++
|
||||
switch usage {
|
||||
case "known":
|
||||
w.Write(usageResponse("ses_native", .001, 3))
|
||||
case "missing":
|
||||
io.WriteString(w, `{"data":{"id":"ses_native"}}`)
|
||||
case "malformed":
|
||||
io.WriteString(w, `{"data":`)
|
||||
case "unavailable":
|
||||
http.Error(w, "unavailable", http.StatusServiceUnavailable)
|
||||
default:
|
||||
t.Error("nil budget should not require usage")
|
||||
}
|
||||
default:
|
||||
t.Errorf("unexpected native operation (replay, generate or interrupt): %s", path)
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
client := OpenCodeClient{Backend: "v2", Directory: "/work", sessionPollInterval: time.Millisecond}
|
||||
client.OnGenerationSession = func(id, backend, reference string, active bool) error {
|
||||
if id != "ses_native" || backend != "v2" || reference != "" {
|
||||
t.Fatalf("incorrect ownership: %s %s %s", id, backend, reference)
|
||||
}
|
||||
if active {
|
||||
events = append(events, "own")
|
||||
} else {
|
||||
if polls != 2 || len(turns) != 2 || !strings.Contains(client.FormatAudit, "Native final result") {
|
||||
t.Fatal("ownership released before settled final evidence")
|
||||
}
|
||||
events = append(events, "release")
|
||||
}
|
||||
owned = active
|
||||
return nil
|
||||
}
|
||||
client.command = func(ctx context.Context, _ string, args ...string) ([]byte, error) {
|
||||
var body io.Reader
|
||||
if len(args) > 4 {
|
||||
body = strings.NewReader(args[4])
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, strings.ToUpper(args[1]), server.URL+args[2], body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := server.Client().Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, &openCodeError{status: res.StatusCode}
|
||||
}
|
||||
return io.ReadAll(res.Body)
|
||||
}
|
||||
text, err := client.Generate(context.Background(), ModelConfig{Provider: "fixture", Model: "tracked", Budget: budget}, prompt)
|
||||
if err != nil || text != final || owned || prompts != 1 || !reflect.DeepEqual(turns, []string{"user: " + prompt, "assistant: " + final}) {
|
||||
t.Fatalf("tracked generation: text=%q err=%v owned=%v prompts=%d turns=%v", text, err, owned, prompts, turns)
|
||||
}
|
||||
want := []string{"POST /api/session", "own", "POST /api/session/ses_native/prompt", "GET /api/session/active", "GET /api/session/active", "GET /api/session/ses_native" + finalAssistantReadback}
|
||||
if budget != nil {
|
||||
want = append(want, "GET /api/session/ses_native")
|
||||
}
|
||||
want = append(want, "release")
|
||||
if !reflect.DeepEqual(events, want) || usageReads != map[bool]int{false: 0, true: 1}[budget != nil] {
|
||||
t.Fatalf("lifecycle = %v; want %v; usage reads=%d", events, want, usageReads)
|
||||
}
|
||||
if usage == "known" && budget.Snapshot().UsedTokens != 3 {
|
||||
t.Fatal("unbudgeted usage not retained", budget.Snapshot())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackedModelFailureSettlesAndRetainsLateEvidenceWithoutReplay(t *testing.T) {
|
||||
for _, failure := range []string{"prompt", "wait", "readback", "cleanup", "canceled"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
var events []string
|
||||
prompts, reads := 0, 0
|
||||
interrupted, owned := false, false
|
||||
cause := errors.New("native transport failure")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
if failure == "canceled" {
|
||||
cause = context.Canceled
|
||||
}
|
||||
client := OpenCodeClient{Backend: "v2", sessionPollInterval: time.Millisecond}
|
||||
client.OnGenerationSession = func(_ string, _, _ string, active bool) error {
|
||||
owned = active
|
||||
if active {
|
||||
events = append(events, "own")
|
||||
} else {
|
||||
if !interrupted || reads == 0 {
|
||||
t.Fatal("released before settlement and late evidence read")
|
||||
}
|
||||
events = append(events, "release")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
client.command = func(ctx context.Context, _ string, args ...string) ([]byte, error) {
|
||||
path := args[2]
|
||||
events = append(events, path)
|
||||
switch path {
|
||||
case "/api/session":
|
||||
return []byte(`{"data":{"id":"ses_failed"}}`), nil
|
||||
case "/api/session/ses_failed/prompt":
|
||||
prompts++
|
||||
if !owned {
|
||||
t.Fatal("prompt before ownership")
|
||||
}
|
||||
if failure == "prompt" || failure == "canceled" {
|
||||
if failure == "canceled" {
|
||||
cancel()
|
||||
}
|
||||
return nil, cause
|
||||
}
|
||||
return []byte(`{}`), nil
|
||||
case "/api/session/active":
|
||||
if !interrupted && (failure == "wait" || failure == "cleanup") || interrupted && failure == "cleanup" {
|
||||
return nil, cause
|
||||
}
|
||||
return []byte(`{"data":{}}`), nil
|
||||
case "/api/session/ses_failed/interrupt":
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal("cleanup inherited canceled context")
|
||||
}
|
||||
interrupted = true
|
||||
return []byte(`{}`), nil
|
||||
case "/api/session/ses_failed" + finalAssistantReadback:
|
||||
reads++
|
||||
if !interrupted {
|
||||
return nil, cause
|
||||
}
|
||||
return []byte(`{"data":[{"finish":"stop","content":[{"type":"text","text":"late failure evidence"}]}]}`), nil
|
||||
default:
|
||||
t.Fatalf("unexpected call: %s", path)
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
text, err := client.Generate(ctx, ModelConfig{}, "unchanged prompt")
|
||||
if !errors.Is(err, cause) || text != "" || prompts != 1 || !strings.Contains(client.FormatAudit, "late failure evidence") || owned != (failure == "cleanup") {
|
||||
t.Fatalf("failure lost evidence/ownership or replayed: text=%q err=%v prompts=%d owned=%v audit=%s events=%v", text, err, prompts, owned, client.FormatAudit, events)
|
||||
}
|
||||
if failure == "cleanup" && !errors.Is(err, ErrStageCleanup) {
|
||||
t.Fatal("unconfirmed cleanup must retain ownership", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackedModelOwnershipFailureDoesNotAdmitPrompt(t *testing.T) {
|
||||
cause := errors.New("ownership persistence failed")
|
||||
var events []string
|
||||
client := OpenCodeClient{Backend: "v2", FormatAudit: "old unrelated evidence"}
|
||||
client.OnGenerationSession = func(_ string, _, _ string, active bool) error {
|
||||
if active {
|
||||
events = append(events, "own")
|
||||
return cause
|
||||
}
|
||||
events = append(events, "release")
|
||||
return nil
|
||||
}
|
||||
client.command = func(_ context.Context, _ string, args ...string) ([]byte, error) {
|
||||
events = append(events, args[2])
|
||||
switch args[2] {
|
||||
case "/api/session":
|
||||
return []byte(`{"data":{"id":"ses_unowned"}}`), nil
|
||||
case "/api/session/ses_unowned/interrupt":
|
||||
return []byte(`{}`), nil
|
||||
case "/api/session/active":
|
||||
return []byte(`{"data":{}}`), nil
|
||||
default:
|
||||
t.Fatalf("admitted work without recorded ownership: %v", args)
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
text, err := client.Generate(context.Background(), ModelConfig{}, "must not run")
|
||||
want := []string{"/api/session", "own", "/api/session/ses_unowned/interrupt", "/api/session/active", "release"}
|
||||
if !errors.Is(err, cause) || text != "" || client.FormatAudit != "" || !reflect.DeepEqual(events, want) {
|
||||
t.Fatalf("ownership failure: text=%q err=%v audit=%q events=%v", text, err, client.FormatAudit, events)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user