diff --git a/docs/notes/budget-accounting-read-timeouts.md b/docs/notes/budget-accounting-read-timeouts.md new file mode 100644 index 00000000..c0b4b4d3 --- /dev/null +++ b/docs/notes/budget-accounting-read-timeouts.md @@ -0,0 +1,33 @@ +# Budget accounting reads under CPU load + +Mobile run 249 stopped before Code edits because a usage GET's two-second +context expired while its OpenCode CLI subprocess was starting under load. This +was an accounting transport deadline, not evidence that the configured token cap +had been reached. The accounting failure correctly failed closed, but one short +startup deadline was enough to reject a healthy stage. + +The independent usage-read path now allows ten seconds per attempt and one +retry after 100 ms, only when the failure is a context deadline or a transport +error reporting Timeout(). Parent cancellation stops the current read or retry +backoff. Non-timeout transport failures are not retried. This is read-only +accounting retry; no session creation, generation, prompt or tool operation is +replayed. Initial, polling and final usage reads use the same policy. + +A successful read still passes through the existing identity/shape/known-usage +checks and accumulated cap enforcement. Missing or malformed accounting, a +mismatched session, and exceeded caps are not retried or bypassed. Two persistent +timeouts still produce ErrBudgetAccounting and cancel work. Final accounting +retains its cancellation-independent context and must still report valid usage; +its failure is joined to the stage error. The finite transport grace can delay a +poll's accounting failure by at most approximately 20.1 seconds, or the parent's +shorter deadline; it does not change either configured cap or final totals. + +Focused race tests use injected millisecond read timeouts. They cover transient +context/wrapped/network timeouts, independent retry contexts, one-retry bounds, +permanent failure and cancellation, cancelled backoff, initial/final retries, +malformed/missing/mismatched accounting, persistent polling failure, and final +cap enforcement. Existing live-agent no-replay and accounting-ledger tests are +included. No live service, run, workflow definition or budget cap was changed. + +Validation: `go test -race ./internal/stage` passes (53.952s), as do the focused +race regression set, `go vet ./internal/stage`, gofmt and diff checks. diff --git a/internal/stage/budget.go b/internal/stage/budget.go index 821843c1..9d04b940 100644 --- a/internal/stage/budget.go +++ b/internal/stage/budget.go @@ -266,15 +266,12 @@ func (c OpenCodeClient) watchBudget(ctx context.Context, budget *Budget, id stri reader.Operations = nil w := &budgetWatch{stop: make(chan struct{}), done: make(chan struct{})} w.read = func(parent context.Context, final bool) error { - readCtx, cancel := context.WithTimeout(parent, 2*time.Second) - defer cancel() - var data []byte - var err error - if reader.connection != nil && reader.Backend == "v1" { - data, err = reader.connection.BudgetMessages(readCtx, id) - } else { - data, err = reader.call(readCtx, "get", "/api/session/"+id, nil) - } + data, err := readBudgetUsage(parent, reader.budgetReadPolicy, func(readCtx context.Context) ([]byte, error) { + if reader.connection != nil && reader.Backend == "v1" { + return reader.connection.BudgetMessages(readCtx, id) + } + return reader.call(readCtx, "get", "/api/session/"+id, nil) + }) if err != nil { return fmt.Errorf("%w: usage read failed: %v", ErrBudgetAccounting, err) } diff --git a/internal/stage/budget_read.go b/internal/stage/budget_read.go new file mode 100644 index 00000000..45516dcd --- /dev/null +++ b/internal/stage/budget_read.go @@ -0,0 +1,52 @@ +package stage + +import ( + "context" + "errors" + "net" + "time" +) + +// A usage GET can include CLI startup. Two seconds was shorter than startup +// under CPU load, falsely making otherwise healthy accounting unavailable. +// Retry only this read, never the generation or tool work it accounts for. +type budgetReadPolicy struct { + timeout time.Duration + retryDelay time.Duration +} + +func readBudgetUsage(ctx context.Context, policy budgetReadPolicy, read func(context.Context) ([]byte, error)) ([]byte, error) { + if policy.timeout <= 0 { + policy.timeout = 10 * time.Second + } + if policy.retryDelay <= 0 { + policy.retryDelay = 100 * time.Millisecond + } + for attempt := 0; ; attempt++ { + if err := ctx.Err(); err != nil { + return nil, err + } + readCtx, cancel := context.WithTimeout(ctx, policy.timeout) + data, err := read(readCtx) + deadline := errors.Is(readCtx.Err(), context.DeadlineExceeded) + cancel() + if err == nil { + return data, nil + } + if parentErr := ctx.Err(); parentErr != nil { + return nil, parentErr + } + var transport net.Error + timeout := deadline || errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &transport) && transport.Timeout()) + if attempt == 1 || !timeout { + return nil, err + } + timer := time.NewTimer(policy.retryDelay) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } +} diff --git a/internal/stage/budget_read_test.go b/internal/stage/budget_read_test.go new file mode 100644 index 00000000..33dbee71 --- /dev/null +++ b/internal/stage/budget_read_test.go @@ -0,0 +1,168 @@ +package stage + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + "time" +) + +var tinyBudgetReadPolicy = budgetReadPolicy{timeout: 5 * time.Millisecond, retryDelay: time.Nanosecond} + +type timeoutReadError struct{} + +func (timeoutReadError) Error() string { return "temporary transport timeout" } +func (timeoutReadError) Timeout() bool { return true } +func (timeoutReadError) Temporary() bool { return true } + +func TestBudgetUsageReadRetriesOnlyTransientTimeout(t *testing.T) { + for _, kind := range []string{"context deadline", "wrapped deadline", "transport timeout"} { + t.Run(kind, func(t *testing.T) { + calls := 0 + got, err := readBudgetUsage(context.Background(), tinyBudgetReadPolicy, func(ctx context.Context) ([]byte, error) { + calls++ + if calls == 1 { + switch kind { + case "context deadline": + <-ctx.Done() + return nil, ctx.Err() + case "wrapped deadline": + return nil, fmt.Errorf("CLI startup: %w", context.DeadlineExceeded) + default: + return nil, timeoutReadError{} + } + } + if err := ctx.Err(); err != nil { + t.Fatalf("retry inherited expired read context: %v", err) + } + return []byte("usage"), nil + }) + if err != nil || string(got) != "usage" || calls != 2 { + t.Fatalf("retry: data=%s calls=%d err=%v", got, calls, err) + } + }) + } +} + +func TestBudgetUsageReadStopsAfterOneRetry(t *testing.T) { + calls := 0 + _, err := readBudgetUsage(context.Background(), tinyBudgetReadPolicy, func(ctx context.Context) ([]byte, error) { calls++; <-ctx.Done(); return nil, ctx.Err() }) + if !errors.Is(err, context.DeadlineExceeded) || calls != 2 { + t.Fatalf("persistent timeout: calls=%d err=%v", calls, err) + } +} + +func TestBudgetUsageReadDoesNotRetryPermanentOrCancelledRead(t *testing.T) { + permanent := errors.New("authorization refused") + for _, failure := range []error{permanent, context.Canceled} { + calls := 0 + _, err := readBudgetUsage(context.Background(), tinyBudgetReadPolicy, func(context.Context) ([]byte, error) { calls++; return nil, failure }) + if !errors.Is(err, failure) || calls != 1 { + t.Fatalf("retried permanent failure: calls=%d err=%v", calls, err) + } + } + ctx, cancel := context.WithCancel(context.Background()) + calls := 0 + _, err := readBudgetUsage(ctx, tinyBudgetReadPolicy, func(context.Context) ([]byte, error) { calls++; cancel(); return nil, context.DeadlineExceeded }) + if !errors.Is(err, context.Canceled) || calls != 1 { + t.Fatalf("retried after parent cancelled: calls=%d err=%v", calls, err) + } +} + +func TestBudgetUsageReadCancellationStopsBackoff(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + first := make(chan struct{}) + done := make(chan error, 1) + go func() { + _, err := readBudgetUsage(ctx, budgetReadPolicy{timeout: time.Second, retryDelay: time.Hour}, func(context.Context) ([]byte, error) { close(first); return nil, context.DeadlineExceeded }) + done <- err + }() + <-first + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled retry: %v", err) + } + case <-time.After(time.Second): + t.Fatal("parent cancellation did not stop retry backoff") + } +} + +func TestBudgetWatchRetriesInitialAndFinalReadsWithoutReplayingWork(t *testing.T) { + budget := NewBudget(0, 10) + budget.pollInterval = time.Hour + var reads atomic.Int32 + client := OpenCodeClient{Backend: "v2", budgetReadPolicy: tinyBudgetReadPolicy, command: func(ctx context.Context, _ string, args ...string) ([]byte, error) { + if args[1] != "get" || args[2] != "/api/session/ses_budget" { + t.Errorf("budget retry called non-accounting operation: %v", args) + } + switch reads.Add(1) { + case 1, 3: + <-ctx.Done() + return nil, ctx.Err() + case 2: + return usageResponse("ses_budget", 0, 0), nil + default: + return usageResponse("ses_budget", 0, 9), nil + } + }} + ctx, watch, err := client.watchBudget(context.Background(), budget, "ses_budget") + if err != nil { + t.Fatal(err) + } + var terminal error + watch.stopWatch(ctx, &terminal) + cancelled, cancel := context.WithCancel(ctx) + cancel() + watch.finish(cancelled, &terminal) // final read retains its independent bounded context + if terminal != nil || reads.Load() != 4 || budget.Snapshot().UsedTokens != 9 { + t.Fatalf("final accounting: reads=%d budget=%+v err=%v", reads.Load(), budget.Snapshot(), terminal) + } +} + +func TestBudgetWatchRejectsMalformedAndMissingAccountingWithoutRetry(t *testing.T) { + for _, body := range []string{`garbage`, `{"data":{"id":"ses_budget"}}`, string(usageResponse("ses_other", 0, 1))} { + calls := 0 + client := OpenCodeClient{Backend: "v2", budgetReadPolicy: tinyBudgetReadPolicy, command: func(context.Context, string, ...string) ([]byte, error) { calls++; return []byte(body), nil }} + _, watch, err := client.watchBudget(context.Background(), NewBudget(0, 10), "ses_budget") + if !errors.Is(err, ErrBudgetAccounting) || watch != nil || calls != 1 { + t.Fatalf("invalid accounting was retried or bypassed: calls=%d watch=%v err=%v", calls, watch, err) + } + } +} + +func TestBudgetWatchPersistentReadTimeoutCancelsAndFinalReadStillEnforcesCap(t *testing.T) { + budget := NewBudget(0, 10) + budget.pollInterval = time.Millisecond + var reads atomic.Int32 + client := OpenCodeClient{Backend: "v2", budgetReadPolicy: tinyBudgetReadPolicy, command: func(ctx context.Context, _ string, args ...string) ([]byte, error) { + switch reads.Add(1) { + case 1: + return usageResponse("ses_budget", 0, 0), nil + case 2, 3: + <-ctx.Done() + return nil, ctx.Err() + default: + return usageResponse("ses_budget", 0, 12), nil + } + }} + ctx, watch, err := client.watchBudget(context.Background(), budget, "ses_budget") + if err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + case <-time.After(time.Second): + t.Fatal("persistent missing accounting did not stop work") + } + var terminal error + watch.stopWatch(ctx, &terminal) + watch.finish(ctx, &terminal) + if !errors.Is(terminal, ErrBudgetAccounting) || !errors.Is(terminal, ErrBudgetExceeded) || budget.Snapshot().UsedTokens != 12 || reads.Load() != 4 { + t.Fatalf("failed closed/final cap: calls=%d budget=%+v err=%v", reads.Load(), budget.Snapshot(), terminal) + } +} diff --git a/internal/stage/opencode.go b/internal/stage/opencode.go index 7540c17e..1ba4bf06 100644 --- a/internal/stage/opencode.go +++ b/internal/stage/opencode.go @@ -38,7 +38,8 @@ type OpenCodeClient struct { OnGenerationSession func(id, backend, reference string, active bool) error connection *opencode.Connection command func(context.Context, string, ...string) ([]byte, error) - sessionPollInterval time.Duration // test seam; defaults to defaultSessionPollInterval + sessionPollInterval time.Duration // test seam; defaults to defaultSessionPollInterval + budgetReadPolicy budgetReadPolicy // test seam; zero uses bounded production defaults } // ArtifactsEnv names the environment variable carrying a stage's artifacts