Retry bounded budget reads after transient timeouts

This commit is contained in:
2026-11-05 15:48:14 +00:00
parent 3a9ba55277
commit 46f85556af
5 changed files with 261 additions and 10 deletions
@@ -0,0 +1,33 @@
# Budget accounting reads under CPU load
Mobile run 249 stopped before Code edits because a usage GET's two-second
context expired while its OpenCode CLI subprocess was starting under load. This
was an accounting transport deadline, not evidence that the configured token cap
had been reached. The accounting failure correctly failed closed, but one short
startup deadline was enough to reject a healthy stage.
The independent usage-read path now allows ten seconds per attempt and one
retry after 100 ms, only when the failure is a context deadline or a transport
error reporting Timeout(). Parent cancellation stops the current read or retry
backoff. Non-timeout transport failures are not retried. This is read-only
accounting retry; no session creation, generation, prompt or tool operation is
replayed. Initial, polling and final usage reads use the same policy.
A successful read still passes through the existing identity/shape/known-usage
checks and accumulated cap enforcement. Missing or malformed accounting, a
mismatched session, and exceeded caps are not retried or bypassed. Two persistent
timeouts still produce ErrBudgetAccounting and cancel work. Final accounting
retains its cancellation-independent context and must still report valid usage;
its failure is joined to the stage error. The finite transport grace can delay a
poll's accounting failure by at most approximately 20.1 seconds, or the parent's
shorter deadline; it does not change either configured cap or final totals.
Focused race tests use injected millisecond read timeouts. They cover transient
context/wrapped/network timeouts, independent retry contexts, one-retry bounds,
permanent failure and cancellation, cancelled backoff, initial/final retries,
malformed/missing/mismatched accounting, persistent polling failure, and final
cap enforcement. Existing live-agent no-replay and accounting-ledger tests are
included. No live service, run, workflow definition or budget cap was changed.
Validation: `go test -race ./internal/stage` passes (53.952s), as do the focused
race regression set, `go vet ./internal/stage`, gofmt and diff checks.
+6 -9
View File
@@ -266,15 +266,12 @@ func (c OpenCodeClient) watchBudget(ctx context.Context, budget *Budget, id stri
reader.Operations = nil
w := &budgetWatch{stop: make(chan struct{}), done: make(chan struct{})}
w.read = func(parent context.Context, final bool) error {
readCtx, cancel := context.WithTimeout(parent, 2*time.Second)
defer cancel()
var data []byte
var err error
if reader.connection != nil && reader.Backend == "v1" {
data, err = reader.connection.BudgetMessages(readCtx, id)
} else {
data, err = reader.call(readCtx, "get", "/api/session/"+id, nil)
}
data, err := readBudgetUsage(parent, reader.budgetReadPolicy, func(readCtx context.Context) ([]byte, error) {
if reader.connection != nil && reader.Backend == "v1" {
return reader.connection.BudgetMessages(readCtx, id)
}
return reader.call(readCtx, "get", "/api/session/"+id, nil)
})
if err != nil {
return fmt.Errorf("%w: usage read failed: %v", ErrBudgetAccounting, err)
}
+52
View File
@@ -0,0 +1,52 @@
package stage
import (
"context"
"errors"
"net"
"time"
)
// A usage GET can include CLI startup. Two seconds was shorter than startup
// under CPU load, falsely making otherwise healthy accounting unavailable.
// Retry only this read, never the generation or tool work it accounts for.
type budgetReadPolicy struct {
timeout time.Duration
retryDelay time.Duration
}
func readBudgetUsage(ctx context.Context, policy budgetReadPolicy, read func(context.Context) ([]byte, error)) ([]byte, error) {
if policy.timeout <= 0 {
policy.timeout = 10 * time.Second
}
if policy.retryDelay <= 0 {
policy.retryDelay = 100 * time.Millisecond
}
for attempt := 0; ; attempt++ {
if err := ctx.Err(); err != nil {
return nil, err
}
readCtx, cancel := context.WithTimeout(ctx, policy.timeout)
data, err := read(readCtx)
deadline := errors.Is(readCtx.Err(), context.DeadlineExceeded)
cancel()
if err == nil {
return data, nil
}
if parentErr := ctx.Err(); parentErr != nil {
return nil, parentErr
}
var transport net.Error
timeout := deadline || errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &transport) && transport.Timeout())
if attempt == 1 || !timeout {
return nil, err
}
timer := time.NewTimer(policy.retryDelay)
select {
case <-ctx.Done():
timer.Stop()
return nil, ctx.Err()
case <-timer.C:
}
}
}
+168
View File
@@ -0,0 +1,168 @@
package stage
import (
"context"
"errors"
"fmt"
"sync/atomic"
"testing"
"time"
)
var tinyBudgetReadPolicy = budgetReadPolicy{timeout: 5 * time.Millisecond, retryDelay: time.Nanosecond}
type timeoutReadError struct{}
func (timeoutReadError) Error() string { return "temporary transport timeout" }
func (timeoutReadError) Timeout() bool { return true }
func (timeoutReadError) Temporary() bool { return true }
func TestBudgetUsageReadRetriesOnlyTransientTimeout(t *testing.T) {
for _, kind := range []string{"context deadline", "wrapped deadline", "transport timeout"} {
t.Run(kind, func(t *testing.T) {
calls := 0
got, err := readBudgetUsage(context.Background(), tinyBudgetReadPolicy, func(ctx context.Context) ([]byte, error) {
calls++
if calls == 1 {
switch kind {
case "context deadline":
<-ctx.Done()
return nil, ctx.Err()
case "wrapped deadline":
return nil, fmt.Errorf("CLI startup: %w", context.DeadlineExceeded)
default:
return nil, timeoutReadError{}
}
}
if err := ctx.Err(); err != nil {
t.Fatalf("retry inherited expired read context: %v", err)
}
return []byte("usage"), nil
})
if err != nil || string(got) != "usage" || calls != 2 {
t.Fatalf("retry: data=%s calls=%d err=%v", got, calls, err)
}
})
}
}
func TestBudgetUsageReadStopsAfterOneRetry(t *testing.T) {
calls := 0
_, err := readBudgetUsage(context.Background(), tinyBudgetReadPolicy, func(ctx context.Context) ([]byte, error) { calls++; <-ctx.Done(); return nil, ctx.Err() })
if !errors.Is(err, context.DeadlineExceeded) || calls != 2 {
t.Fatalf("persistent timeout: calls=%d err=%v", calls, err)
}
}
func TestBudgetUsageReadDoesNotRetryPermanentOrCancelledRead(t *testing.T) {
permanent := errors.New("authorization refused")
for _, failure := range []error{permanent, context.Canceled} {
calls := 0
_, err := readBudgetUsage(context.Background(), tinyBudgetReadPolicy, func(context.Context) ([]byte, error) { calls++; return nil, failure })
if !errors.Is(err, failure) || calls != 1 {
t.Fatalf("retried permanent failure: calls=%d err=%v", calls, err)
}
}
ctx, cancel := context.WithCancel(context.Background())
calls := 0
_, err := readBudgetUsage(ctx, tinyBudgetReadPolicy, func(context.Context) ([]byte, error) { calls++; cancel(); return nil, context.DeadlineExceeded })
if !errors.Is(err, context.Canceled) || calls != 1 {
t.Fatalf("retried after parent cancelled: calls=%d err=%v", calls, err)
}
}
func TestBudgetUsageReadCancellationStopsBackoff(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
first := make(chan struct{})
done := make(chan error, 1)
go func() {
_, err := readBudgetUsage(ctx, budgetReadPolicy{timeout: time.Second, retryDelay: time.Hour}, func(context.Context) ([]byte, error) { close(first); return nil, context.DeadlineExceeded })
done <- err
}()
<-first
cancel()
select {
case err := <-done:
if !errors.Is(err, context.Canceled) {
t.Fatalf("cancelled retry: %v", err)
}
case <-time.After(time.Second):
t.Fatal("parent cancellation did not stop retry backoff")
}
}
func TestBudgetWatchRetriesInitialAndFinalReadsWithoutReplayingWork(t *testing.T) {
budget := NewBudget(0, 10)
budget.pollInterval = time.Hour
var reads atomic.Int32
client := OpenCodeClient{Backend: "v2", budgetReadPolicy: tinyBudgetReadPolicy, command: func(ctx context.Context, _ string, args ...string) ([]byte, error) {
if args[1] != "get" || args[2] != "/api/session/ses_budget" {
t.Errorf("budget retry called non-accounting operation: %v", args)
}
switch reads.Add(1) {
case 1, 3:
<-ctx.Done()
return nil, ctx.Err()
case 2:
return usageResponse("ses_budget", 0, 0), nil
default:
return usageResponse("ses_budget", 0, 9), nil
}
}}
ctx, watch, err := client.watchBudget(context.Background(), budget, "ses_budget")
if err != nil {
t.Fatal(err)
}
var terminal error
watch.stopWatch(ctx, &terminal)
cancelled, cancel := context.WithCancel(ctx)
cancel()
watch.finish(cancelled, &terminal) // final read retains its independent bounded context
if terminal != nil || reads.Load() != 4 || budget.Snapshot().UsedTokens != 9 {
t.Fatalf("final accounting: reads=%d budget=%+v err=%v", reads.Load(), budget.Snapshot(), terminal)
}
}
func TestBudgetWatchRejectsMalformedAndMissingAccountingWithoutRetry(t *testing.T) {
for _, body := range []string{`garbage`, `{"data":{"id":"ses_budget"}}`, string(usageResponse("ses_other", 0, 1))} {
calls := 0
client := OpenCodeClient{Backend: "v2", budgetReadPolicy: tinyBudgetReadPolicy, command: func(context.Context, string, ...string) ([]byte, error) { calls++; return []byte(body), nil }}
_, watch, err := client.watchBudget(context.Background(), NewBudget(0, 10), "ses_budget")
if !errors.Is(err, ErrBudgetAccounting) || watch != nil || calls != 1 {
t.Fatalf("invalid accounting was retried or bypassed: calls=%d watch=%v err=%v", calls, watch, err)
}
}
}
func TestBudgetWatchPersistentReadTimeoutCancelsAndFinalReadStillEnforcesCap(t *testing.T) {
budget := NewBudget(0, 10)
budget.pollInterval = time.Millisecond
var reads atomic.Int32
client := OpenCodeClient{Backend: "v2", budgetReadPolicy: tinyBudgetReadPolicy, command: func(ctx context.Context, _ string, args ...string) ([]byte, error) {
switch reads.Add(1) {
case 1:
return usageResponse("ses_budget", 0, 0), nil
case 2, 3:
<-ctx.Done()
return nil, ctx.Err()
default:
return usageResponse("ses_budget", 0, 12), nil
}
}}
ctx, watch, err := client.watchBudget(context.Background(), budget, "ses_budget")
if err != nil {
t.Fatal(err)
}
select {
case <-ctx.Done():
case <-time.After(time.Second):
t.Fatal("persistent missing accounting did not stop work")
}
var terminal error
watch.stopWatch(ctx, &terminal)
watch.finish(ctx, &terminal)
if !errors.Is(terminal, ErrBudgetAccounting) || !errors.Is(terminal, ErrBudgetExceeded) || budget.Snapshot().UsedTokens != 12 || reads.Load() != 4 {
t.Fatalf("failed closed/final cap: calls=%d budget=%+v err=%v", reads.Load(), budget.Snapshot(), terminal)
}
}
+2 -1
View File
@@ -38,7 +38,8 @@ type OpenCodeClient struct {
OnGenerationSession func(id, backend, reference string, active bool) error
connection *opencode.Connection
command func(context.Context, string, ...string) ([]byte, error)
sessionPollInterval time.Duration // test seam; defaults to defaultSessionPollInterval
sessionPollInterval time.Duration // test seam; defaults to defaultSessionPollInterval
budgetReadPolicy budgetReadPolicy // test seam; zero uses bounded production defaults
}
// ArtifactsEnv names the environment variable carrying a stage's artifacts