Retry bounded budget reads after transient timeouts

This commit is contained in:
2026-11-05 15:48:14 +00:00
parent 3a9ba55277
commit 46f85556af
5 changed files with 261 additions and 10 deletions
+6 -9
View File
@@ -266,15 +266,12 @@ func (c OpenCodeClient) watchBudget(ctx context.Context, budget *Budget, id stri
reader.Operations = nil
w := &budgetWatch{stop: make(chan struct{}), done: make(chan struct{})}
w.read = func(parent context.Context, final bool) error {
readCtx, cancel := context.WithTimeout(parent, 2*time.Second)
defer cancel()
var data []byte
var err error
if reader.connection != nil && reader.Backend == "v1" {
data, err = reader.connection.BudgetMessages(readCtx, id)
} else {
data, err = reader.call(readCtx, "get", "/api/session/"+id, nil)
}
data, err := readBudgetUsage(parent, reader.budgetReadPolicy, func(readCtx context.Context) ([]byte, error) {
if reader.connection != nil && reader.Backend == "v1" {
return reader.connection.BudgetMessages(readCtx, id)
}
return reader.call(readCtx, "get", "/api/session/"+id, nil)
})
if err != nil {
return fmt.Errorf("%w: usage read failed: %v", ErrBudgetAccounting, err)
}
+52
View File
@@ -0,0 +1,52 @@
package stage
import (
"context"
"errors"
"net"
"time"
)
// A usage GET can include CLI startup. Two seconds was shorter than startup
// under CPU load, falsely making otherwise healthy accounting unavailable.
// Retry only this read, never the generation or tool work it accounts for.
type budgetReadPolicy struct {
timeout time.Duration
retryDelay time.Duration
}
func readBudgetUsage(ctx context.Context, policy budgetReadPolicy, read func(context.Context) ([]byte, error)) ([]byte, error) {
if policy.timeout <= 0 {
policy.timeout = 10 * time.Second
}
if policy.retryDelay <= 0 {
policy.retryDelay = 100 * time.Millisecond
}
for attempt := 0; ; attempt++ {
if err := ctx.Err(); err != nil {
return nil, err
}
readCtx, cancel := context.WithTimeout(ctx, policy.timeout)
data, err := read(readCtx)
deadline := errors.Is(readCtx.Err(), context.DeadlineExceeded)
cancel()
if err == nil {
return data, nil
}
if parentErr := ctx.Err(); parentErr != nil {
return nil, parentErr
}
var transport net.Error
timeout := deadline || errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &transport) && transport.Timeout())
if attempt == 1 || !timeout {
return nil, err
}
timer := time.NewTimer(policy.retryDelay)
select {
case <-ctx.Done():
timer.Stop()
return nil, ctx.Err()
case <-timer.C:
}
}
}
+168
View File
@@ -0,0 +1,168 @@
package stage
import (
"context"
"errors"
"fmt"
"sync/atomic"
"testing"
"time"
)
var tinyBudgetReadPolicy = budgetReadPolicy{timeout: 5 * time.Millisecond, retryDelay: time.Nanosecond}
type timeoutReadError struct{}
func (timeoutReadError) Error() string { return "temporary transport timeout" }
func (timeoutReadError) Timeout() bool { return true }
func (timeoutReadError) Temporary() bool { return true }
func TestBudgetUsageReadRetriesOnlyTransientTimeout(t *testing.T) {
for _, kind := range []string{"context deadline", "wrapped deadline", "transport timeout"} {
t.Run(kind, func(t *testing.T) {
calls := 0
got, err := readBudgetUsage(context.Background(), tinyBudgetReadPolicy, func(ctx context.Context) ([]byte, error) {
calls++
if calls == 1 {
switch kind {
case "context deadline":
<-ctx.Done()
return nil, ctx.Err()
case "wrapped deadline":
return nil, fmt.Errorf("CLI startup: %w", context.DeadlineExceeded)
default:
return nil, timeoutReadError{}
}
}
if err := ctx.Err(); err != nil {
t.Fatalf("retry inherited expired read context: %v", err)
}
return []byte("usage"), nil
})
if err != nil || string(got) != "usage" || calls != 2 {
t.Fatalf("retry: data=%s calls=%d err=%v", got, calls, err)
}
})
}
}
func TestBudgetUsageReadStopsAfterOneRetry(t *testing.T) {
calls := 0
_, err := readBudgetUsage(context.Background(), tinyBudgetReadPolicy, func(ctx context.Context) ([]byte, error) { calls++; <-ctx.Done(); return nil, ctx.Err() })
if !errors.Is(err, context.DeadlineExceeded) || calls != 2 {
t.Fatalf("persistent timeout: calls=%d err=%v", calls, err)
}
}
func TestBudgetUsageReadDoesNotRetryPermanentOrCancelledRead(t *testing.T) {
permanent := errors.New("authorization refused")
for _, failure := range []error{permanent, context.Canceled} {
calls := 0
_, err := readBudgetUsage(context.Background(), tinyBudgetReadPolicy, func(context.Context) ([]byte, error) { calls++; return nil, failure })
if !errors.Is(err, failure) || calls != 1 {
t.Fatalf("retried permanent failure: calls=%d err=%v", calls, err)
}
}
ctx, cancel := context.WithCancel(context.Background())
calls := 0
_, err := readBudgetUsage(ctx, tinyBudgetReadPolicy, func(context.Context) ([]byte, error) { calls++; cancel(); return nil, context.DeadlineExceeded })
if !errors.Is(err, context.Canceled) || calls != 1 {
t.Fatalf("retried after parent cancelled: calls=%d err=%v", calls, err)
}
}
func TestBudgetUsageReadCancellationStopsBackoff(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
first := make(chan struct{})
done := make(chan error, 1)
go func() {
_, err := readBudgetUsage(ctx, budgetReadPolicy{timeout: time.Second, retryDelay: time.Hour}, func(context.Context) ([]byte, error) { close(first); return nil, context.DeadlineExceeded })
done <- err
}()
<-first
cancel()
select {
case err := <-done:
if !errors.Is(err, context.Canceled) {
t.Fatalf("cancelled retry: %v", err)
}
case <-time.After(time.Second):
t.Fatal("parent cancellation did not stop retry backoff")
}
}
func TestBudgetWatchRetriesInitialAndFinalReadsWithoutReplayingWork(t *testing.T) {
budget := NewBudget(0, 10)
budget.pollInterval = time.Hour
var reads atomic.Int32
client := OpenCodeClient{Backend: "v2", budgetReadPolicy: tinyBudgetReadPolicy, command: func(ctx context.Context, _ string, args ...string) ([]byte, error) {
if args[1] != "get" || args[2] != "/api/session/ses_budget" {
t.Errorf("budget retry called non-accounting operation: %v", args)
}
switch reads.Add(1) {
case 1, 3:
<-ctx.Done()
return nil, ctx.Err()
case 2:
return usageResponse("ses_budget", 0, 0), nil
default:
return usageResponse("ses_budget", 0, 9), nil
}
}}
ctx, watch, err := client.watchBudget(context.Background(), budget, "ses_budget")
if err != nil {
t.Fatal(err)
}
var terminal error
watch.stopWatch(ctx, &terminal)
cancelled, cancel := context.WithCancel(ctx)
cancel()
watch.finish(cancelled, &terminal) // final read retains its independent bounded context
if terminal != nil || reads.Load() != 4 || budget.Snapshot().UsedTokens != 9 {
t.Fatalf("final accounting: reads=%d budget=%+v err=%v", reads.Load(), budget.Snapshot(), terminal)
}
}
func TestBudgetWatchRejectsMalformedAndMissingAccountingWithoutRetry(t *testing.T) {
for _, body := range []string{`garbage`, `{"data":{"id":"ses_budget"}}`, string(usageResponse("ses_other", 0, 1))} {
calls := 0
client := OpenCodeClient{Backend: "v2", budgetReadPolicy: tinyBudgetReadPolicy, command: func(context.Context, string, ...string) ([]byte, error) { calls++; return []byte(body), nil }}
_, watch, err := client.watchBudget(context.Background(), NewBudget(0, 10), "ses_budget")
if !errors.Is(err, ErrBudgetAccounting) || watch != nil || calls != 1 {
t.Fatalf("invalid accounting was retried or bypassed: calls=%d watch=%v err=%v", calls, watch, err)
}
}
}
func TestBudgetWatchPersistentReadTimeoutCancelsAndFinalReadStillEnforcesCap(t *testing.T) {
budget := NewBudget(0, 10)
budget.pollInterval = time.Millisecond
var reads atomic.Int32
client := OpenCodeClient{Backend: "v2", budgetReadPolicy: tinyBudgetReadPolicy, command: func(ctx context.Context, _ string, args ...string) ([]byte, error) {
switch reads.Add(1) {
case 1:
return usageResponse("ses_budget", 0, 0), nil
case 2, 3:
<-ctx.Done()
return nil, ctx.Err()
default:
return usageResponse("ses_budget", 0, 12), nil
}
}}
ctx, watch, err := client.watchBudget(context.Background(), budget, "ses_budget")
if err != nil {
t.Fatal(err)
}
select {
case <-ctx.Done():
case <-time.After(time.Second):
t.Fatal("persistent missing accounting did not stop work")
}
var terminal error
watch.stopWatch(ctx, &terminal)
watch.finish(ctx, &terminal)
if !errors.Is(terminal, ErrBudgetAccounting) || !errors.Is(terminal, ErrBudgetExceeded) || budget.Snapshot().UsedTokens != 12 || reads.Load() != 4 {
t.Fatalf("failed closed/final cap: calls=%d budget=%+v err=%v", reads.Load(), budget.Snapshot(), terminal)
}
}
+2 -1
View File
@@ -38,7 +38,8 @@ type OpenCodeClient struct {
OnGenerationSession func(id, backend, reference string, active bool) error
connection *opencode.Connection
command func(context.Context, string, ...string) ([]byte, error)
sessionPollInterval time.Duration // test seam; defaults to defaultSessionPollInterval
sessionPollInterval time.Duration // test seam; defaults to defaultSessionPollInterval
budgetReadPolicy budgetReadPolicy // test seam; zero uses bounded production defaults
}
// ArtifactsEnv names the environment variable carrying a stage's artifacts