Retry bounded budget reads after transient timeouts
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
# Budget accounting reads under CPU load
|
||||
|
||||
Mobile run 249 stopped before Code edits because a usage GET's two-second
|
||||
context expired while its OpenCode CLI subprocess was starting under load. This
|
||||
was an accounting transport deadline, not evidence that the configured token cap
|
||||
had been reached. The accounting failure correctly failed closed, but one short
|
||||
startup deadline was enough to reject a healthy stage.
|
||||
|
||||
The independent usage-read path now allows ten seconds per attempt and one
|
||||
retry after 100 ms, only when the failure is a context deadline or a transport
|
||||
error reporting Timeout(). Parent cancellation stops the current read or retry
|
||||
backoff. Non-timeout transport failures are not retried. This is read-only
|
||||
accounting retry; no session creation, generation, prompt or tool operation is
|
||||
replayed. Initial, polling and final usage reads use the same policy.
|
||||
|
||||
A successful read still passes through the existing identity/shape/known-usage
|
||||
checks and accumulated cap enforcement. Missing or malformed accounting, a
|
||||
mismatched session, and exceeded caps are not retried or bypassed. Two persistent
|
||||
timeouts still produce ErrBudgetAccounting and cancel work. Final accounting
|
||||
retains its cancellation-independent context and must still report valid usage;
|
||||
its failure is joined to the stage error. The finite transport grace can delay a
|
||||
poll's accounting failure by at most approximately 20.1 seconds, or the parent's
|
||||
shorter deadline; it does not change either configured cap or final totals.
|
||||
|
||||
Focused race tests use injected millisecond read timeouts. They cover transient
|
||||
context/wrapped/network timeouts, independent retry contexts, one-retry bounds,
|
||||
permanent failure and cancellation, cancelled backoff, initial/final retries,
|
||||
malformed/missing/mismatched accounting, persistent polling failure, and final
|
||||
cap enforcement. Existing live-agent no-replay and accounting-ledger tests are
|
||||
included. No live service, run, workflow definition or budget cap was changed.
|
||||
|
||||
Validation: `go test -race ./internal/stage` passes (53.952s), as do the focused
|
||||
race regression set, `go vet ./internal/stage`, gofmt and diff checks.
|
||||
@@ -266,15 +266,12 @@ func (c OpenCodeClient) watchBudget(ctx context.Context, budget *Budget, id stri
|
||||
reader.Operations = nil
|
||||
w := &budgetWatch{stop: make(chan struct{}), done: make(chan struct{})}
|
||||
w.read = func(parent context.Context, final bool) error {
|
||||
readCtx, cancel := context.WithTimeout(parent, 2*time.Second)
|
||||
defer cancel()
|
||||
var data []byte
|
||||
var err error
|
||||
if reader.connection != nil && reader.Backend == "v1" {
|
||||
data, err = reader.connection.BudgetMessages(readCtx, id)
|
||||
} else {
|
||||
data, err = reader.call(readCtx, "get", "/api/session/"+id, nil)
|
||||
}
|
||||
data, err := readBudgetUsage(parent, reader.budgetReadPolicy, func(readCtx context.Context) ([]byte, error) {
|
||||
if reader.connection != nil && reader.Backend == "v1" {
|
||||
return reader.connection.BudgetMessages(readCtx, id)
|
||||
}
|
||||
return reader.call(readCtx, "get", "/api/session/"+id, nil)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: usage read failed: %v", ErrBudgetAccounting, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A usage GET can include CLI startup. Two seconds was shorter than startup
|
||||
// under CPU load, falsely making otherwise healthy accounting unavailable.
|
||||
// Retry only this read, never the generation or tool work it accounts for.
|
||||
type budgetReadPolicy struct {
|
||||
timeout time.Duration
|
||||
retryDelay time.Duration
|
||||
}
|
||||
|
||||
func readBudgetUsage(ctx context.Context, policy budgetReadPolicy, read func(context.Context) ([]byte, error)) ([]byte, error) {
|
||||
if policy.timeout <= 0 {
|
||||
policy.timeout = 10 * time.Second
|
||||
}
|
||||
if policy.retryDelay <= 0 {
|
||||
policy.retryDelay = 100 * time.Millisecond
|
||||
}
|
||||
for attempt := 0; ; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
readCtx, cancel := context.WithTimeout(ctx, policy.timeout)
|
||||
data, err := read(readCtx)
|
||||
deadline := errors.Is(readCtx.Err(), context.DeadlineExceeded)
|
||||
cancel()
|
||||
if err == nil {
|
||||
return data, nil
|
||||
}
|
||||
if parentErr := ctx.Err(); parentErr != nil {
|
||||
return nil, parentErr
|
||||
}
|
||||
var transport net.Error
|
||||
timeout := deadline || errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &transport) && transport.Timeout())
|
||||
if attempt == 1 || !timeout {
|
||||
return nil, err
|
||||
}
|
||||
timer := time.NewTimer(policy.retryDelay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return nil, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var tinyBudgetReadPolicy = budgetReadPolicy{timeout: 5 * time.Millisecond, retryDelay: time.Nanosecond}
|
||||
|
||||
type timeoutReadError struct{}
|
||||
|
||||
func (timeoutReadError) Error() string { return "temporary transport timeout" }
|
||||
func (timeoutReadError) Timeout() bool { return true }
|
||||
func (timeoutReadError) Temporary() bool { return true }
|
||||
|
||||
func TestBudgetUsageReadRetriesOnlyTransientTimeout(t *testing.T) {
|
||||
for _, kind := range []string{"context deadline", "wrapped deadline", "transport timeout"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
calls := 0
|
||||
got, err := readBudgetUsage(context.Background(), tinyBudgetReadPolicy, func(ctx context.Context) ([]byte, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
switch kind {
|
||||
case "context deadline":
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
case "wrapped deadline":
|
||||
return nil, fmt.Errorf("CLI startup: %w", context.DeadlineExceeded)
|
||||
default:
|
||||
return nil, timeoutReadError{}
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
t.Fatalf("retry inherited expired read context: %v", err)
|
||||
}
|
||||
return []byte("usage"), nil
|
||||
})
|
||||
if err != nil || string(got) != "usage" || calls != 2 {
|
||||
t.Fatalf("retry: data=%s calls=%d err=%v", got, calls, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgetUsageReadStopsAfterOneRetry(t *testing.T) {
|
||||
calls := 0
|
||||
_, err := readBudgetUsage(context.Background(), tinyBudgetReadPolicy, func(ctx context.Context) ([]byte, error) { calls++; <-ctx.Done(); return nil, ctx.Err() })
|
||||
if !errors.Is(err, context.DeadlineExceeded) || calls != 2 {
|
||||
t.Fatalf("persistent timeout: calls=%d err=%v", calls, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgetUsageReadDoesNotRetryPermanentOrCancelledRead(t *testing.T) {
|
||||
permanent := errors.New("authorization refused")
|
||||
for _, failure := range []error{permanent, context.Canceled} {
|
||||
calls := 0
|
||||
_, err := readBudgetUsage(context.Background(), tinyBudgetReadPolicy, func(context.Context) ([]byte, error) { calls++; return nil, failure })
|
||||
if !errors.Is(err, failure) || calls != 1 {
|
||||
t.Fatalf("retried permanent failure: calls=%d err=%v", calls, err)
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
calls := 0
|
||||
_, err := readBudgetUsage(ctx, tinyBudgetReadPolicy, func(context.Context) ([]byte, error) { calls++; cancel(); return nil, context.DeadlineExceeded })
|
||||
if !errors.Is(err, context.Canceled) || calls != 1 {
|
||||
t.Fatalf("retried after parent cancelled: calls=%d err=%v", calls, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgetUsageReadCancellationStopsBackoff(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
first := make(chan struct{})
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := readBudgetUsage(ctx, budgetReadPolicy{timeout: time.Second, retryDelay: time.Hour}, func(context.Context) ([]byte, error) { close(first); return nil, context.DeadlineExceeded })
|
||||
done <- err
|
||||
}()
|
||||
<-first
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancelled retry: %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("parent cancellation did not stop retry backoff")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgetWatchRetriesInitialAndFinalReadsWithoutReplayingWork(t *testing.T) {
|
||||
budget := NewBudget(0, 10)
|
||||
budget.pollInterval = time.Hour
|
||||
var reads atomic.Int32
|
||||
client := OpenCodeClient{Backend: "v2", budgetReadPolicy: tinyBudgetReadPolicy, command: func(ctx context.Context, _ string, args ...string) ([]byte, error) {
|
||||
if args[1] != "get" || args[2] != "/api/session/ses_budget" {
|
||||
t.Errorf("budget retry called non-accounting operation: %v", args)
|
||||
}
|
||||
switch reads.Add(1) {
|
||||
case 1, 3:
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
case 2:
|
||||
return usageResponse("ses_budget", 0, 0), nil
|
||||
default:
|
||||
return usageResponse("ses_budget", 0, 9), nil
|
||||
}
|
||||
}}
|
||||
ctx, watch, err := client.watchBudget(context.Background(), budget, "ses_budget")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var terminal error
|
||||
watch.stopWatch(ctx, &terminal)
|
||||
cancelled, cancel := context.WithCancel(ctx)
|
||||
cancel()
|
||||
watch.finish(cancelled, &terminal) // final read retains its independent bounded context
|
||||
if terminal != nil || reads.Load() != 4 || budget.Snapshot().UsedTokens != 9 {
|
||||
t.Fatalf("final accounting: reads=%d budget=%+v err=%v", reads.Load(), budget.Snapshot(), terminal)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgetWatchRejectsMalformedAndMissingAccountingWithoutRetry(t *testing.T) {
|
||||
for _, body := range []string{`garbage`, `{"data":{"id":"ses_budget"}}`, string(usageResponse("ses_other", 0, 1))} {
|
||||
calls := 0
|
||||
client := OpenCodeClient{Backend: "v2", budgetReadPolicy: tinyBudgetReadPolicy, command: func(context.Context, string, ...string) ([]byte, error) { calls++; return []byte(body), nil }}
|
||||
_, watch, err := client.watchBudget(context.Background(), NewBudget(0, 10), "ses_budget")
|
||||
if !errors.Is(err, ErrBudgetAccounting) || watch != nil || calls != 1 {
|
||||
t.Fatalf("invalid accounting was retried or bypassed: calls=%d watch=%v err=%v", calls, watch, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgetWatchPersistentReadTimeoutCancelsAndFinalReadStillEnforcesCap(t *testing.T) {
|
||||
budget := NewBudget(0, 10)
|
||||
budget.pollInterval = time.Millisecond
|
||||
var reads atomic.Int32
|
||||
client := OpenCodeClient{Backend: "v2", budgetReadPolicy: tinyBudgetReadPolicy, command: func(ctx context.Context, _ string, args ...string) ([]byte, error) {
|
||||
switch reads.Add(1) {
|
||||
case 1:
|
||||
return usageResponse("ses_budget", 0, 0), nil
|
||||
case 2, 3:
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
return usageResponse("ses_budget", 0, 12), nil
|
||||
}
|
||||
}}
|
||||
ctx, watch, err := client.watchBudget(context.Background(), budget, "ses_budget")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("persistent missing accounting did not stop work")
|
||||
}
|
||||
var terminal error
|
||||
watch.stopWatch(ctx, &terminal)
|
||||
watch.finish(ctx, &terminal)
|
||||
if !errors.Is(terminal, ErrBudgetAccounting) || !errors.Is(terminal, ErrBudgetExceeded) || budget.Snapshot().UsedTokens != 12 || reads.Load() != 4 {
|
||||
t.Fatalf("failed closed/final cap: calls=%d budget=%+v err=%v", reads.Load(), budget.Snapshot(), terminal)
|
||||
}
|
||||
}
|
||||
@@ -38,7 +38,8 @@ type OpenCodeClient struct {
|
||||
OnGenerationSession func(id, backend, reference string, active bool) error
|
||||
connection *opencode.Connection
|
||||
command func(context.Context, string, ...string) ([]byte, error)
|
||||
sessionPollInterval time.Duration // test seam; defaults to defaultSessionPollInterval
|
||||
sessionPollInterval time.Duration // test seam; defaults to defaultSessionPollInterval
|
||||
budgetReadPolicy budgetReadPolicy // test seam; zero uses bounded production defaults
|
||||
}
|
||||
|
||||
// ArtifactsEnv names the environment variable carrying a stage's artifacts
|
||||
|
||||
Reference in New Issue
Block a user