Files
pipelines/internal/stage/budget.go
T

396 lines
14 KiB
Go

package stage
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"math"
"strconv"
"sync"
"time"
)
const DefaultRetries = 2
const DefaultTimeout = time.Hour
var ErrBudgetExceeded = errors.New("stage budget exceeded")
var ErrBudgetAccounting = errors.New("stage budget accounting unavailable")
var ErrStageCleanup = errors.New("stage cleanup not confirmed")
func StageTimeout(def Definition) (time.Duration, error) {
if def.Configs["timeout"] == "" {
return DefaultTimeout, nil
}
duration, err := time.ParseDuration(def.Configs["timeout"])
if err != nil || duration <= 0 {
return 0, fmt.Errorf("timeout must be a positive duration, such as 1h or 30m")
}
return duration, nil
}
func parseBudget(def Definition) (float64, int64, error) {
var usd float64
var tokens int64
var err error
if value := def.Configs["budget_usd"]; value != "" {
usd, err = strconv.ParseFloat(value, 64)
if err != nil || usd <= 0 || math.IsNaN(usd) || math.IsInf(usd, 0) {
return 0, 0, fmt.Errorf("budget_usd must be a positive finite USD amount")
}
}
if value := def.Configs["budget_tokens"]; value != "" {
tokens, err = strconv.ParseInt(value, 10, 64)
if err != nil || tokens <= 0 {
return 0, 0, fmt.Errorf("budget_tokens must be a positive integer")
}
}
return usd, tokens, nil
}
type BudgetSnapshot struct {
LimitUSD float64 `json:"limit_usd,omitempty"`
LimitTokens int64 `json:"limit_tokens,omitempty"`
UsedUSD float64 `json:"used_usd"`
UsedTokens int64 `json:"used_tokens"`
USDKnown bool `json:"usd_known"`
TokensKnown bool `json:"tokens_known"`
Attempts int `json:"attempts"`
// Component token counts the harness reported, accumulated with the same
// per-key max/delta accounting as UsedTokens. They are meaningful only when
// SplitKnown is true: a harness that reports only a total (Pi's
// get_session_stats) leaves them at zero with SplitKnown false, so its total
// is never presented as input tokens.
TokensInput int64 `json:"tokens_input,omitempty"`
TokensOutput int64 `json:"tokens_output,omitempty"`
TokensReasoning int64 `json:"tokens_reasoning,omitempty"`
CacheRead int64 `json:"cache_read,omitempty"`
CacheWrite int64 `json:"cache_write,omitempty"`
SplitKnown bool `json:"split_known,omitempty"`
// Report counters advance only when a harness report of that kind is
// observed. Since uses them to tell a real increment from a visit that
// observed nothing: a revisit that reports no new tokens or cost must not
// present the budget's unchanged known flags as a known zero for that visit.
// They are internal bookkeeping and stay out of the audit snapshot.
TokenReports int `json:"-"`
CostReports int `json:"-"`
SplitReports int `json:"-"`
}
type reportedUsage struct {
USD *float64 `json:"cost"`
Tokens *struct {
Input, Output, Reasoning *int64
Cache *struct{ Read, Write *int64 }
// Total is a reported total with no component split (Pi). Exactly one of
// the component set or Total is expected; components win when both appear.
Total *int64
} `json:"tokens"`
}
type Budget struct {
mu sync.Mutex
state BudgetSnapshot
records map[string]BudgetSnapshot
pollInterval time.Duration
}
func NewBudget(usd float64, tokens int64) *Budget {
return &Budget{state: BudgetSnapshot{LimitUSD: usd, LimitTokens: tokens}, records: map[string]BudgetSnapshot{}, pollInterval: time.Second}
}
// RestoreUsage seeds persisted earlier visits when a durable handoff resumes.
// New sessions have new accounting keys; prior records must not be polled again.
func (b *Budget) RestoreUsage(used BudgetSnapshot) {
b.mu.Lock()
defer b.mu.Unlock()
used.LimitUSD, used.LimitTokens = b.state.LimitUSD, b.state.LimitTokens
b.state = used
}
func (b *Budget) Enabled() bool { return b != nil && (b.state.LimitUSD > 0 || b.state.LimitTokens > 0) }
func (b *Budget) Snapshot() BudgetSnapshot { b.mu.Lock(); defer b.mu.Unlock(); return b.state }
// Since returns only the usage this budget added after an earlier snapshot of
// itself. A budget is shared across every visit to a stage key, so recording its
// cumulative Snapshot() on each visit would count the earlier visits' usage
// again; a visit records Snapshot().Since(before) so the run total sums to what
// the provider actually reported. The counts and cost only accumulate, so the
// deltas are non-negative.
//
// The known flags describe the increment, not the later cumulative snapshot: a
// revisit that observed no new report of a kind (a failure or timeout before any
// usage poll, or a Pi visit that never reached get_session_stats) must not
// present the unchanged known flag as a known zero. Such a kind is reported as
// unknown for this visit, so the caller records no usage at all rather than a
// fabricated zero cost or token count.
func (s BudgetSnapshot) Since(before BudgetSnapshot) BudgetSnapshot {
after := s
after.UsedUSD -= before.UsedUSD
after.UsedTokens -= before.UsedTokens
after.TokensInput -= before.TokensInput
after.TokensOutput -= before.TokensOutput
after.TokensReasoning -= before.TokensReasoning
after.CacheRead -= before.CacheRead
after.CacheWrite -= before.CacheWrite
if after.CostReports == before.CostReports {
after.USDKnown = false
}
if after.TokenReports == before.TokenReports {
after.TokensKnown = false
}
if after.SplitReports == before.SplitReports {
after.SplitKnown = false
}
return after
}
func (b *Budget) Attempt() { b.mu.Lock(); b.state.Attempts++; b.mu.Unlock() }
func (b *Budget) checkLocked() error {
if b.state.LimitUSD > 0 && b.state.UsedUSD >= b.state.LimitUSD {
return fmt.Errorf("%w: USD %.6f / %.6f", ErrBudgetExceeded, b.state.UsedUSD, b.state.LimitUSD)
}
if b.state.LimitTokens > 0 && b.state.UsedTokens >= b.state.LimitTokens {
return fmt.Errorf("%w: tokens %d / %d", ErrBudgetExceeded, b.state.UsedTokens, b.state.LimitTokens)
}
return nil
}
func (b *Budget) Check() error { b.mu.Lock(); defer b.mu.Unlock(); return b.checkLocked() }
func (b *Budget) observe(key string, usage reportedUsage) error {
b.mu.Lock()
defer b.mu.Unlock()
old := b.records[key]
defer func() { b.records[key] = old }()
if usage.USD != nil {
if *usage.USD < 0 || math.IsNaN(*usage.USD) || math.IsInf(*usage.USD, 0) {
return fmt.Errorf("%w: invalid cost", ErrBudgetAccounting)
}
value := max(old.UsedUSD, *usage.USD)
total := b.state.UsedUSD + (value - old.UsedUSD)
if math.IsNaN(total) || math.IsInf(total, 0) {
return fmt.Errorf("%w: cost overflow", ErrBudgetAccounting)
}
b.state.UsedUSD = total
old.UsedUSD, old.USDKnown, b.state.USDKnown = value, true, true
b.state.CostReports++
} else if b.state.LimitUSD > 0 {
return fmt.Errorf("%w: cost was not reported", ErrBudgetAccounting)
}
t := usage.Tokens
components := t != nil && t.Input != nil && t.Output != nil && t.Reasoning != nil && t.Cache != nil && t.Cache.Read != nil && t.Cache.Write != nil
switch {
case components:
var total int64
for _, value := range []int64{*t.Input, *t.Output, *t.Reasoning, *t.Cache.Read, *t.Cache.Write} {
if value < 0 || value > math.MaxInt64-total {
return fmt.Errorf("%w: invalid token count", ErrBudgetAccounting)
}
total += value
}
total = max(total, old.UsedTokens)
if total-old.UsedTokens > math.MaxInt64-b.state.UsedTokens {
return fmt.Errorf("%w: token count overflow", ErrBudgetAccounting)
}
b.state.UsedTokens += total - old.UsedTokens
old.UsedTokens, old.TokensKnown, b.state.TokensKnown = total, true, true
// Component tokens use the same per-key max/delta accounting so a repeated
// poll cannot double count. The split is recorded only when the harness
// actually reported separate components, never synthesised from a total.
for _, component := range []struct {
reported int64
state *int64
record *int64
}{
{*t.Input, &b.state.TokensInput, &old.TokensInput},
{*t.Output, &b.state.TokensOutput, &old.TokensOutput},
{*t.Reasoning, &b.state.TokensReasoning, &old.TokensReasoning},
{*t.Cache.Read, &b.state.CacheRead, &old.CacheRead},
{*t.Cache.Write, &b.state.CacheWrite, &old.CacheWrite},
} {
value := max(component.reported, *component.record)
*component.state += value - *component.record
*component.record = value
}
old.SplitKnown, b.state.SplitKnown = true, true
b.state.TokenReports++
b.state.SplitReports++
case t != nil && t.Total != nil:
// A total-only report (Pi) counts toward the token total and the limit but
// leaves the in/out split unknown rather than presenting the total as
// input tokens.
if *t.Total < 0 {
return fmt.Errorf("%w: invalid token count", ErrBudgetAccounting)
}
total := max(*t.Total, old.UsedTokens)
if total-old.UsedTokens > math.MaxInt64-b.state.UsedTokens {
return fmt.Errorf("%w: token count overflow", ErrBudgetAccounting)
}
b.state.UsedTokens += total - old.UsedTokens
old.UsedTokens, old.TokensKnown, b.state.TokensKnown = total, true, true
b.state.TokenReports++
case b.state.LimitTokens > 0:
return fmt.Errorf("%w: complete token usage was not reported", ErrBudgetAccounting)
}
return b.checkLocked()
}
type budgetContextKey struct{}
func WithBudget(ctx context.Context, budget *Budget) context.Context {
return context.WithValue(ctx, budgetContextKey{}, budget)
}
func budgetFromContext(ctx context.Context) *Budget {
b, _ := ctx.Value(budgetContextKey{}).(*Budget)
return b
}
type budgetWatch struct {
read func(context.Context, bool) error
cancel context.CancelCauseFunc
stop chan struct{}
done chan struct{}
}
func (c OpenCodeClient) BudgetCapable() bool { return true }
func (c OpenCodeClient) watchBudget(ctx context.Context, budget *Budget, id string) (context.Context, *budgetWatch, error) {
if !budget.Enabled() {
return ctx, nil, nil
}
if err := budget.Check(); err != nil {
return ctx, nil, err
}
// Poll through an independent reader. Native calls have per-request mutable
// diagnostic fields, which must not race the long-running prompt request.
reader := c
reader.Operations = nil
w := &budgetWatch{stop: make(chan struct{}), done: make(chan struct{})}
w.read = func(parent context.Context, final bool) error {
readCtx, cancel := context.WithTimeout(parent, 2*time.Second)
defer cancel()
var data []byte
var err error
if reader.connection != nil && reader.Backend == "v1" {
data, err = reader.connection.BudgetMessages(readCtx, id)
} else {
data, err = reader.call(readCtx, "get", "/api/session/"+id, nil)
}
if err != nil {
return fmt.Errorf("%w: usage read failed: %v", ErrBudgetAccounting, err)
}
return budget.readUsage(id, data, reader.Backend == "v1", final)
}
if err := w.read(ctx, false); err != nil {
return ctx, nil, err
}
workCtx, cancel := context.WithCancelCause(ctx)
w.cancel = cancel
go func() {
defer close(w.done)
ticker := time.NewTicker(budget.pollInterval)
defer ticker.Stop()
for {
select {
case <-w.stop:
return
case <-workCtx.Done():
return
case <-ticker.C:
if err := w.read(workCtx, false); err != nil {
if workCtx.Err() == nil {
cancel(err)
}
return
}
}
}
}()
return workCtx, w, nil
}
func (w *budgetWatch) stopWatch(ctx context.Context, err *error) {
if w == nil {
return
}
close(w.stop)
<-w.done
if cause := context.Cause(ctx); errors.Is(cause, ErrBudgetExceeded) || errors.Is(cause, ErrBudgetAccounting) {
*err = errors.Join(*err, cause)
}
}
func (w *budgetWatch) finish(ctx context.Context, err *error) {
if w == nil {
return
}
defer w.cancel(nil)
if usageErr := w.read(context.WithoutCancel(ctx), true); usageErr != nil {
*err = errors.Join(*err, usageErr)
}
}
func (b *Budget) readUsage(id string, data []byte, native bool, final ...bool) error {
if !native {
var response struct {
Data struct {
ID string `json:"id"`
reportedUsage
} `json:"data"`
}
if json.Unmarshal(data, &response) != nil || response.Data.ID != id {
return fmt.Errorf("%w: invalid or mismatched usage response", ErrBudgetAccounting)
}
return b.observe(id, response.Data.reportedUsage)
}
var messages []struct {
Info struct {
ID, Role, SessionID string
Time struct{ Completed int64 }
reportedUsage
}
}
if len(bytes.TrimSpace(data)) == 0 || bytes.TrimSpace(data)[0] != '[' || json.Unmarshal(data, &messages) != nil {
return fmt.Errorf("%w: invalid native usage response", ErrBudgetAccounting)
}
var accountingErr error
completed := 0
for _, m := range messages {
if m.Info.Role != "assistant" {
continue
}
if m.Info.Time.Completed != 0 {
completed++
}
if m.Info.Time.Completed == 0 && (m.Info.USD == nil || m.Info.Tokens == nil) {
continue
}
if m.Info.ID == "" || (m.Info.SessionID != "" && m.Info.SessionID != id) {
return fmt.Errorf("%w: invalid native message identity", ErrBudgetAccounting)
}
if err := b.observe(id+"/"+m.Info.ID, m.Info.reportedUsage); err != nil {
if errors.Is(err, ErrBudgetAccounting) && accountingErr == nil {
accountingErr = err
}
}
}
if accountingErr != nil {
return accountingErr
}
if len(final) > 0 && final[0] && completed == 0 {
return fmt.Errorf("%w: no completed assistant usage was reported", ErrBudgetAccounting)
}
return b.Check()
}
func BudgetTranscript(original string, budget *Budget, timeout time.Duration, retries int) string {
var data map[string]json.RawMessage
if json.Unmarshal([]byte(original), &data) != nil || data == nil {
data = map[string]json.RawMessage{}
if original != "" {
data["original_transcript"], _ = json.Marshal(original)
}
}
data["stage_limits"], _ = json.Marshal(map[string]any{"timeout": timeout.String(), "retries": retries, "budget": budget.Snapshot()})
encoded, _ := json.Marshal(data)
return string(encoded)
}