Files
pipelines/internal/stage/budget.go
T

293 lines
9.0 KiB
Go

package stage
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"math"
"strconv"
"sync"
"time"
)
const DefaultRetries = 2
const DefaultTimeout = time.Hour
var ErrBudgetExceeded = errors.New("stage budget exceeded")
var ErrBudgetAccounting = errors.New("stage budget accounting unavailable")
var ErrStageCleanup = errors.New("stage cleanup not confirmed")
func StageTimeout(def Definition) (time.Duration, error) {
if def.Configs["timeout"] == "" {
return DefaultTimeout, nil
}
duration, err := time.ParseDuration(def.Configs["timeout"])
if err != nil || duration <= 0 {
return 0, fmt.Errorf("timeout must be a positive duration, such as 1h or 30m")
}
return duration, nil
}
func parseBudget(def Definition) (float64, int64, error) {
var usd float64
var tokens int64
var err error
if value := def.Configs["budget_usd"]; value != "" {
usd, err = strconv.ParseFloat(value, 64)
if err != nil || usd <= 0 || math.IsNaN(usd) || math.IsInf(usd, 0) {
return 0, 0, fmt.Errorf("budget_usd must be a positive finite USD amount")
}
}
if value := def.Configs["budget_tokens"]; value != "" {
tokens, err = strconv.ParseInt(value, 10, 64)
if err != nil || tokens <= 0 {
return 0, 0, fmt.Errorf("budget_tokens must be a positive integer")
}
}
return usd, tokens, nil
}
type BudgetSnapshot struct {
LimitUSD float64 `json:"limit_usd,omitempty"`
LimitTokens int64 `json:"limit_tokens,omitempty"`
UsedUSD float64 `json:"used_usd"`
UsedTokens int64 `json:"used_tokens"`
USDKnown bool `json:"usd_known"`
TokensKnown bool `json:"tokens_known"`
Attempts int `json:"attempts"`
}
type reportedUsage struct {
USD *float64 `json:"cost"`
Tokens *struct {
Input, Output, Reasoning *int64
Cache *struct{ Read, Write *int64 }
} `json:"tokens"`
}
type Budget struct {
mu sync.Mutex
state BudgetSnapshot
records map[string]BudgetSnapshot
pollInterval time.Duration
}
func NewBudget(usd float64, tokens int64) *Budget {
return &Budget{state: BudgetSnapshot{LimitUSD: usd, LimitTokens: tokens}, records: map[string]BudgetSnapshot{}, pollInterval: time.Second}
}
func (b *Budget) Enabled() bool { return b != nil && (b.state.LimitUSD > 0 || b.state.LimitTokens > 0) }
func (b *Budget) Snapshot() BudgetSnapshot { b.mu.Lock(); defer b.mu.Unlock(); return b.state }
func (b *Budget) Attempt() { b.mu.Lock(); b.state.Attempts++; b.mu.Unlock() }
func (b *Budget) checkLocked() error {
if b.state.LimitUSD > 0 && b.state.UsedUSD >= b.state.LimitUSD {
return fmt.Errorf("%w: USD %.6f / %.6f", ErrBudgetExceeded, b.state.UsedUSD, b.state.LimitUSD)
}
if b.state.LimitTokens > 0 && b.state.UsedTokens >= b.state.LimitTokens {
return fmt.Errorf("%w: tokens %d / %d", ErrBudgetExceeded, b.state.UsedTokens, b.state.LimitTokens)
}
return nil
}
func (b *Budget) Check() error { b.mu.Lock(); defer b.mu.Unlock(); return b.checkLocked() }
func (b *Budget) observe(key string, usage reportedUsage) error {
b.mu.Lock()
defer b.mu.Unlock()
old := b.records[key]
defer func() { b.records[key] = old }()
if usage.USD != nil {
if *usage.USD < 0 || math.IsNaN(*usage.USD) || math.IsInf(*usage.USD, 0) {
return fmt.Errorf("%w: invalid cost", ErrBudgetAccounting)
}
value := max(old.UsedUSD, *usage.USD)
total := b.state.UsedUSD + (value - old.UsedUSD)
if math.IsNaN(total) || math.IsInf(total, 0) {
return fmt.Errorf("%w: cost overflow", ErrBudgetAccounting)
}
b.state.UsedUSD = total
old.UsedUSD, old.USDKnown, b.state.USDKnown = value, true, true
} else if b.state.LimitUSD > 0 {
return fmt.Errorf("%w: cost was not reported", ErrBudgetAccounting)
}
t := usage.Tokens
if t != nil && t.Input != nil && t.Output != nil && t.Reasoning != nil && t.Cache != nil && t.Cache.Read != nil && t.Cache.Write != nil {
var total int64
for _, value := range []int64{*t.Input, *t.Output, *t.Reasoning, *t.Cache.Read, *t.Cache.Write} {
if value < 0 || value > math.MaxInt64-total {
return fmt.Errorf("%w: invalid token count", ErrBudgetAccounting)
}
total += value
}
total = max(total, old.UsedTokens)
if total-old.UsedTokens > math.MaxInt64-b.state.UsedTokens {
return fmt.Errorf("%w: token count overflow", ErrBudgetAccounting)
}
b.state.UsedTokens += total - old.UsedTokens
old.UsedTokens, old.TokensKnown, b.state.TokensKnown = total, true, true
} else if b.state.LimitTokens > 0 {
return fmt.Errorf("%w: complete token usage was not reported", ErrBudgetAccounting)
}
return b.checkLocked()
}
type budgetContextKey struct{}
func WithBudget(ctx context.Context, budget *Budget) context.Context {
return context.WithValue(ctx, budgetContextKey{}, budget)
}
func budgetFromContext(ctx context.Context) *Budget {
b, _ := ctx.Value(budgetContextKey{}).(*Budget)
return b
}
type budgetWatch struct {
read func(context.Context, bool) error
cancel context.CancelCauseFunc
stop chan struct{}
done chan struct{}
}
func (c OpenCodeClient) BudgetCapable() bool { return true }
func (c OpenCodeClient) watchBudget(ctx context.Context, budget *Budget, id string) (context.Context, *budgetWatch, error) {
if !budget.Enabled() {
return ctx, nil, nil
}
if err := budget.Check(); err != nil {
return ctx, nil, err
}
// Poll through an independent reader. Native calls have per-request mutable
// diagnostic fields, which must not race the long-running prompt request.
reader := c
reader.Operations = nil
w := &budgetWatch{stop: make(chan struct{}), done: make(chan struct{})}
w.read = func(parent context.Context, final bool) error {
readCtx, cancel := context.WithTimeout(parent, 2*time.Second)
defer cancel()
var data []byte
var err error
if reader.connection != nil && reader.Backend == "v1" {
data, err = reader.connection.BudgetMessages(readCtx, id)
} else {
data, err = reader.call(readCtx, "get", "/api/session/"+id, nil)
}
if err != nil {
return fmt.Errorf("%w: usage read failed: %v", ErrBudgetAccounting, err)
}
return budget.readUsage(id, data, reader.Backend == "v1", final)
}
if err := w.read(ctx, false); err != nil {
return ctx, nil, err
}
workCtx, cancel := context.WithCancelCause(ctx)
w.cancel = cancel
go func() {
defer close(w.done)
ticker := time.NewTicker(budget.pollInterval)
defer ticker.Stop()
for {
select {
case <-w.stop:
return
case <-workCtx.Done():
return
case <-ticker.C:
if err := w.read(workCtx, false); err != nil {
if workCtx.Err() == nil {
cancel(err)
}
return
}
}
}
}()
return workCtx, w, nil
}
func (w *budgetWatch) stopWatch(ctx context.Context, err *error) {
if w == nil {
return
}
close(w.stop)
<-w.done
if cause := context.Cause(ctx); errors.Is(cause, ErrBudgetExceeded) || errors.Is(cause, ErrBudgetAccounting) {
*err = errors.Join(*err, cause)
}
}
func (w *budgetWatch) finish(ctx context.Context, err *error) {
if w == nil {
return
}
defer w.cancel(nil)
if usageErr := w.read(context.WithoutCancel(ctx), true); usageErr != nil {
*err = errors.Join(*err, usageErr)
}
}
func (b *Budget) readUsage(id string, data []byte, native bool, final ...bool) error {
if !native {
var response struct {
Data struct {
ID string `json:"id"`
reportedUsage
} `json:"data"`
}
if json.Unmarshal(data, &response) != nil || response.Data.ID != id {
return fmt.Errorf("%w: invalid or mismatched usage response", ErrBudgetAccounting)
}
return b.observe(id, response.Data.reportedUsage)
}
var messages []struct {
Info struct {
ID, Role, SessionID string
Time struct{ Completed int64 }
reportedUsage
}
}
if len(bytes.TrimSpace(data)) == 0 || bytes.TrimSpace(data)[0] != '[' || json.Unmarshal(data, &messages) != nil {
return fmt.Errorf("%w: invalid native usage response", ErrBudgetAccounting)
}
var accountingErr error
completed := 0
for _, m := range messages {
if m.Info.Role != "assistant" {
continue
}
if m.Info.Time.Completed != 0 {
completed++
}
if m.Info.Time.Completed == 0 && (m.Info.USD == nil || m.Info.Tokens == nil) {
continue
}
if m.Info.ID == "" || (m.Info.SessionID != "" && m.Info.SessionID != id) {
return fmt.Errorf("%w: invalid native message identity", ErrBudgetAccounting)
}
if err := b.observe(id+"/"+m.Info.ID, m.Info.reportedUsage); err != nil {
if errors.Is(err, ErrBudgetAccounting) && accountingErr == nil {
accountingErr = err
}
}
}
if accountingErr != nil {
return accountingErr
}
if len(final) > 0 && final[0] && completed == 0 {
return fmt.Errorf("%w: no completed assistant usage was reported", ErrBudgetAccounting)
}
return b.Check()
}
func BudgetTranscript(original string, budget *Budget, timeout time.Duration, retries int) string {
var data map[string]json.RawMessage
if json.Unmarshal([]byte(original), &data) != nil || data == nil {
data = map[string]json.RawMessage{}
if original != "" {
data["original_transcript"], _ = json.Marshal(original)
}
}
data["stage_limits"], _ = json.Marshal(map[string]any{"timeout": timeout.String(), "retries": retries, "budget": budget.Snapshot()})
encoded, _ := json.Marshal(data)
return string(encoded)
}