396 lines
14 KiB
Go
396 lines
14 KiB
Go
package stage
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"math"
|
|
"strconv"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
const DefaultRetries = 2
|
|
const DefaultTimeout = time.Hour
|
|
|
|
var ErrBudgetExceeded = errors.New("stage budget exceeded")
|
|
var ErrBudgetAccounting = errors.New("stage budget accounting unavailable")
|
|
var ErrStageCleanup = errors.New("stage cleanup not confirmed")
|
|
|
|
func StageTimeout(def Definition) (time.Duration, error) {
|
|
if def.Configs["timeout"] == "" {
|
|
return DefaultTimeout, nil
|
|
}
|
|
duration, err := time.ParseDuration(def.Configs["timeout"])
|
|
if err != nil || duration <= 0 {
|
|
return 0, fmt.Errorf("timeout must be a positive duration, such as 1h or 30m")
|
|
}
|
|
return duration, nil
|
|
}
|
|
|
|
func parseBudget(def Definition) (float64, int64, error) {
|
|
var usd float64
|
|
var tokens int64
|
|
var err error
|
|
if value := def.Configs["budget_usd"]; value != "" {
|
|
usd, err = strconv.ParseFloat(value, 64)
|
|
if err != nil || usd <= 0 || math.IsNaN(usd) || math.IsInf(usd, 0) {
|
|
return 0, 0, fmt.Errorf("budget_usd must be a positive finite USD amount")
|
|
}
|
|
}
|
|
if value := def.Configs["budget_tokens"]; value != "" {
|
|
tokens, err = strconv.ParseInt(value, 10, 64)
|
|
if err != nil || tokens <= 0 {
|
|
return 0, 0, fmt.Errorf("budget_tokens must be a positive integer")
|
|
}
|
|
}
|
|
return usd, tokens, nil
|
|
}
|
|
|
|
type BudgetSnapshot struct {
|
|
LimitUSD float64 `json:"limit_usd,omitempty"`
|
|
LimitTokens int64 `json:"limit_tokens,omitempty"`
|
|
UsedUSD float64 `json:"used_usd"`
|
|
UsedTokens int64 `json:"used_tokens"`
|
|
USDKnown bool `json:"usd_known"`
|
|
TokensKnown bool `json:"tokens_known"`
|
|
Attempts int `json:"attempts"`
|
|
// Component token counts the harness reported, accumulated with the same
|
|
// per-key max/delta accounting as UsedTokens. They are meaningful only when
|
|
// SplitKnown is true: a harness that reports only a total (Pi's
|
|
// get_session_stats) leaves them at zero with SplitKnown false, so its total
|
|
// is never presented as input tokens.
|
|
TokensInput int64 `json:"tokens_input,omitempty"`
|
|
TokensOutput int64 `json:"tokens_output,omitempty"`
|
|
TokensReasoning int64 `json:"tokens_reasoning,omitempty"`
|
|
CacheRead int64 `json:"cache_read,omitempty"`
|
|
CacheWrite int64 `json:"cache_write,omitempty"`
|
|
SplitKnown bool `json:"split_known,omitempty"`
|
|
// Report counters advance only when a harness report of that kind is
|
|
// observed. Since uses them to tell a real increment from a visit that
|
|
// observed nothing: a revisit that reports no new tokens or cost must not
|
|
// present the budget's unchanged known flags as a known zero for that visit.
|
|
// They are internal bookkeeping and stay out of the audit snapshot.
|
|
TokenReports int `json:"-"`
|
|
CostReports int `json:"-"`
|
|
SplitReports int `json:"-"`
|
|
}
|
|
|
|
type reportedUsage struct {
|
|
USD *float64 `json:"cost"`
|
|
Tokens *struct {
|
|
Input, Output, Reasoning *int64
|
|
Cache *struct{ Read, Write *int64 }
|
|
// Total is a reported total with no component split (Pi). Exactly one of
|
|
// the component set or Total is expected; components win when both appear.
|
|
Total *int64
|
|
} `json:"tokens"`
|
|
}
|
|
|
|
type Budget struct {
|
|
mu sync.Mutex
|
|
state BudgetSnapshot
|
|
records map[string]BudgetSnapshot
|
|
pollInterval time.Duration
|
|
}
|
|
|
|
func NewBudget(usd float64, tokens int64) *Budget {
|
|
return &Budget{state: BudgetSnapshot{LimitUSD: usd, LimitTokens: tokens}, records: map[string]BudgetSnapshot{}, pollInterval: time.Second}
|
|
}
|
|
|
|
// RestoreUsage seeds persisted earlier visits when a durable handoff resumes.
|
|
// New sessions have new accounting keys; prior records must not be polled again.
|
|
func (b *Budget) RestoreUsage(used BudgetSnapshot) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
used.LimitUSD, used.LimitTokens = b.state.LimitUSD, b.state.LimitTokens
|
|
b.state = used
|
|
}
|
|
func (b *Budget) Enabled() bool { return b != nil && (b.state.LimitUSD > 0 || b.state.LimitTokens > 0) }
|
|
func (b *Budget) Snapshot() BudgetSnapshot { b.mu.Lock(); defer b.mu.Unlock(); return b.state }
|
|
|
|
// Since returns only the usage this budget added after an earlier snapshot of
|
|
// itself. A budget is shared across every visit to a stage key, so recording its
|
|
// cumulative Snapshot() on each visit would count the earlier visits' usage
|
|
// again; a visit records Snapshot().Since(before) so the run total sums to what
|
|
// the provider actually reported. The counts and cost only accumulate, so the
|
|
// deltas are non-negative.
|
|
//
|
|
// The known flags describe the increment, not the later cumulative snapshot: a
|
|
// revisit that observed no new report of a kind (a failure or timeout before any
|
|
// usage poll, or a Pi visit that never reached get_session_stats) must not
|
|
// present the unchanged known flag as a known zero. Such a kind is reported as
|
|
// unknown for this visit, so the caller records no usage at all rather than a
|
|
// fabricated zero cost or token count.
|
|
func (s BudgetSnapshot) Since(before BudgetSnapshot) BudgetSnapshot {
|
|
after := s
|
|
after.UsedUSD -= before.UsedUSD
|
|
after.UsedTokens -= before.UsedTokens
|
|
after.TokensInput -= before.TokensInput
|
|
after.TokensOutput -= before.TokensOutput
|
|
after.TokensReasoning -= before.TokensReasoning
|
|
after.CacheRead -= before.CacheRead
|
|
after.CacheWrite -= before.CacheWrite
|
|
if after.CostReports == before.CostReports {
|
|
after.USDKnown = false
|
|
}
|
|
if after.TokenReports == before.TokenReports {
|
|
after.TokensKnown = false
|
|
}
|
|
if after.SplitReports == before.SplitReports {
|
|
after.SplitKnown = false
|
|
}
|
|
return after
|
|
}
|
|
func (b *Budget) Attempt() { b.mu.Lock(); b.state.Attempts++; b.mu.Unlock() }
|
|
func (b *Budget) checkLocked() error {
|
|
if b.state.LimitUSD > 0 && b.state.UsedUSD >= b.state.LimitUSD {
|
|
return fmt.Errorf("%w: USD %.6f / %.6f", ErrBudgetExceeded, b.state.UsedUSD, b.state.LimitUSD)
|
|
}
|
|
if b.state.LimitTokens > 0 && b.state.UsedTokens >= b.state.LimitTokens {
|
|
return fmt.Errorf("%w: tokens %d / %d", ErrBudgetExceeded, b.state.UsedTokens, b.state.LimitTokens)
|
|
}
|
|
return nil
|
|
}
|
|
func (b *Budget) Check() error { b.mu.Lock(); defer b.mu.Unlock(); return b.checkLocked() }
|
|
|
|
func (b *Budget) observe(key string, usage reportedUsage) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
old := b.records[key]
|
|
defer func() { b.records[key] = old }()
|
|
if usage.USD != nil {
|
|
if *usage.USD < 0 || math.IsNaN(*usage.USD) || math.IsInf(*usage.USD, 0) {
|
|
return fmt.Errorf("%w: invalid cost", ErrBudgetAccounting)
|
|
}
|
|
value := max(old.UsedUSD, *usage.USD)
|
|
total := b.state.UsedUSD + (value - old.UsedUSD)
|
|
if math.IsNaN(total) || math.IsInf(total, 0) {
|
|
return fmt.Errorf("%w: cost overflow", ErrBudgetAccounting)
|
|
}
|
|
b.state.UsedUSD = total
|
|
old.UsedUSD, old.USDKnown, b.state.USDKnown = value, true, true
|
|
b.state.CostReports++
|
|
} else if b.state.LimitUSD > 0 {
|
|
return fmt.Errorf("%w: cost was not reported", ErrBudgetAccounting)
|
|
}
|
|
t := usage.Tokens
|
|
components := t != nil && t.Input != nil && t.Output != nil && t.Reasoning != nil && t.Cache != nil && t.Cache.Read != nil && t.Cache.Write != nil
|
|
switch {
|
|
case components:
|
|
var total int64
|
|
for _, value := range []int64{*t.Input, *t.Output, *t.Reasoning, *t.Cache.Read, *t.Cache.Write} {
|
|
if value < 0 || value > math.MaxInt64-total {
|
|
return fmt.Errorf("%w: invalid token count", ErrBudgetAccounting)
|
|
}
|
|
total += value
|
|
}
|
|
total = max(total, old.UsedTokens)
|
|
if total-old.UsedTokens > math.MaxInt64-b.state.UsedTokens {
|
|
return fmt.Errorf("%w: token count overflow", ErrBudgetAccounting)
|
|
}
|
|
b.state.UsedTokens += total - old.UsedTokens
|
|
old.UsedTokens, old.TokensKnown, b.state.TokensKnown = total, true, true
|
|
// Component tokens use the same per-key max/delta accounting so a repeated
|
|
// poll cannot double count. The split is recorded only when the harness
|
|
// actually reported separate components, never synthesised from a total.
|
|
for _, component := range []struct {
|
|
reported int64
|
|
state *int64
|
|
record *int64
|
|
}{
|
|
{*t.Input, &b.state.TokensInput, &old.TokensInput},
|
|
{*t.Output, &b.state.TokensOutput, &old.TokensOutput},
|
|
{*t.Reasoning, &b.state.TokensReasoning, &old.TokensReasoning},
|
|
{*t.Cache.Read, &b.state.CacheRead, &old.CacheRead},
|
|
{*t.Cache.Write, &b.state.CacheWrite, &old.CacheWrite},
|
|
} {
|
|
value := max(component.reported, *component.record)
|
|
*component.state += value - *component.record
|
|
*component.record = value
|
|
}
|
|
old.SplitKnown, b.state.SplitKnown = true, true
|
|
b.state.TokenReports++
|
|
b.state.SplitReports++
|
|
case t != nil && t.Total != nil:
|
|
// A total-only report (Pi) counts toward the token total and the limit but
|
|
// leaves the in/out split unknown rather than presenting the total as
|
|
// input tokens.
|
|
if *t.Total < 0 {
|
|
return fmt.Errorf("%w: invalid token count", ErrBudgetAccounting)
|
|
}
|
|
total := max(*t.Total, old.UsedTokens)
|
|
if total-old.UsedTokens > math.MaxInt64-b.state.UsedTokens {
|
|
return fmt.Errorf("%w: token count overflow", ErrBudgetAccounting)
|
|
}
|
|
b.state.UsedTokens += total - old.UsedTokens
|
|
old.UsedTokens, old.TokensKnown, b.state.TokensKnown = total, true, true
|
|
b.state.TokenReports++
|
|
case b.state.LimitTokens > 0:
|
|
return fmt.Errorf("%w: complete token usage was not reported", ErrBudgetAccounting)
|
|
}
|
|
return b.checkLocked()
|
|
}
|
|
|
|
type budgetContextKey struct{}
|
|
|
|
func WithBudget(ctx context.Context, budget *Budget) context.Context {
|
|
return context.WithValue(ctx, budgetContextKey{}, budget)
|
|
}
|
|
func budgetFromContext(ctx context.Context) *Budget {
|
|
b, _ := ctx.Value(budgetContextKey{}).(*Budget)
|
|
return b
|
|
}
|
|
|
|
type budgetWatch struct {
|
|
read func(context.Context, bool) error
|
|
cancel context.CancelCauseFunc
|
|
stop chan struct{}
|
|
done chan struct{}
|
|
}
|
|
|
|
func (c OpenCodeClient) BudgetCapable() bool { return true }
|
|
|
|
func (c OpenCodeClient) watchBudget(ctx context.Context, budget *Budget, id string) (context.Context, *budgetWatch, error) {
|
|
if !budget.Enabled() {
|
|
return ctx, nil, nil
|
|
}
|
|
if err := budget.Check(); err != nil {
|
|
return ctx, nil, err
|
|
}
|
|
// Poll through an independent reader. Native calls have per-request mutable
|
|
// diagnostic fields, which must not race the long-running prompt request.
|
|
reader := c
|
|
reader.Operations = nil
|
|
w := &budgetWatch{stop: make(chan struct{}), done: make(chan struct{})}
|
|
w.read = func(parent context.Context, final bool) error {
|
|
readCtx, cancel := context.WithTimeout(parent, 2*time.Second)
|
|
defer cancel()
|
|
var data []byte
|
|
var err error
|
|
if reader.connection != nil && reader.Backend == "v1" {
|
|
data, err = reader.connection.BudgetMessages(readCtx, id)
|
|
} else {
|
|
data, err = reader.call(readCtx, "get", "/api/session/"+id, nil)
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("%w: usage read failed: %v", ErrBudgetAccounting, err)
|
|
}
|
|
return budget.readUsage(id, data, reader.Backend == "v1", final)
|
|
}
|
|
if err := w.read(ctx, false); err != nil {
|
|
return ctx, nil, err
|
|
}
|
|
workCtx, cancel := context.WithCancelCause(ctx)
|
|
w.cancel = cancel
|
|
go func() {
|
|
defer close(w.done)
|
|
ticker := time.NewTicker(budget.pollInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-w.stop:
|
|
return
|
|
case <-workCtx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
if err := w.read(workCtx, false); err != nil {
|
|
if workCtx.Err() == nil {
|
|
cancel(err)
|
|
}
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
return workCtx, w, nil
|
|
}
|
|
|
|
func (w *budgetWatch) stopWatch(ctx context.Context, err *error) {
|
|
if w == nil {
|
|
return
|
|
}
|
|
close(w.stop)
|
|
<-w.done
|
|
if cause := context.Cause(ctx); errors.Is(cause, ErrBudgetExceeded) || errors.Is(cause, ErrBudgetAccounting) {
|
|
*err = errors.Join(*err, cause)
|
|
}
|
|
}
|
|
func (w *budgetWatch) finish(ctx context.Context, err *error) {
|
|
if w == nil {
|
|
return
|
|
}
|
|
defer w.cancel(nil)
|
|
if usageErr := w.read(context.WithoutCancel(ctx), true); usageErr != nil {
|
|
*err = errors.Join(*err, usageErr)
|
|
}
|
|
}
|
|
|
|
func (b *Budget) readUsage(id string, data []byte, native bool, final ...bool) error {
|
|
if !native {
|
|
var response struct {
|
|
Data struct {
|
|
ID string `json:"id"`
|
|
reportedUsage
|
|
} `json:"data"`
|
|
}
|
|
if json.Unmarshal(data, &response) != nil || response.Data.ID != id {
|
|
return fmt.Errorf("%w: invalid or mismatched usage response", ErrBudgetAccounting)
|
|
}
|
|
return b.observe(id, response.Data.reportedUsage)
|
|
}
|
|
var messages []struct {
|
|
Info struct {
|
|
ID, Role, SessionID string
|
|
Time struct{ Completed int64 }
|
|
reportedUsage
|
|
}
|
|
}
|
|
if len(bytes.TrimSpace(data)) == 0 || bytes.TrimSpace(data)[0] != '[' || json.Unmarshal(data, &messages) != nil {
|
|
return fmt.Errorf("%w: invalid native usage response", ErrBudgetAccounting)
|
|
}
|
|
var accountingErr error
|
|
completed := 0
|
|
for _, m := range messages {
|
|
if m.Info.Role != "assistant" {
|
|
continue
|
|
}
|
|
if m.Info.Time.Completed != 0 {
|
|
completed++
|
|
}
|
|
if m.Info.Time.Completed == 0 && (m.Info.USD == nil || m.Info.Tokens == nil) {
|
|
continue
|
|
}
|
|
if m.Info.ID == "" || (m.Info.SessionID != "" && m.Info.SessionID != id) {
|
|
return fmt.Errorf("%w: invalid native message identity", ErrBudgetAccounting)
|
|
}
|
|
if err := b.observe(id+"/"+m.Info.ID, m.Info.reportedUsage); err != nil {
|
|
if errors.Is(err, ErrBudgetAccounting) && accountingErr == nil {
|
|
accountingErr = err
|
|
}
|
|
}
|
|
}
|
|
if accountingErr != nil {
|
|
return accountingErr
|
|
}
|
|
if len(final) > 0 && final[0] && completed == 0 {
|
|
return fmt.Errorf("%w: no completed assistant usage was reported", ErrBudgetAccounting)
|
|
}
|
|
return b.Check()
|
|
}
|
|
|
|
func BudgetTranscript(original string, budget *Budget, timeout time.Duration, retries int) string {
|
|
var data map[string]json.RawMessage
|
|
if json.Unmarshal([]byte(original), &data) != nil || data == nil {
|
|
data = map[string]json.RawMessage{}
|
|
if original != "" {
|
|
data["original_transcript"], _ = json.Marshal(original)
|
|
}
|
|
}
|
|
data["stage_limits"], _ = json.Marshal(map[string]any{"timeout": timeout.String(), "retries": retries, "budget": budget.Snapshot()})
|
|
encoded, _ := json.Marshal(data)
|
|
return string(encoded)
|
|
}
|