Files
pipelines/internal/pipeline/usage_test.go
T
alan 3f5c11b7bf Record and expose per-run model usage (tokens, split, cost)
Carry the usage a model stage already measures for its budget
(stage.Budget.Snapshot()) onto the step as typed evidence and total it in
one pass, rather than a second harness read, a transcript parse or an
output-length estimate. A shared budget across stage revisits records only
the visit's increment, and a revisit that observes no new report records no
usage instead of a fabricated known zero.

- protocol: StageUsage/RunUsage/TotalUsage and UsageTotals.
- pipeline: step_usage table + migration, COALESCEd into the one step
  projection, FinishStep persistence, one json_extract history aggregate,
  retention removes the rows with their run.
- server: stageUsage maps the budget snapshot, stamping per-stage model and
  usage for every model stage (including failures).
- cli: inspect/stats human and JSON output.
- budget/pi: retain the reported in/out split, mark a total-only (Pi) report
  split-unknown, and keep an unreported cost unknown (never zero).

Evidence and docs updated in the packaged skill references and docs/cli.md;
the live database has no step_usage rows yet (all history predates this), so
per-run figures are reported as unavailable, not zero.
2026-10-17 20:40:56 +01:00

193 lines
7.1 KiB
Go

package pipeline
import (
"context"
"path/filepath"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// finishUsageStep begins and finishes one completed step carrying the given
// reported usage, the same store path the runner uses.
func finishUsageStep(t *testing.T, store *Store, run int64, sequence int, key string, usage *protocol.StageUsage) {
t.Helper()
if err := store.BeginStep(context.Background(), run, sequence, key, nil); err != nil {
t.Fatal(err)
}
if err := store.FinishStep(context.Background(), run, protocol.RunStep{
Sequence: sequence, Stage: key, Status: "completed", Outcome: "success", Target: "#success",
Output: map[string]any{"summary": "done"}, Usage: usage,
}); err != nil {
t.Fatal(err)
}
}
// TestStepUsageRoundTripRunTotalAndStatsSum proves per-step usage round-trips
// through the store, the run total is the single-pass sum of the reported stages,
// and the history-wide aggregate equals the sum of the per-run figures over the
// same rows.
func TestStepUsageRoundTripRunTotalAndStatsSum(t *testing.T) {
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), "usage.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "usage", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
// Run 1: a split-cost stage and a Pi-style total-only stage.
run1, _, err := store.CreateRun(ctx, definition, "usage-1", "/repo", false, nil, nil)
if err != nil {
t.Fatal(err)
}
finishUsageStep(t, store, run1, 1, "one", &protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true, CostUSD: .5, CostKnown: true})
finishUsageStep(t, store, run1, 2, "one", &protocol.StageUsage{TokensTotal: 17, TokensKnown: true, CostUSD: .25, CostKnown: true})
// Run 2: tokens with no reported cost.
run2, _, err := store.CreateRun(ctx, definition, "usage-2", "/repo", false, nil, nil)
if err != nil {
t.Fatal(err)
}
finishUsageStep(t, store, run2, 1, "one", &protocol.StageUsage{TokensTotal: 5, TokensKnown: true})
detail, err := store.GetRun(ctx, run1)
if err != nil {
t.Fatal(err)
}
if len(detail.Steps) != 2 || detail.Steps[0].Usage == nil || detail.Steps[1].Usage == nil {
t.Fatalf("per-step usage did not round-trip: %+v", detail.Steps)
}
if got := detail.Steps[0].Usage; got.TokensIn != 3 || got.TokensOut != 4 || got.TokensTotal != 7 || !got.SplitKnown || got.CostUSD != .5 || !got.CostKnown {
t.Fatalf("split stage usage = %+v", got)
}
if got := detail.Steps[1].Usage; got.TokensTotal != 17 || got.SplitKnown || got.CostUSD != .25 || !got.CostKnown {
t.Fatalf("total-only stage usage = %+v", got)
}
if detail.Usage == nil || detail.Usage.TokensTotal != 24 || detail.Usage.CostUSD != .75 || detail.Usage.Stages != 2 || detail.Usage.CostKnownStages != 2 || detail.Usage.SplitUnknownStages != 1 || detail.Usage.SplitKnown {
t.Fatalf("run total = %+v", detail.Usage)
}
stats, err := store.UsageStats(ctx)
if err != nil {
t.Fatal(err)
}
if stats.Usage == nil {
t.Fatal("history-wide usage totals missing")
}
var tokens int64
var cost float64
knownCostStages := 0
for _, id := range []int64{run1, run2} {
run, err := store.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
if run.Usage == nil {
continue
}
tokens += run.Usage.TokensTotal
cost += run.Usage.CostUSD
knownCostStages += run.Usage.CostKnownStages
}
if stats.Usage.TokensTotal != tokens || stats.Usage.CostUSD != cost || stats.Usage.CostKnownStages != knownCostStages {
t.Fatalf("stats totals %+v do not equal the summed per-run figures tokens=%d cost=%v costStages=%d", stats.Usage, tokens, cost, knownCostStages)
}
if stats.Usage.SplitKnown || stats.Usage.SplitUnknownStages != 2 {
t.Fatalf("partial split presented as complete: %+v", stats.Usage)
}
// The run with no cost must never contribute a fabricated cost.
if stats.Usage.CostUnknownStages != 1 {
t.Fatalf("cost-unknown stage counted as known: %+v", stats.Usage)
}
}
// TestStepUsageReadIsBoundedOverManySteps proves the read path stays a single
// per-step indexed projection plus one pass for the run total: a run with many
// recorded steps loads them all, totals them and preserves each step's usage,
// with no per-step usage query beyond the existing step loop.
func TestStepUsageReadIsBoundedOverManySteps(t *testing.T) {
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), "bounded.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "bounded", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, _ := store.Get(ctx, items[0].ID, 1)
id, _, err := store.CreateRun(ctx, definition, "bounded", "/repo", false, nil, nil)
if err != nil {
t.Fatal(err)
}
const steps = 120
for sequence := 1; sequence <= steps; sequence++ {
finishUsageStep(t, store, id, sequence, "one", &protocol.StageUsage{TokensIn: 1, TokensOut: 1, TokensTotal: 2, TokensKnown: true, SplitKnown: true, CostUSD: .001, CostKnown: true})
}
detail, err := store.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
if len(detail.Steps) != steps || detail.Usage == nil || detail.Usage.Stages != steps || detail.Usage.TokensTotal != 2*steps {
t.Fatalf("bounded read lost steps or totals: %d %+v", len(detail.Steps), detail.Usage)
}
}
// TestStepUsageStoreUpgrade proves a database created before step_usage exists
// opens cleanly, gains the empty table on init with no backfill, keeps listing
// runs and, for the historical step, reports usage absent rather than zero.
func TestStepUsageStoreUpgrade(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "upgrade-usage.db")
store, err := Open(path)
if err != nil {
t.Fatal(err)
}
if _, err := store.Submit(ctx, "upgrade-usage", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, _ := store.Get(ctx, items[0].ID, 1)
id, _, err := store.CreateRun(ctx, definition, "upgrade-usage", "/repo", false, nil, nil)
if err != nil {
t.Fatal(err)
}
finishUsageStep(t, store, id, 1, "one", &protocol.StageUsage{TokensTotal: 9, TokensKnown: true})
// Simulate a database created before step_usage existed.
if _, err := store.db.ExecContext(ctx, "DROP TABLE step_usage"); err != nil {
t.Fatal(err)
}
if err := store.Close(); err != nil {
t.Fatal(err)
}
store, err = Open(path)
if err != nil {
t.Fatal("a pre-usage database did not open cleanly", err)
}
defer store.Close()
runs, err := store.ListRuns(ctx)
if err != nil || len(runs) != 1 {
t.Fatalf("legacy run list lost history: %v %v", runs, err)
}
detail, err := store.GetRun(ctx, id)
if err != nil || len(detail.Steps) != 1 {
t.Fatalf("legacy run/step did not load: %v %v", detail, err)
}
if detail.Steps[0].Usage != nil || detail.Usage != nil {
t.Fatalf("historical usage was backfilled or shown as zero: %+v", detail)
}
stats, err := store.UsageStats(ctx)
if err != nil || stats.Usage != nil {
t.Fatalf("post-upgrade stats = %+v err=%v", stats.Usage, err)
}
}