3f5c11b7bf
Carry the usage a model stage already measures for its budget (stage.Budget.Snapshot()) onto the step as typed evidence and total it in one pass, rather than a second harness read, a transcript parse or an output-length estimate. A shared budget across stage revisits records only the visit's increment, and a revisit that observes no new report records no usage instead of a fabricated known zero. - protocol: StageUsage/RunUsage/TotalUsage and UsageTotals. - pipeline: step_usage table + migration, COALESCEd into the one step projection, FinishStep persistence, one json_extract history aggregate, retention removes the rows with their run. - server: stageUsage maps the budget snapshot, stamping per-stage model and usage for every model stage (including failures). - cli: inspect/stats human and JSON output. - budget/pi: retain the reported in/out split, mark a total-only (Pi) report split-unknown, and keep an unreported cost unknown (never zero). Evidence and docs updated in the packaged skill references and docs/cli.md; the live database has no step_usage rows yet (all history predates this), so per-run figures are reported as unavailable, not zero.
193 lines
7.1 KiB
Go
193 lines
7.1 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// finishUsageStep begins and finishes one completed step carrying the given
|
|
// reported usage, the same store path the runner uses.
|
|
func finishUsageStep(t *testing.T, store *Store, run int64, sequence int, key string, usage *protocol.StageUsage) {
|
|
t.Helper()
|
|
if err := store.BeginStep(context.Background(), run, sequence, key, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.FinishStep(context.Background(), run, protocol.RunStep{
|
|
Sequence: sequence, Stage: key, Status: "completed", Outcome: "success", Target: "#success",
|
|
Output: map[string]any{"summary": "done"}, Usage: usage,
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// TestStepUsageRoundTripRunTotalAndStatsSum proves per-step usage round-trips
|
|
// through the store, the run total is the single-pass sum of the reported stages,
|
|
// and the history-wide aggregate equals the sum of the per-run figures over the
|
|
// same rows.
|
|
func TestStepUsageRoundTripRunTotalAndStatsSum(t *testing.T) {
|
|
ctx := context.Background()
|
|
store, err := Open(filepath.Join(t.TempDir(), "usage.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "usage", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
definition, err := store.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Run 1: a split-cost stage and a Pi-style total-only stage.
|
|
run1, _, err := store.CreateRun(ctx, definition, "usage-1", "/repo", false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
finishUsageStep(t, store, run1, 1, "one", &protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true, CostUSD: .5, CostKnown: true})
|
|
finishUsageStep(t, store, run1, 2, "one", &protocol.StageUsage{TokensTotal: 17, TokensKnown: true, CostUSD: .25, CostKnown: true})
|
|
|
|
// Run 2: tokens with no reported cost.
|
|
run2, _, err := store.CreateRun(ctx, definition, "usage-2", "/repo", false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
finishUsageStep(t, store, run2, 1, "one", &protocol.StageUsage{TokensTotal: 5, TokensKnown: true})
|
|
|
|
detail, err := store.GetRun(ctx, run1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(detail.Steps) != 2 || detail.Steps[0].Usage == nil || detail.Steps[1].Usage == nil {
|
|
t.Fatalf("per-step usage did not round-trip: %+v", detail.Steps)
|
|
}
|
|
if got := detail.Steps[0].Usage; got.TokensIn != 3 || got.TokensOut != 4 || got.TokensTotal != 7 || !got.SplitKnown || got.CostUSD != .5 || !got.CostKnown {
|
|
t.Fatalf("split stage usage = %+v", got)
|
|
}
|
|
if got := detail.Steps[1].Usage; got.TokensTotal != 17 || got.SplitKnown || got.CostUSD != .25 || !got.CostKnown {
|
|
t.Fatalf("total-only stage usage = %+v", got)
|
|
}
|
|
if detail.Usage == nil || detail.Usage.TokensTotal != 24 || detail.Usage.CostUSD != .75 || detail.Usage.Stages != 2 || detail.Usage.CostKnownStages != 2 || detail.Usage.SplitUnknownStages != 1 || detail.Usage.SplitKnown {
|
|
t.Fatalf("run total = %+v", detail.Usage)
|
|
}
|
|
|
|
stats, err := store.UsageStats(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stats.Usage == nil {
|
|
t.Fatal("history-wide usage totals missing")
|
|
}
|
|
var tokens int64
|
|
var cost float64
|
|
knownCostStages := 0
|
|
for _, id := range []int64{run1, run2} {
|
|
run, err := store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if run.Usage == nil {
|
|
continue
|
|
}
|
|
tokens += run.Usage.TokensTotal
|
|
cost += run.Usage.CostUSD
|
|
knownCostStages += run.Usage.CostKnownStages
|
|
}
|
|
if stats.Usage.TokensTotal != tokens || stats.Usage.CostUSD != cost || stats.Usage.CostKnownStages != knownCostStages {
|
|
t.Fatalf("stats totals %+v do not equal the summed per-run figures tokens=%d cost=%v costStages=%d", stats.Usage, tokens, cost, knownCostStages)
|
|
}
|
|
if stats.Usage.SplitKnown || stats.Usage.SplitUnknownStages != 2 {
|
|
t.Fatalf("partial split presented as complete: %+v", stats.Usage)
|
|
}
|
|
// The run with no cost must never contribute a fabricated cost.
|
|
if stats.Usage.CostUnknownStages != 1 {
|
|
t.Fatalf("cost-unknown stage counted as known: %+v", stats.Usage)
|
|
}
|
|
}
|
|
|
|
// TestStepUsageReadIsBoundedOverManySteps proves the read path stays a single
|
|
// per-step indexed projection plus one pass for the run total: a run with many
|
|
// recorded steps loads them all, totals them and preserves each step's usage,
|
|
// with no per-step usage query beyond the existing step loop.
|
|
func TestStepUsageReadIsBoundedOverManySteps(t *testing.T) {
|
|
ctx := context.Background()
|
|
store, err := Open(filepath.Join(t.TempDir(), "bounded.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "bounded", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
definition, _ := store.Get(ctx, items[0].ID, 1)
|
|
id, _, err := store.CreateRun(ctx, definition, "bounded", "/repo", false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
const steps = 120
|
|
for sequence := 1; sequence <= steps; sequence++ {
|
|
finishUsageStep(t, store, id, sequence, "one", &protocol.StageUsage{TokensIn: 1, TokensOut: 1, TokensTotal: 2, TokensKnown: true, SplitKnown: true, CostUSD: .001, CostKnown: true})
|
|
}
|
|
detail, err := store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(detail.Steps) != steps || detail.Usage == nil || detail.Usage.Stages != steps || detail.Usage.TokensTotal != 2*steps {
|
|
t.Fatalf("bounded read lost steps or totals: %d %+v", len(detail.Steps), detail.Usage)
|
|
}
|
|
}
|
|
|
|
// TestStepUsageStoreUpgrade proves a database created before step_usage exists
|
|
// opens cleanly, gains the empty table on init with no backfill, keeps listing
|
|
// runs and, for the historical step, reports usage absent rather than zero.
|
|
func TestStepUsageStoreUpgrade(t *testing.T) {
|
|
ctx := context.Background()
|
|
path := filepath.Join(t.TempDir(), "upgrade-usage.db")
|
|
store, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Submit(ctx, "upgrade-usage", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
definition, _ := store.Get(ctx, items[0].ID, 1)
|
|
id, _, err := store.CreateRun(ctx, definition, "upgrade-usage", "/repo", false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
finishUsageStep(t, store, id, 1, "one", &protocol.StageUsage{TokensTotal: 9, TokensKnown: true})
|
|
// Simulate a database created before step_usage existed.
|
|
if _, err := store.db.ExecContext(ctx, "DROP TABLE step_usage"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err = Open(path)
|
|
if err != nil {
|
|
t.Fatal("a pre-usage database did not open cleanly", err)
|
|
}
|
|
defer store.Close()
|
|
runs, err := store.ListRuns(ctx)
|
|
if err != nil || len(runs) != 1 {
|
|
t.Fatalf("legacy run list lost history: %v %v", runs, err)
|
|
}
|
|
detail, err := store.GetRun(ctx, id)
|
|
if err != nil || len(detail.Steps) != 1 {
|
|
t.Fatalf("legacy run/step did not load: %v %v", detail, err)
|
|
}
|
|
if detail.Steps[0].Usage != nil || detail.Usage != nil {
|
|
t.Fatalf("historical usage was backfilled or shown as zero: %+v", detail)
|
|
}
|
|
stats, err := store.UsageStats(ctx)
|
|
if err != nil || stats.Usage != nil {
|
|
t.Fatalf("post-upgrade stats = %+v err=%v", stats.Usage, err)
|
|
}
|
|
}
|