package pipeline import ( "context" "path/filepath" "testing" "git.a-wan.net/alan/pipelines/internal/protocol" ) // finishUsageStep begins and finishes one completed step carrying the given // reported usage, the same store path the runner uses. func finishUsageStep(t *testing.T, store *Store, run int64, sequence int, key string, usage *protocol.StageUsage) { t.Helper() if err := store.BeginStep(context.Background(), run, sequence, key, nil); err != nil { t.Fatal(err) } if err := store.FinishStep(context.Background(), run, protocol.RunStep{ Sequence: sequence, Stage: key, Status: "completed", Outcome: "success", Target: "#success", Output: map[string]any{"summary": "done"}, Usage: usage, }); err != nil { t.Fatal(err) } } // TestStepUsageRoundTripRunTotalAndStatsSum proves per-step usage round-trips // through the store, the run total is the single-pass sum of the reported stages, // and the history-wide aggregate equals the sum of the per-run figures over the // same rows. func TestStepUsageRoundTripRunTotalAndStatsSum(t *testing.T) { ctx := context.Background() store, err := Open(filepath.Join(t.TempDir(), "usage.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "usage", Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) definition, err := store.Get(ctx, items[0].ID, 1) if err != nil { t.Fatal(err) } // Run 1: a split-cost stage and a Pi-style total-only stage. run1, _, err := store.CreateRun(ctx, definition, "usage-1", "/repo", false, nil, nil) if err != nil { t.Fatal(err) } finishUsageStep(t, store, run1, 1, "one", &protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true, CostUSD: .5, CostKnown: true}) finishUsageStep(t, store, run1, 2, "one", &protocol.StageUsage{TokensTotal: 17, TokensKnown: true, CostUSD: .25, CostKnown: true}) // Run 2: tokens with no reported cost. run2, _, err := store.CreateRun(ctx, definition, "usage-2", "/repo", false, nil, nil) if err != nil { t.Fatal(err) } finishUsageStep(t, store, run2, 1, "one", &protocol.StageUsage{TokensTotal: 5, TokensKnown: true}) detail, err := store.GetRun(ctx, run1) if err != nil { t.Fatal(err) } if len(detail.Steps) != 2 || detail.Steps[0].Usage == nil || detail.Steps[1].Usage == nil { t.Fatalf("per-step usage did not round-trip: %+v", detail.Steps) } if got := detail.Steps[0].Usage; got.TokensIn != 3 || got.TokensOut != 4 || got.TokensTotal != 7 || !got.SplitKnown || got.CostUSD != .5 || !got.CostKnown { t.Fatalf("split stage usage = %+v", got) } if got := detail.Steps[1].Usage; got.TokensTotal != 17 || got.SplitKnown || got.CostUSD != .25 || !got.CostKnown { t.Fatalf("total-only stage usage = %+v", got) } if detail.Usage == nil || detail.Usage.TokensTotal != 24 || detail.Usage.CostUSD != .75 || detail.Usage.Stages != 2 || detail.Usage.CostKnownStages != 2 || detail.Usage.SplitUnknownStages != 1 || detail.Usage.SplitKnown { t.Fatalf("run total = %+v", detail.Usage) } stats, err := store.UsageStats(ctx) if err != nil { t.Fatal(err) } if stats.Usage == nil { t.Fatal("history-wide usage totals missing") } var tokens int64 var cost float64 knownCostStages := 0 for _, id := range []int64{run1, run2} { run, err := store.GetRun(ctx, id) if err != nil { t.Fatal(err) } if run.Usage == nil { continue } tokens += run.Usage.TokensTotal cost += run.Usage.CostUSD knownCostStages += run.Usage.CostKnownStages } if stats.Usage.TokensTotal != tokens || stats.Usage.CostUSD != cost || stats.Usage.CostKnownStages != knownCostStages { t.Fatalf("stats totals %+v do not equal the summed per-run figures tokens=%d cost=%v costStages=%d", stats.Usage, tokens, cost, knownCostStages) } if stats.Usage.SplitKnown || stats.Usage.SplitUnknownStages != 2 { t.Fatalf("partial split presented as complete: %+v", stats.Usage) } // The run with no cost must never contribute a fabricated cost. if stats.Usage.CostUnknownStages != 1 { t.Fatalf("cost-unknown stage counted as known: %+v", stats.Usage) } } // TestStepUsageReadIsBoundedOverManySteps proves the read path stays a single // per-step indexed projection plus one pass for the run total: a run with many // recorded steps loads them all, totals them and preserves each step's usage, // with no per-step usage query beyond the existing step loop. func TestStepUsageReadIsBoundedOverManySteps(t *testing.T) { ctx := context.Background() store, err := Open(filepath.Join(t.TempDir(), "bounded.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "bounded", Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) definition, _ := store.Get(ctx, items[0].ID, 1) id, _, err := store.CreateRun(ctx, definition, "bounded", "/repo", false, nil, nil) if err != nil { t.Fatal(err) } const steps = 120 for sequence := 1; sequence <= steps; sequence++ { finishUsageStep(t, store, id, sequence, "one", &protocol.StageUsage{TokensIn: 1, TokensOut: 1, TokensTotal: 2, TokensKnown: true, SplitKnown: true, CostUSD: .001, CostKnown: true}) } detail, err := store.GetRun(ctx, id) if err != nil { t.Fatal(err) } if len(detail.Steps) != steps || detail.Usage == nil || detail.Usage.Stages != steps || detail.Usage.TokensTotal != 2*steps { t.Fatalf("bounded read lost steps or totals: %d %+v", len(detail.Steps), detail.Usage) } } // TestStepUsageStoreUpgrade proves a database created before step_usage exists // opens cleanly, gains the empty table on init with no backfill, keeps listing // runs and, for the historical step, reports usage absent rather than zero. func TestStepUsageStoreUpgrade(t *testing.T) { ctx := context.Background() path := filepath.Join(t.TempDir(), "upgrade-usage.db") store, err := Open(path) if err != nil { t.Fatal(err) } if _, err := store.Submit(ctx, "upgrade-usage", Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) definition, _ := store.Get(ctx, items[0].ID, 1) id, _, err := store.CreateRun(ctx, definition, "upgrade-usage", "/repo", false, nil, nil) if err != nil { t.Fatal(err) } finishUsageStep(t, store, id, 1, "one", &protocol.StageUsage{TokensTotal: 9, TokensKnown: true}) // Simulate a database created before step_usage existed. if _, err := store.db.ExecContext(ctx, "DROP TABLE step_usage"); err != nil { t.Fatal(err) } if err := store.Close(); err != nil { t.Fatal(err) } store, err = Open(path) if err != nil { t.Fatal("a pre-usage database did not open cleanly", err) } defer store.Close() runs, err := store.ListRuns(ctx) if err != nil || len(runs) != 1 { t.Fatalf("legacy run list lost history: %v %v", runs, err) } detail, err := store.GetRun(ctx, id) if err != nil || len(detail.Steps) != 1 { t.Fatalf("legacy run/step did not load: %v %v", detail, err) } if detail.Steps[0].Usage != nil || detail.Usage != nil { t.Fatalf("historical usage was backfilled or shown as zero: %+v", detail) } stats, err := store.UsageStats(ctx) if err != nil || stats.Usage != nil { t.Fatalf("post-upgrade stats = %+v err=%v", stats.Usage, err) } }