package pipeline import ( "context" "fmt" "path/filepath" "testing" "git.a-wan.net/alan/pipelines/internal/protocol" ) // TestListRunsCarriesRunUsageAggregate proves the run list carries each run's // usage total from the grouped aggregate, equal to the detail's TotalUsage over // the same rows, marks a partial total by its stage counts, leaves a run with // no recorded usage absent rather than zero, and stays correct across list // batches. func TestListRunsCarriesRunUsageAggregate(t *testing.T) { ctx := context.Background() store, err := Open(filepath.Join(t.TempDir(), "list-usage.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "list-usage", Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) definition, _ := store.Get(ctx, items[0].ID, 1) create := func(key string) int64 { id, _, err := store.CreateRun(ctx, definition, key, "/repo", false, nil, nil) if err != nil { t.Fatal(err) } return id } full := create("full") finishUsageStep(t, store, full, 1, "one", &protocol.StageUsage{TokensIn: 20_000_000, TokensOut: 2_400_000, TokensTotal: 22_400_000, TokensKnown: true, SplitKnown: true, CostUSD: 1.5, CostKnown: true}) finishUsageStep(t, store, full, 2, "one", &protocol.StageUsage{TokensIn: 10, TokensOut: 5, TokensTotal: 15, TokensKnown: true, SplitKnown: true, CostUSD: .25, CostKnown: true}) partial := create("partial") finishUsageStep(t, store, partial, 1, "one", &protocol.StageUsage{TokensTotal: 9, TokensKnown: true}) finishUsageStep(t, store, partial, 2, "one", &protocol.StageUsage{CostUSD: .1, CostKnown: true}) none := create("none") finishUsageStep(t, store, none, 1, "one", nil) // Enough further runs to push the first ones past one list batch. const fillerRuns = 105 for i := range fillerRuns { filler := create(fmt.Sprintf("filler-%d", i)) finishUsageStep(t, store, filler, 1, "one", &protocol.StageUsage{TokensTotal: int64(i + 1), TokensKnown: true}) } runs, err := store.ListRuns(ctx) if err != nil { t.Fatal(err) } byID := map[int64]protocol.RunSummary{} for _, run := range runs { byID[run.ID] = run } for _, id := range []int64{full, partial} { detail, err := store.GetRun(ctx, id) if err != nil { t.Fatal(err) } listed := byID[id].Usage if listed == nil || detail.Usage == nil || *listed != *detail.Usage { t.Fatalf("run %d list usage %+v != detail usage %+v", id, listed, detail.Usage) } } if got := byID[full].Usage; got.TokensTotal != 22_400_015 || got.TokensIn != 20_000_010 || got.TokensOut != 2_400_005 || !got.SplitKnown || got.CostUSD != 1.75 || got.TokensKnownStages != 2 || got.Stages != 2 { t.Fatalf("full run usage = %+v", got) } if got := byID[partial].Usage; got.TokensTotal != 9 || got.TokensKnownStages != 1 || got.Stages != 2 || got.SplitKnown || got.CostKnownStages != 1 || got.CostUnknownStages != 1 { t.Fatalf("partial run usage = %+v", got) } if got := byID[none].Usage; got != nil { t.Fatalf("run without usage listed a total: %+v", got) } var fillers int64 for _, run := range runs { if run.ID == full || run.ID == partial || run.ID == none { continue } if run.Usage == nil { t.Fatalf("filler run %d lost its usage", run.ID) } fillers += run.Usage.TokensTotal } if fillers != fillerRuns*(fillerRuns+1)/2 { t.Fatalf("filler totals = %d", fillers) } } // TestRunUsageCountsCacheAndOnlyRecordedSteps proves the cache/reasoning // components are summed per run (equal in list, detail and stats), a run mixing // a pre-cache row with a cache-reporting one keeps CacheKnown false, and a usage // row whose step or run no longer exists (an old history wipe) is counted // nowhere and left in place. func TestRunUsageCountsCacheAndOnlyRecordedSteps(t *testing.T) { ctx := context.Background() store, err := Open(filepath.Join(t.TempDir(), "cache-usage.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "cache-usage", Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) definition, _ := store.Get(ctx, items[0].ID, 1) cached, _, err := store.CreateRun(ctx, definition, "cached", "/repo", false, nil, nil) if err != nil { t.Fatal(err) } finishUsageStep(t, store, cached, 1, "one", &protocol.StageUsage{TokensIn: 900_000, TokensOut: 150_000, TokensTotal: 22_400_000, TokensKnown: true, SplitKnown: true, TokensReasoning: 50_000, CacheRead: 21_250_000, CacheWrite: 50_000, CacheKnown: true}) mixed, _, err := store.CreateRun(ctx, definition, "mixed", "/repo", false, nil, nil) if err != nil { t.Fatal(err) } finishUsageStep(t, store, mixed, 1, "one", &protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true}) finishUsageStep(t, store, mixed, 2, "one", &protocol.StageUsage{TokensIn: 1, TokensOut: 1, TokensTotal: 10, TokensKnown: true, SplitKnown: true, CacheRead: 8, CacheKnown: true}) // Orphans: one for a run that no longer exists and one for a step the // listed run never recorded. Foreign keys are off only on this connection. conn, err := store.db.Conn(ctx) if err != nil { t.Fatal(err) } if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=OFF"); err != nil { t.Fatal(err) } orphan := `{"tokens_total":1000000000,"tokens_known":true,"cost_usd":15.67,"cost_known":true}` for _, key := range [][2]int64{{9999, 1}, {cached, 99}} { if _, err := conn.ExecContext(ctx, "INSERT INTO step_usage(run_id,sequence,usage_json) VALUES(?,?,?)", key[0], key[1], orphan); err != nil { t.Fatal(err) } } if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=ON"); err != nil { t.Fatal(err) } conn.Close() runs, err := store.ListRuns(ctx) if err != nil { t.Fatal(err) } listed := map[int64]*protocol.RunUsage{} for _, run := range runs { listed[run.ID] = run.Usage } for _, id := range []int64{cached, mixed} { detail, err := store.GetRun(ctx, id) if err != nil { t.Fatal(err) } if listed[id] == nil || detail.Usage == nil || *listed[id] != *detail.Usage { t.Fatalf("run %d list usage %+v != detail usage %+v", id, listed[id], detail.Usage) } } if got := listed[cached]; got.TokensTotal != 22_400_000 || got.CacheRead != 21_250_000 || got.CacheWrite != 50_000 || got.TokensReasoning != 50_000 || !got.CacheKnown || got.CacheKnownStages != 1 || got.Stages != 1 || got.CostKnownStages != 0 { t.Fatalf("cached run usage counted an orphan or lost the cache split: %+v", got) } if got := listed[mixed]; got.CacheKnown || got.CacheKnownStages != 1 || got.CacheRead != 8 || got.TokensTotal != 17 { t.Fatalf("mixed run presented a partial cache split as complete: %+v", got) } stats, err := store.UsageStats(ctx) if err != nil || stats.Usage == nil { t.Fatalf("stats = %+v err=%v", stats, err) } if got := stats.Usage; got.Stages != 3 || got.TokensTotal != 22_400_017 || got.CostUSD != 0 || got.CostKnownStages != 0 || got.CacheRead != 21_250_008 || got.CacheKnown { t.Fatalf("history totals counted orphan rows: %+v", got) } var remaining int if err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM step_usage").Scan(&remaining); err != nil || remaining != 5 { t.Fatalf("usage rows were deleted: %d %v", remaining, err) } }