Files

177 lines
7.1 KiB
Go

package pipeline
import (
"context"
"fmt"
"path/filepath"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// TestListRunsCarriesRunUsageAggregate proves the run list carries each run's
// usage total from the grouped aggregate, equal to the detail's TotalUsage over
// the same rows, marks a partial total by its stage counts, leaves a run with
// no recorded usage absent rather than zero, and stays correct across list
// batches.
func TestListRunsCarriesRunUsageAggregate(t *testing.T) {
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), "list-usage.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "list-usage", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, _ := store.Get(ctx, items[0].ID, 1)
create := func(key string) int64 {
id, _, err := store.CreateRun(ctx, definition, key, "/repo", false, nil, nil)
if err != nil {
t.Fatal(err)
}
return id
}
full := create("full")
finishUsageStep(t, store, full, 1, "one", &protocol.StageUsage{TokensIn: 20_000_000, TokensOut: 2_400_000, TokensTotal: 22_400_000, TokensKnown: true, SplitKnown: true, CostUSD: 1.5, CostKnown: true})
finishUsageStep(t, store, full, 2, "one", &protocol.StageUsage{TokensIn: 10, TokensOut: 5, TokensTotal: 15, TokensKnown: true, SplitKnown: true, CostUSD: .25, CostKnown: true})
partial := create("partial")
finishUsageStep(t, store, partial, 1, "one", &protocol.StageUsage{TokensTotal: 9, TokensKnown: true})
finishUsageStep(t, store, partial, 2, "one", &protocol.StageUsage{CostUSD: .1, CostKnown: true})
none := create("none")
finishUsageStep(t, store, none, 1, "one", nil)
// Enough further runs to push the first ones past one list batch.
const fillerRuns = 105
for i := range fillerRuns {
filler := create(fmt.Sprintf("filler-%d", i))
finishUsageStep(t, store, filler, 1, "one", &protocol.StageUsage{TokensTotal: int64(i + 1), TokensKnown: true})
}
runs, err := store.ListRuns(ctx)
if err != nil {
t.Fatal(err)
}
byID := map[int64]protocol.RunSummary{}
for _, run := range runs {
byID[run.ID] = run
}
for _, id := range []int64{full, partial} {
detail, err := store.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
listed := byID[id].Usage
if listed == nil || detail.Usage == nil || *listed != *detail.Usage {
t.Fatalf("run %d list usage %+v != detail usage %+v", id, listed, detail.Usage)
}
}
if got := byID[full].Usage; got.TokensTotal != 22_400_015 || got.TokensIn != 20_000_010 || got.TokensOut != 2_400_005 || !got.SplitKnown || got.CostUSD != 1.75 || got.TokensKnownStages != 2 || got.Stages != 2 {
t.Fatalf("full run usage = %+v", got)
}
if got := byID[partial].Usage; got.TokensTotal != 9 || got.TokensKnownStages != 1 || got.Stages != 2 || got.SplitKnown || got.CostKnownStages != 1 || got.CostUnknownStages != 1 {
t.Fatalf("partial run usage = %+v", got)
}
if got := byID[none].Usage; got != nil {
t.Fatalf("run without usage listed a total: %+v", got)
}
var fillers int64
for _, run := range runs {
if run.ID == full || run.ID == partial || run.ID == none {
continue
}
if run.Usage == nil {
t.Fatalf("filler run %d lost its usage", run.ID)
}
fillers += run.Usage.TokensTotal
}
if fillers != fillerRuns*(fillerRuns+1)/2 {
t.Fatalf("filler totals = %d", fillers)
}
}
// TestRunUsageCountsCacheAndOnlyRecordedSteps proves the cache/reasoning
// components are summed per run (equal in list, detail and stats), a run mixing
// a pre-cache row with a cache-reporting one keeps CacheKnown false, and a usage
// row whose step or run no longer exists (an old history wipe) is counted
// nowhere and left in place.
func TestRunUsageCountsCacheAndOnlyRecordedSteps(t *testing.T) {
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), "cache-usage.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "cache-usage", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, _ := store.Get(ctx, items[0].ID, 1)
cached, _, err := store.CreateRun(ctx, definition, "cached", "/repo", false, nil, nil)
if err != nil {
t.Fatal(err)
}
finishUsageStep(t, store, cached, 1, "one", &protocol.StageUsage{TokensIn: 900_000, TokensOut: 150_000, TokensTotal: 22_400_000, TokensKnown: true, SplitKnown: true, TokensReasoning: 50_000, CacheRead: 21_250_000, CacheWrite: 50_000, CacheKnown: true})
mixed, _, err := store.CreateRun(ctx, definition, "mixed", "/repo", false, nil, nil)
if err != nil {
t.Fatal(err)
}
finishUsageStep(t, store, mixed, 1, "one", &protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true})
finishUsageStep(t, store, mixed, 2, "one", &protocol.StageUsage{TokensIn: 1, TokensOut: 1, TokensTotal: 10, TokensKnown: true, SplitKnown: true, CacheRead: 8, CacheKnown: true})
// Orphans: one for a run that no longer exists and one for a step the
// listed run never recorded. Foreign keys are off only on this connection.
conn, err := store.db.Conn(ctx)
if err != nil {
t.Fatal(err)
}
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=OFF"); err != nil {
t.Fatal(err)
}
orphan := `{"tokens_total":1000000000,"tokens_known":true,"cost_usd":15.67,"cost_known":true}`
for _, key := range [][2]int64{{9999, 1}, {cached, 99}} {
if _, err := conn.ExecContext(ctx, "INSERT INTO step_usage(run_id,sequence,usage_json) VALUES(?,?,?)", key[0], key[1], orphan); err != nil {
t.Fatal(err)
}
}
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=ON"); err != nil {
t.Fatal(err)
}
conn.Close()
runs, err := store.ListRuns(ctx)
if err != nil {
t.Fatal(err)
}
listed := map[int64]*protocol.RunUsage{}
for _, run := range runs {
listed[run.ID] = run.Usage
}
for _, id := range []int64{cached, mixed} {
detail, err := store.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
if listed[id] == nil || detail.Usage == nil || *listed[id] != *detail.Usage {
t.Fatalf("run %d list usage %+v != detail usage %+v", id, listed[id], detail.Usage)
}
}
if got := listed[cached]; got.TokensTotal != 22_400_000 || got.CacheRead != 21_250_000 || got.CacheWrite != 50_000 || got.TokensReasoning != 50_000 || !got.CacheKnown || got.CacheKnownStages != 1 || got.Stages != 1 || got.CostKnownStages != 0 {
t.Fatalf("cached run usage counted an orphan or lost the cache split: %+v", got)
}
if got := listed[mixed]; got.CacheKnown || got.CacheKnownStages != 1 || got.CacheRead != 8 || got.TokensTotal != 17 {
t.Fatalf("mixed run presented a partial cache split as complete: %+v", got)
}
stats, err := store.UsageStats(ctx)
if err != nil || stats.Usage == nil {
t.Fatalf("stats = %+v err=%v", stats, err)
}
if got := stats.Usage; got.Stages != 3 || got.TokensTotal != 22_400_017 || got.CostUSD != 0 || got.CostKnownStages != 0 || got.CacheRead != 21_250_008 || got.CacheKnown {
t.Fatalf("history totals counted orphan rows: %+v", got)
}
var remaining int
if err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM step_usage").Scan(&remaining); err != nil || remaining != 5 {
t.Fatalf("usage rows were deleted: %d %v", remaining, err)
}
}