177 lines
7.1 KiB
Go
177 lines
7.1 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// TestListRunsCarriesRunUsageAggregate proves the run list carries each run's
|
|
// usage total from the grouped aggregate, equal to the detail's TotalUsage over
|
|
// the same rows, marks a partial total by its stage counts, leaves a run with
|
|
// no recorded usage absent rather than zero, and stays correct across list
|
|
// batches.
|
|
func TestListRunsCarriesRunUsageAggregate(t *testing.T) {
|
|
ctx := context.Background()
|
|
store, err := Open(filepath.Join(t.TempDir(), "list-usage.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "list-usage", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
definition, _ := store.Get(ctx, items[0].ID, 1)
|
|
create := func(key string) int64 {
|
|
id, _, err := store.CreateRun(ctx, definition, key, "/repo", false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return id
|
|
}
|
|
full := create("full")
|
|
finishUsageStep(t, store, full, 1, "one", &protocol.StageUsage{TokensIn: 20_000_000, TokensOut: 2_400_000, TokensTotal: 22_400_000, TokensKnown: true, SplitKnown: true, CostUSD: 1.5, CostKnown: true})
|
|
finishUsageStep(t, store, full, 2, "one", &protocol.StageUsage{TokensIn: 10, TokensOut: 5, TokensTotal: 15, TokensKnown: true, SplitKnown: true, CostUSD: .25, CostKnown: true})
|
|
partial := create("partial")
|
|
finishUsageStep(t, store, partial, 1, "one", &protocol.StageUsage{TokensTotal: 9, TokensKnown: true})
|
|
finishUsageStep(t, store, partial, 2, "one", &protocol.StageUsage{CostUSD: .1, CostKnown: true})
|
|
none := create("none")
|
|
finishUsageStep(t, store, none, 1, "one", nil)
|
|
// Enough further runs to push the first ones past one list batch.
|
|
const fillerRuns = 105
|
|
for i := range fillerRuns {
|
|
filler := create(fmt.Sprintf("filler-%d", i))
|
|
finishUsageStep(t, store, filler, 1, "one", &protocol.StageUsage{TokensTotal: int64(i + 1), TokensKnown: true})
|
|
}
|
|
|
|
runs, err := store.ListRuns(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
byID := map[int64]protocol.RunSummary{}
|
|
for _, run := range runs {
|
|
byID[run.ID] = run
|
|
}
|
|
for _, id := range []int64{full, partial} {
|
|
detail, err := store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
listed := byID[id].Usage
|
|
if listed == nil || detail.Usage == nil || *listed != *detail.Usage {
|
|
t.Fatalf("run %d list usage %+v != detail usage %+v", id, listed, detail.Usage)
|
|
}
|
|
}
|
|
if got := byID[full].Usage; got.TokensTotal != 22_400_015 || got.TokensIn != 20_000_010 || got.TokensOut != 2_400_005 || !got.SplitKnown || got.CostUSD != 1.75 || got.TokensKnownStages != 2 || got.Stages != 2 {
|
|
t.Fatalf("full run usage = %+v", got)
|
|
}
|
|
if got := byID[partial].Usage; got.TokensTotal != 9 || got.TokensKnownStages != 1 || got.Stages != 2 || got.SplitKnown || got.CostKnownStages != 1 || got.CostUnknownStages != 1 {
|
|
t.Fatalf("partial run usage = %+v", got)
|
|
}
|
|
if got := byID[none].Usage; got != nil {
|
|
t.Fatalf("run without usage listed a total: %+v", got)
|
|
}
|
|
var fillers int64
|
|
for _, run := range runs {
|
|
if run.ID == full || run.ID == partial || run.ID == none {
|
|
continue
|
|
}
|
|
if run.Usage == nil {
|
|
t.Fatalf("filler run %d lost its usage", run.ID)
|
|
}
|
|
fillers += run.Usage.TokensTotal
|
|
}
|
|
if fillers != fillerRuns*(fillerRuns+1)/2 {
|
|
t.Fatalf("filler totals = %d", fillers)
|
|
}
|
|
}
|
|
|
|
// TestRunUsageCountsCacheAndOnlyRecordedSteps proves the cache/reasoning
|
|
// components are summed per run (equal in list, detail and stats), a run mixing
|
|
// a pre-cache row with a cache-reporting one keeps CacheKnown false, and a usage
|
|
// row whose step or run no longer exists (an old history wipe) is counted
|
|
// nowhere and left in place.
|
|
func TestRunUsageCountsCacheAndOnlyRecordedSteps(t *testing.T) {
|
|
ctx := context.Background()
|
|
store, err := Open(filepath.Join(t.TempDir(), "cache-usage.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "cache-usage", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
definition, _ := store.Get(ctx, items[0].ID, 1)
|
|
cached, _, err := store.CreateRun(ctx, definition, "cached", "/repo", false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
finishUsageStep(t, store, cached, 1, "one", &protocol.StageUsage{TokensIn: 900_000, TokensOut: 150_000, TokensTotal: 22_400_000, TokensKnown: true, SplitKnown: true, TokensReasoning: 50_000, CacheRead: 21_250_000, CacheWrite: 50_000, CacheKnown: true})
|
|
mixed, _, err := store.CreateRun(ctx, definition, "mixed", "/repo", false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
finishUsageStep(t, store, mixed, 1, "one", &protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true})
|
|
finishUsageStep(t, store, mixed, 2, "one", &protocol.StageUsage{TokensIn: 1, TokensOut: 1, TokensTotal: 10, TokensKnown: true, SplitKnown: true, CacheRead: 8, CacheKnown: true})
|
|
|
|
// Orphans: one for a run that no longer exists and one for a step the
|
|
// listed run never recorded. Foreign keys are off only on this connection.
|
|
conn, err := store.db.Conn(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=OFF"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
orphan := `{"tokens_total":1000000000,"tokens_known":true,"cost_usd":15.67,"cost_known":true}`
|
|
for _, key := range [][2]int64{{9999, 1}, {cached, 99}} {
|
|
if _, err := conn.ExecContext(ctx, "INSERT INTO step_usage(run_id,sequence,usage_json) VALUES(?,?,?)", key[0], key[1], orphan); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=ON"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
conn.Close()
|
|
|
|
runs, err := store.ListRuns(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
listed := map[int64]*protocol.RunUsage{}
|
|
for _, run := range runs {
|
|
listed[run.ID] = run.Usage
|
|
}
|
|
for _, id := range []int64{cached, mixed} {
|
|
detail, err := store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if listed[id] == nil || detail.Usage == nil || *listed[id] != *detail.Usage {
|
|
t.Fatalf("run %d list usage %+v != detail usage %+v", id, listed[id], detail.Usage)
|
|
}
|
|
}
|
|
if got := listed[cached]; got.TokensTotal != 22_400_000 || got.CacheRead != 21_250_000 || got.CacheWrite != 50_000 || got.TokensReasoning != 50_000 || !got.CacheKnown || got.CacheKnownStages != 1 || got.Stages != 1 || got.CostKnownStages != 0 {
|
|
t.Fatalf("cached run usage counted an orphan or lost the cache split: %+v", got)
|
|
}
|
|
if got := listed[mixed]; got.CacheKnown || got.CacheKnownStages != 1 || got.CacheRead != 8 || got.TokensTotal != 17 {
|
|
t.Fatalf("mixed run presented a partial cache split as complete: %+v", got)
|
|
}
|
|
stats, err := store.UsageStats(ctx)
|
|
if err != nil || stats.Usage == nil {
|
|
t.Fatalf("stats = %+v err=%v", stats, err)
|
|
}
|
|
if got := stats.Usage; got.Stages != 3 || got.TokensTotal != 22_400_017 || got.CostUSD != 0 || got.CostKnownStages != 0 || got.CacheRead != 21_250_008 || got.CacheKnown {
|
|
t.Fatalf("history totals counted orphan rows: %+v", got)
|
|
}
|
|
var remaining int
|
|
if err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM step_usage").Scan(&remaining); err != nil || remaining != 5 {
|
|
t.Fatalf("usage rows were deleted: %d %v", remaining, err)
|
|
}
|
|
}
|