Files
pipelines/internal/server/usage_test.go
T

231 lines
9.5 KiB
Go

package server
import (
"bytes"
"context"
"encoding/json"
"fmt"
"math"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/cli"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// TestStageUsageCarriesHonestUnknowns proves the mapping from the stage's
// already-measured budget snapshot to the run record never fabricates a cost or
// a split: a reported split and cost carry through, a total-only report (Pi)
// keeps the total with the split unavailable, a missing cost stays unknown, and
// no observed usage stays absent.
func TestStageUsageCarriesHonestUnknowns(t *testing.T) {
cases := []struct {
name string
snapshot stage.BudgetSnapshot
want *protocol.StageUsage
}{
{"no usage stays absent", stage.BudgetSnapshot{}, nil},
{"split and cost", stage.BudgetSnapshot{UsedTokens: 7, TokensKnown: true, SplitKnown: true, TokensInput: 3, TokensOutput: 4, UsedUSD: .5, USDKnown: true},
&protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true, CostUSD: .5, CostKnown: true, CacheKnown: true}},
{"cache and reasoning components carry through", stage.BudgetSnapshot{UsedTokens: 22_400_000, TokensKnown: true, SplitKnown: true, TokensInput: 900_000, TokensOutput: 150_000, TokensReasoning: 50_000, CacheRead: 21_250_000, CacheWrite: 50_000},
&protocol.StageUsage{TokensIn: 900_000, TokensOut: 150_000, TokensTotal: 22_400_000, TokensKnown: true, SplitKnown: true, TokensReasoning: 50_000, CacheRead: 21_250_000, CacheWrite: 50_000, CacheKnown: true}},
{"total only (Pi)", stage.BudgetSnapshot{UsedTokens: 17, TokensKnown: true},
&protocol.StageUsage{TokensTotal: 17, TokensKnown: true}},
{"no cost", stage.BudgetSnapshot{UsedTokens: 5, TokensKnown: true, SplitKnown: true, TokensInput: 5},
&protocol.StageUsage{TokensIn: 5, TokensTotal: 5, TokensKnown: true, SplitKnown: true, CacheKnown: true}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := stageUsage(tc.snapshot)
if (got == nil) != (tc.want == nil) {
t.Fatalf("stageUsage = %+v, want %+v", got, tc.want)
}
if got != nil && *got != *tc.want {
t.Fatalf("stageUsage = %+v, want %+v", got, tc.want)
}
})
}
}
// fakeBudgetOpencode installs a fake OpenCode CLI that reports per-session usage
// for a tracked V2 stage, optionally omitting the cost so a no-cost report can be
// exercised. It mirrors the existing stage-limits fake-budget script.
func fakeBudgetOpencode(t *testing.T, reportCost bool) {
t.Helper()
dir := t.TempDir()
usageLine := `printf '{"data":{"id":"%%s","cost":%%s,"tokens":{"input":%%s,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}' "$id" "$cost" "$tokens"`
if !reportCost {
usageLine = `printf '{"data":{"id":"%%s","tokens":{"input":%%s,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}' "$id" "$tokens"`
}
script := fmt.Sprintf(`#!/bin/sh
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
case "$3" in
/api/session)
n=0; if [ -f %q/count ]; then n=$(cat %q/count); fi
n=$((n+1)); echo "$n" > %q/count
printf '{"data":{"id":"ses_usage_%%s"}}' "$n" ;;
*/prompt)
id=${3#/api/session/}; id=${id%%/prompt}
touch %q/"$id"
echo '{}' ;;
/api/session/active) echo '{"data":{}}' ;;
*/message*) printf '%%s\n' '{"data":[{"finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess\n"}]}]}' ;;
*/interrupt) echo '{}' ;;
/api/session/ses_usage_*)
id=${3#/api/session/}; tokens=0; cost=0
if [ -f %q/"$id" ]; then tokens=3; cost=0.4; fi
`+usageLine+` ;;
*) exit 99 ;;
esac
`, dir, dir, dir, dir, dir)
exe := filepath.Join(dir, "opencode")
if err := os.WriteFile(exe, []byte(script), 0700); err != nil {
t.Fatal(err)
}
t.Setenv("PIPELINES_OPENCODE", exe)
localExportService(t, "http://127.0.0.1:1")
}
func budgetPipeline(t *testing.T, ctx context.Context, store *pipeline.Store, names []string, configs string) {
t.Helper()
source := strings.ReplaceAll(pipeline.Template(names, true), "timeout=1h", "timeout=1h\n"+configs)
if _, err := store.Submit(ctx, "budget-usage", source); err != nil {
t.Fatal(err)
}
}
// TestRunnerRecordsPerStageUsageAcrossMixedModels proves a run with a per-stage
// model override records each visit's effective model and the usage that visit
// already measured, and that the run total spans those stages. It uses the
// existing fake-budget pattern (a fake OpenCode CLI) rather than a second
// measurement.
func TestRunnerRecordsPerStageUsageAcrossMixedModels(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
fakeBudgetOpencode(t, true)
store, err := pipeline.Open(filepath.Join(t.TempDir(), "mixed.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
budgetPipeline(t, ctx, store, []string{"one", "two"}, "budget_tokens=100")
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.exportSession = validGateExport
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mixed", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "succeeded" || len(detail.Steps) != 2 {
t.Fatalf("mixed run did not complete: %+v", detail)
}
if detail.Steps[0].Provider != "acme" || detail.Steps[0].Model != "strong" {
t.Fatalf("overridden stage model = %+v", detail.Steps[0])
}
if detail.Steps[1].Provider != "opencode-go" || detail.Steps[1].Model != "deepseek-v4.1-flash" {
t.Fatalf("pinned stage model = %+v", detail.Steps[1])
}
for i, step := range detail.Steps {
if step.Usage == nil || step.Usage.TokensTotal != 3 || !step.Usage.SplitKnown || step.Usage.TokensIn != 3 || step.Usage.CostUSD != .4 || !step.Usage.CostKnown {
t.Fatalf("step %d usage = %+v", i, step.Usage)
}
}
if detail.Usage == nil || detail.Usage.TokensTotal != 6 || math.Abs(detail.Usage.CostUSD-.8) > 1e-9 || detail.Usage.Stages != 2 || !detail.Usage.SplitKnown {
t.Fatalf("run total = %+v", detail.Usage)
}
}
// TestRunnerKeepsCostUnknownWithTokensShown proves a stage whose harness reports
// tokens but no cost exposes the tokens and marks the cost unavailable, never
// zero, and that the run aggregate counts the cost-unknown stage.
func TestRunnerKeepsCostUnknownWithTokensShown(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
fakeBudgetOpencode(t, false)
store, err := pipeline.Open(filepath.Join(t.TempDir(), "nocost.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
budgetPipeline(t, ctx, store, []string{"one"}, "budget_tokens=100")
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "nocost", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil || len(detail.Steps) != 1 {
t.Fatalf("run did not complete: %+v %v", detail, err)
}
usage := detail.Steps[0].Usage
if usage == nil || usage.TokensTotal != 3 || !usage.TokensKnown || usage.CostKnown || usage.CostUSD != 0 {
t.Fatalf("cost-unknown usage = %+v", usage)
}
if detail.Usage == nil || detail.Usage.CostKnownStages != 0 || detail.Usage.CostUnknownStages != 1 || detail.Usage.TokensTotal != 3 {
t.Fatalf("cost-unknown run total = %+v", detail.Usage)
}
}
// TestInspectJSONExposesUsageThroughCLI proves the real `pipelines inspect
// --json` surface carries the per-step usage and the run total over a websocket
// round trip, after a stage whose usage came from the harness, not a second read.
func TestInspectJSONExposesUsageThroughCLI(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
fakeBudgetOpencode(t, true)
store, err := pipeline.Open(filepath.Join(t.TempDir(), "inspect.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
budgetPipeline(t, ctx, store, []string{"one"}, "budget_tokens=100")
items, _ := store.List(ctx)
handler := Handler(ctx, store).(*runHandler)
defer func() { cancel(); WaitRuns(handler) }()
srv := httptest.NewServer(handler)
defer srv.Close()
run, err := handler.runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "inspect-usage", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
handler.runner.wg.Wait()
url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
var out, errOut bytes.Buffer
app := cli.App{Out: &out, Err: &errOut, In: strings.NewReader("")}
if err := app.Run(ctx, []string{"--url", url, "inspect", fmt.Sprint(run.ID), "--json"}); err != nil {
t.Fatal(err, out.String(), errOut.String())
}
var detail protocol.RunDetail
if err := json.Unmarshal(out.Bytes(), &detail); err != nil {
t.Fatalf("inspect --json was not run detail JSON: %v %s", err, out.String())
}
if len(detail.Steps) != 1 || detail.Steps[0].Usage == nil {
t.Fatalf("inspect --json carried no per-step usage: %s", out.String())
}
usage := detail.Steps[0].Usage
if usage.TokensTotal != 3 || !usage.TokensKnown || !usage.SplitKnown || usage.TokensIn != 3 || usage.CostUSD != .4 || !usage.CostKnown {
t.Fatalf("step usage = %+v", usage)
}
if detail.Usage == nil || detail.Usage.TokensTotal != 3 || detail.Usage.CostUSD != .4 {
t.Fatalf("run usage total = %+v", detail.Usage)
}
}