package server import ( "bytes" "context" "encoding/json" "fmt" "math" "net/http/httptest" "os" "path/filepath" "strings" "testing" "git.a-wan.net/alan/pipelines/internal/cli" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) // TestStageUsageCarriesHonestUnknowns proves the mapping from the stage's // already-measured budget snapshot to the run record never fabricates a cost or // a split: a reported split and cost carry through, a total-only report (Pi) // keeps the total with the split unavailable, a missing cost stays unknown, and // no observed usage stays absent. func TestStageUsageCarriesHonestUnknowns(t *testing.T) { cases := []struct { name string snapshot stage.BudgetSnapshot want *protocol.StageUsage }{ {"no usage stays absent", stage.BudgetSnapshot{}, nil}, {"split and cost", stage.BudgetSnapshot{UsedTokens: 7, TokensKnown: true, SplitKnown: true, TokensInput: 3, TokensOutput: 4, UsedUSD: .5, USDKnown: true}, &protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true, CostUSD: .5, CostKnown: true}}, {"total only (Pi)", stage.BudgetSnapshot{UsedTokens: 17, TokensKnown: true}, &protocol.StageUsage{TokensTotal: 17, TokensKnown: true}}, {"no cost", stage.BudgetSnapshot{UsedTokens: 5, TokensKnown: true, SplitKnown: true, TokensInput: 5}, &protocol.StageUsage{TokensIn: 5, TokensTotal: 5, TokensKnown: true, SplitKnown: true}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got := stageUsage(tc.snapshot) if (got == nil) != (tc.want == nil) { t.Fatalf("stageUsage = %+v, want %+v", got, tc.want) } if got != nil && *got != *tc.want { t.Fatalf("stageUsage = %+v, want %+v", got, tc.want) } }) } } // fakeBudgetOpencode installs a fake OpenCode CLI that reports per-session usage // for a tracked V2 stage, optionally omitting the cost so a no-cost report can be // exercised. It mirrors the existing stage-limits fake-budget script. func fakeBudgetOpencode(t *testing.T, reportCost bool) { t.Helper() dir := t.TempDir() usageLine := `printf '{"data":{"id":"%%s","cost":%%s,"tokens":{"input":%%s,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}' "$id" "$cost" "$tokens"` if !reportCost { usageLine = `printf '{"data":{"id":"%%s","tokens":{"input":%%s,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}' "$id" "$tokens"` } script := fmt.Sprintf(`#!/bin/sh if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi case "$3" in /api/session) n=0; if [ -f %q/count ]; then n=$(cat %q/count); fi n=$((n+1)); echo "$n" > %q/count printf '{"data":{"id":"ses_usage_%%s"}}' "$n" ;; */prompt) id=${3#/api/session/}; id=${id%%/prompt} touch %q/"$id" echo '{}' ;; /api/session/active) echo '{"data":{}}' ;; */message*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"{\"outcome\":\"success\",\"output\":{\"summary\":\"done\"}}"}]}]}' ;; */interrupt) echo '{}' ;; /api/session/ses_usage_*) id=${3#/api/session/}; tokens=0; cost=0 if [ -f %q/"$id" ]; then tokens=3; cost=0.4; fi `+usageLine+` ;; *) exit 99 ;; esac `, dir, dir, dir, dir, dir) exe := filepath.Join(dir, "opencode") if err := os.WriteFile(exe, []byte(script), 0700); err != nil { t.Fatal(err) } t.Setenv("PIPELINES_OPENCODE", exe) } func budgetPipeline(t *testing.T, ctx context.Context, store *pipeline.Store, names []string, configs string) { t.Helper() source := strings.ReplaceAll(pipeline.Template(names, true), "timeout=1h", "timeout=1h\n"+configs) if _, err := store.Submit(ctx, "budget-usage", source); err != nil { t.Fatal(err) } } // TestRunnerRecordsPerStageUsageAcrossMixedModels proves a run with a per-stage // model override records each visit's effective model and the usage that visit // already measured, and that the run total spans those stages. It uses the // existing fake-budget pattern (a fake OpenCode CLI) rather than a second // measurement. func TestRunnerRecordsPerStageUsageAcrossMixedModels(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() fakeBudgetOpencode(t, true) store, err := pipeline.Open(filepath.Join(t.TempDir(), "mixed.db")) if err != nil { t.Fatal(err) } defer store.Close() budgetPipeline(t, ctx, store, []string{"one", "two"}, "budget_tokens=100") items, _ := store.List(ctx) r, err := newRunner(ctx, store) if err != nil { t.Fatal(err) } run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mixed", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}}) if err != nil { t.Fatal(err) } r.wg.Wait() detail, err := store.GetRun(ctx, run.ID) if err != nil { t.Fatal(err) } if detail.Status != "succeeded" || len(detail.Steps) != 2 { t.Fatalf("mixed run did not complete: %+v", detail) } if detail.Steps[0].Provider != "acme" || detail.Steps[0].Model != "strong" { t.Fatalf("overridden stage model = %+v", detail.Steps[0]) } if detail.Steps[1].Provider != "opencode-go" || detail.Steps[1].Model != "deepseek-v4.1-flash" { t.Fatalf("pinned stage model = %+v", detail.Steps[1]) } for i, step := range detail.Steps { if step.Usage == nil || step.Usage.TokensTotal != 3 || !step.Usage.SplitKnown || step.Usage.TokensIn != 3 || step.Usage.CostUSD != .4 || !step.Usage.CostKnown { t.Fatalf("step %d usage = %+v", i, step.Usage) } } if detail.Usage == nil || detail.Usage.TokensTotal != 6 || math.Abs(detail.Usage.CostUSD-.8) > 1e-9 || detail.Usage.Stages != 2 || !detail.Usage.SplitKnown { t.Fatalf("run total = %+v", detail.Usage) } } // TestRunnerKeepsCostUnknownWithTokensShown proves a stage whose harness reports // tokens but no cost exposes the tokens and marks the cost unavailable, never // zero, and that the run aggregate counts the cost-unknown stage. func TestRunnerKeepsCostUnknownWithTokensShown(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() fakeBudgetOpencode(t, false) store, err := pipeline.Open(filepath.Join(t.TempDir(), "nocost.db")) if err != nil { t.Fatal(err) } defer store.Close() budgetPipeline(t, ctx, store, []string{"one"}, "budget_tokens=100") items, _ := store.List(ctx) r, err := newRunner(ctx, store) if err != nil { t.Fatal(err) } run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "nocost", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } r.wg.Wait() detail, err := store.GetRun(ctx, run.ID) if err != nil || len(detail.Steps) != 1 { t.Fatalf("run did not complete: %+v %v", detail, err) } usage := detail.Steps[0].Usage if usage == nil || usage.TokensTotal != 3 || !usage.TokensKnown || usage.CostKnown || usage.CostUSD != 0 { t.Fatalf("cost-unknown usage = %+v", usage) } if detail.Usage == nil || detail.Usage.CostKnownStages != 0 || detail.Usage.CostUnknownStages != 1 || detail.Usage.TokensTotal != 3 { t.Fatalf("cost-unknown run total = %+v", detail.Usage) } } // TestInspectJSONExposesUsageThroughCLI proves the real `pipelines inspect // --json` surface carries the per-step usage and the run total over a websocket // round trip, after a stage whose usage came from the harness, not a second read. func TestInspectJSONExposesUsageThroughCLI(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() fakeBudgetOpencode(t, true) store, err := pipeline.Open(filepath.Join(t.TempDir(), "inspect.db")) if err != nil { t.Fatal(err) } defer store.Close() budgetPipeline(t, ctx, store, []string{"one"}, "budget_tokens=100") items, _ := store.List(ctx) handler := Handler(ctx, store).(*runHandler) defer func() { cancel(); WaitRuns(handler) }() srv := httptest.NewServer(handler) defer srv.Close() run, err := handler.runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "inspect-usage", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } handler.runner.wg.Wait() url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" var out, errOut bytes.Buffer app := cli.App{Out: &out, Err: &errOut, In: strings.NewReader("")} if err := app.Run(ctx, []string{"--url", url, "inspect", fmt.Sprint(run.ID), "--json"}); err != nil { t.Fatal(err, out.String(), errOut.String()) } var detail protocol.RunDetail if err := json.Unmarshal(out.Bytes(), &detail); err != nil { t.Fatalf("inspect --json was not run detail JSON: %v %s", err, out.String()) } if len(detail.Steps) != 1 || detail.Steps[0].Usage == nil { t.Fatalf("inspect --json carried no per-step usage: %s", out.String()) } usage := detail.Steps[0].Usage if usage.TokensTotal != 3 || !usage.TokensKnown || !usage.SplitKnown || usage.TokensIn != 3 || usage.CostUSD != .4 || !usage.CostKnown { t.Fatalf("step usage = %+v", usage) } if detail.Usage == nil || detail.Usage.TokensTotal != 3 || detail.Usage.CostUSD != .4 { t.Fatalf("run usage total = %+v", detail.Usage) } }