e37b864c40
Each model visit now gets <run>/stages/<seq>-<stage>/ with input.md, memory.md, output.md, private/ and a shared run-level public/. The model reads the two input files, follows the stage's instructions verbatim and returns complete markdown; only the # outcome section is parsed. - add internal/turn layout, render and output parsing - write input.md/memory.md before a visit and output.md from the final message; inventory all turn files as stage artifacts - accumulate completed turns and system-written public file additions in memory, including across retries and resume - remove the JSON envelope, strict field validation, line format and the repair ladder for model stages - republish pipeline authoring prompts and docs in the new style Checks: go build ./..., go vet ./..., go test -count=1 ./..., gofmt -l .
227 lines
8.9 KiB
Go
227 lines
8.9 KiB
Go
package server
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"math"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/cli"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
// TestStageUsageCarriesHonestUnknowns proves the mapping from the stage's
|
|
// already-measured budget snapshot to the run record never fabricates a cost or
|
|
// a split: a reported split and cost carry through, a total-only report (Pi)
|
|
// keeps the total with the split unavailable, a missing cost stays unknown, and
|
|
// no observed usage stays absent.
|
|
func TestStageUsageCarriesHonestUnknowns(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
snapshot stage.BudgetSnapshot
|
|
want *protocol.StageUsage
|
|
}{
|
|
{"no usage stays absent", stage.BudgetSnapshot{}, nil},
|
|
{"split and cost", stage.BudgetSnapshot{UsedTokens: 7, TokensKnown: true, SplitKnown: true, TokensInput: 3, TokensOutput: 4, UsedUSD: .5, USDKnown: true},
|
|
&protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true, CostUSD: .5, CostKnown: true}},
|
|
{"total only (Pi)", stage.BudgetSnapshot{UsedTokens: 17, TokensKnown: true},
|
|
&protocol.StageUsage{TokensTotal: 17, TokensKnown: true}},
|
|
{"no cost", stage.BudgetSnapshot{UsedTokens: 5, TokensKnown: true, SplitKnown: true, TokensInput: 5},
|
|
&protocol.StageUsage{TokensIn: 5, TokensTotal: 5, TokensKnown: true, SplitKnown: true}},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got := stageUsage(tc.snapshot)
|
|
if (got == nil) != (tc.want == nil) {
|
|
t.Fatalf("stageUsage = %+v, want %+v", got, tc.want)
|
|
}
|
|
if got != nil && *got != *tc.want {
|
|
t.Fatalf("stageUsage = %+v, want %+v", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// fakeBudgetOpencode installs a fake OpenCode CLI that reports per-session usage
|
|
// for a tracked V2 stage, optionally omitting the cost so a no-cost report can be
|
|
// exercised. It mirrors the existing stage-limits fake-budget script.
|
|
func fakeBudgetOpencode(t *testing.T, reportCost bool) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
usageLine := `printf '{"data":{"id":"%%s","cost":%%s,"tokens":{"input":%%s,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}' "$id" "$cost" "$tokens"`
|
|
if !reportCost {
|
|
usageLine = `printf '{"data":{"id":"%%s","tokens":{"input":%%s,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}' "$id" "$tokens"`
|
|
}
|
|
script := fmt.Sprintf(`#!/bin/sh
|
|
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
|
|
case "$3" in
|
|
/api/session)
|
|
n=0; if [ -f %q/count ]; then n=$(cat %q/count); fi
|
|
n=$((n+1)); echo "$n" > %q/count
|
|
printf '{"data":{"id":"ses_usage_%%s"}}' "$n" ;;
|
|
*/prompt)
|
|
id=${3#/api/session/}; id=${id%%/prompt}
|
|
touch %q/"$id"
|
|
echo '{}' ;;
|
|
/api/session/active) echo '{"data":{}}' ;;
|
|
*/message*) printf '%%s\n' '{"data":[{"finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess\n"}]}]}' ;;
|
|
*/interrupt) echo '{}' ;;
|
|
/api/session/ses_usage_*)
|
|
id=${3#/api/session/}; tokens=0; cost=0
|
|
if [ -f %q/"$id" ]; then tokens=3; cost=0.4; fi
|
|
`+usageLine+` ;;
|
|
*) exit 99 ;;
|
|
esac
|
|
`, dir, dir, dir, dir, dir)
|
|
exe := filepath.Join(dir, "opencode")
|
|
if err := os.WriteFile(exe, []byte(script), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("PIPELINES_OPENCODE", exe)
|
|
}
|
|
|
|
func budgetPipeline(t *testing.T, ctx context.Context, store *pipeline.Store, names []string, configs string) {
|
|
t.Helper()
|
|
source := strings.ReplaceAll(pipeline.Template(names, true), "timeout=1h", "timeout=1h\n"+configs)
|
|
if _, err := store.Submit(ctx, "budget-usage", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// TestRunnerRecordsPerStageUsageAcrossMixedModels proves a run with a per-stage
|
|
// model override records each visit's effective model and the usage that visit
|
|
// already measured, and that the run total spans those stages. It uses the
|
|
// existing fake-budget pattern (a fake OpenCode CLI) rather than a second
|
|
// measurement.
|
|
func TestRunnerRecordsPerStageUsageAcrossMixedModels(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
fakeBudgetOpencode(t, true)
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "mixed.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
budgetPipeline(t, ctx, store, []string{"one", "two"}, "budget_tokens=100")
|
|
items, _ := store.List(ctx)
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mixed", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Status != "succeeded" || len(detail.Steps) != 2 {
|
|
t.Fatalf("mixed run did not complete: %+v", detail)
|
|
}
|
|
if detail.Steps[0].Provider != "acme" || detail.Steps[0].Model != "strong" {
|
|
t.Fatalf("overridden stage model = %+v", detail.Steps[0])
|
|
}
|
|
if detail.Steps[1].Provider != "opencode-go" || detail.Steps[1].Model != "deepseek-v4.1-flash" {
|
|
t.Fatalf("pinned stage model = %+v", detail.Steps[1])
|
|
}
|
|
for i, step := range detail.Steps {
|
|
if step.Usage == nil || step.Usage.TokensTotal != 3 || !step.Usage.SplitKnown || step.Usage.TokensIn != 3 || step.Usage.CostUSD != .4 || !step.Usage.CostKnown {
|
|
t.Fatalf("step %d usage = %+v", i, step.Usage)
|
|
}
|
|
}
|
|
if detail.Usage == nil || detail.Usage.TokensTotal != 6 || math.Abs(detail.Usage.CostUSD-.8) > 1e-9 || detail.Usage.Stages != 2 || !detail.Usage.SplitKnown {
|
|
t.Fatalf("run total = %+v", detail.Usage)
|
|
}
|
|
}
|
|
|
|
// TestRunnerKeepsCostUnknownWithTokensShown proves a stage whose harness reports
|
|
// tokens but no cost exposes the tokens and marks the cost unavailable, never
|
|
// zero, and that the run aggregate counts the cost-unknown stage.
|
|
func TestRunnerKeepsCostUnknownWithTokensShown(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
fakeBudgetOpencode(t, false)
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "nocost.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
budgetPipeline(t, ctx, store, []string{"one"}, "budget_tokens=100")
|
|
items, _ := store.List(ctx)
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "nocost", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil || len(detail.Steps) != 1 {
|
|
t.Fatalf("run did not complete: %+v %v", detail, err)
|
|
}
|
|
usage := detail.Steps[0].Usage
|
|
if usage == nil || usage.TokensTotal != 3 || !usage.TokensKnown || usage.CostKnown || usage.CostUSD != 0 {
|
|
t.Fatalf("cost-unknown usage = %+v", usage)
|
|
}
|
|
if detail.Usage == nil || detail.Usage.CostKnownStages != 0 || detail.Usage.CostUnknownStages != 1 || detail.Usage.TokensTotal != 3 {
|
|
t.Fatalf("cost-unknown run total = %+v", detail.Usage)
|
|
}
|
|
}
|
|
|
|
// TestInspectJSONExposesUsageThroughCLI proves the real `pipelines inspect
|
|
// --json` surface carries the per-step usage and the run total over a websocket
|
|
// round trip, after a stage whose usage came from the harness, not a second read.
|
|
func TestInspectJSONExposesUsageThroughCLI(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
fakeBudgetOpencode(t, true)
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "inspect.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
budgetPipeline(t, ctx, store, []string{"one"}, "budget_tokens=100")
|
|
items, _ := store.List(ctx)
|
|
handler := Handler(ctx, store).(*runHandler)
|
|
defer func() { cancel(); WaitRuns(handler) }()
|
|
srv := httptest.NewServer(handler)
|
|
defer srv.Close()
|
|
run, err := handler.runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "inspect-usage", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
handler.runner.wg.Wait()
|
|
url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
|
var out, errOut bytes.Buffer
|
|
app := cli.App{Out: &out, Err: &errOut, In: strings.NewReader("")}
|
|
if err := app.Run(ctx, []string{"--url", url, "inspect", fmt.Sprint(run.ID), "--json"}); err != nil {
|
|
t.Fatal(err, out.String(), errOut.String())
|
|
}
|
|
var detail protocol.RunDetail
|
|
if err := json.Unmarshal(out.Bytes(), &detail); err != nil {
|
|
t.Fatalf("inspect --json was not run detail JSON: %v %s", err, out.String())
|
|
}
|
|
if len(detail.Steps) != 1 || detail.Steps[0].Usage == nil {
|
|
t.Fatalf("inspect --json carried no per-step usage: %s", out.String())
|
|
}
|
|
usage := detail.Steps[0].Usage
|
|
if usage.TokensTotal != 3 || !usage.TokensKnown || !usage.SplitKnown || usage.TokensIn != 3 || usage.CostUSD != .4 || !usage.CostKnown {
|
|
t.Fatalf("step usage = %+v", usage)
|
|
}
|
|
if detail.Usage == nil || detail.Usage.TokensTotal != 3 || detail.Usage.CostUSD != .4 {
|
|
t.Fatalf("run usage total = %+v", detail.Usage)
|
|
}
|
|
}
|