From a66d540125092f0211118ce8e611d3a3fb9929eb Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sat, 10 Oct 2026 03:03:46 +0100 Subject: [PATCH] Add stage execution limits and result-first workflow views --- README.md | 3 +- docs/cli.md | 43 +++ examples/research.md | 6 +- examples/review.md | 3 +- examples/summary.md | 3 +- examples/task.md | 3 +- examples/three-stage.md | 9 +- internal/author/skill/references/format.md | 21 +- internal/opencode/budget.go | 16 ++ internal/pipeline/definition.go | 2 +- internal/pipeline/definition_test.go | 6 +- internal/pipeline/runs.go | 15 +- internal/protocol/result.go | 58 ++++ internal/protocol/result_test.go | 28 ++ internal/protocol/runs.go | 26 +- internal/server/runner.go | 32 ++- internal/server/stage_limits_test.go | 156 ++++++++++ internal/stage/agent.go | 27 +- internal/stage/budget.go | 292 +++++++++++++++++++ internal/stage/budget_test.go | 315 +++++++++++++++++++++ internal/stage/model.go | 85 ++++-- internal/stage/model_test.go | 4 +- internal/stage/ntfy.go | 8 +- internal/stage/opencode.go | 11 +- internal/stage/result_repair.go | 2 +- internal/tui/dashboard.go | 15 +- internal/tui/model.go | 4 +- internal/tui/panels.go | 20 +- internal/tui/results.go | 131 +++++++++ internal/tui/results_test.go | 76 +++++ internal/tui/run_detail.go | 46 ++- internal/tui/run_graph.go | 2 +- internal/tui/run_graph_test.go | 4 +- internal/tui/run_pane_size_test.go | 9 +- internal/tui/runs_test.go | 4 +- 35 files changed, 1392 insertions(+), 93 deletions(-) create mode 100644 internal/opencode/budget.go create mode 100644 internal/protocol/result.go create mode 100644 internal/protocol/result_test.go create mode 100644 internal/server/stage_limits_test.go create mode 100644 internal/stage/budget.go create mode 100644 internal/stage/budget_test.go create mode 100644 internal/tui/results.go create mode 100644 internal/tui/results_test.go diff --git a/README.md b/README.md index 83c34b6a..6af3f6d6 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,8 @@ Turn input.prompt into a focused plan and acceptance checks. ```configs provider=opencode-go model=deepseek-v4.1-flash -retries=0 +retries=2 +timeout=1h ``` ```edges diff --git a/docs/cli.md b/docs/cli.md index 67630387..ed77cdb4 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -309,6 +309,49 @@ Quit with `:quit`, not `q`; Ctrl+C remains an interrupt. JSON results go to stdout, errors to stderr (JSON error envelopes when requested), and failures exit nonzero. Export/template `--source` deliberately prints raw Markdown. `--timeout` bounds each server request and the optional wait duration. +It does not change the stage execution timeout. + +### Stage limits + +Model-stage `configs` support these optional settings: + +```text +retries=2 +timeout=1h +budget_usd=1.50 +budget_tokens=50000 +``` + +Retries default to **2** (three total attempts); timeout defaults to **one hour +per attempt**. Omitted budgets are unlimited; configured budgets must be positive. +Each budget covers all attempts, format-only repairs, and repeated visits to that +stage in one run. A new run starts new accounting. Tokens include input, output, +reasoning, and cache reads/writes. USD amounts use OpenCode's reported cost. +Missing or invalid required accounting stops the stage rather than counting zero. +Usage is polled during execution and read again afterward; an in-flight request +may exceed a limit before usage arrives. These are not provider-side hard caps. + +Automatic retries cover retryable transport failures and per-attempt timeouts. +They do not replay admitted tool work, invalid results, or budget failures. +Cleanup must be confirmed before another attempt. Existing explicit `retries=0` +settings remain unchanged. The old 30-minute run deadline no longer truncates +stage timeouts; the 30-step and five-visits-per-stage limits still apply. +Native ntfy stages also accept `timeout` (default `1h`), but never automatically +retry potentially delivered notifications and do not accept model budgets. + +Pinned definitions contain the configured limits. Run transcripts retain limits, +attempt counts and reported budget usage, also shown in the TUI's Evidence pane. + +### Result-first views + +Home shows retained result previews and completed visits alongside each run and +pipeline. Pipeline pages show the latest result for the **selected version**, +not a result from a different published definition. Run opens focused on results, +with output before input and the graph below. Failed or ongoing runs preserve +partial output and completed stage reports without labeling them final success. +These previews quote recorded output; they do not independently verify model claims. +Tab still expands the focused graph, stage list, or result pane; `n`/`p` still +navigate recorded visits, including repeated stages. ## Verification scope diff --git a/examples/research.md b/examples/research.md index f3eebc1f..fe45a5fe 100644 --- a/examples/research.md +++ b/examples/research.md @@ -20,7 +20,8 @@ webfetch ```configs provider=opencode-go model=deepseek-v4.1-flash -retries=0 +retries=2 +timeout=1h ``` ```edges @@ -48,7 +49,8 @@ Return success for a supported report or blocked when the evidence is insufficie ```configs provider=opencode-go model=deepseek-v4.1-flash -retries=0 +retries=2 +timeout=1h ``` ```edges diff --git a/examples/review.md b/examples/review.md index 34a3154e..f8367768 100644 --- a/examples/review.md +++ b/examples/review.md @@ -24,7 +24,8 @@ grep ```configs provider=opencode-go model=deepseek-v4.1-flash -retries=0 +retries=2 +timeout=1h ``` ```edges diff --git a/examples/summary.md b/examples/summary.md index 3681062c..309271ea 100644 --- a/examples/summary.md +++ b/examples/summary.md @@ -17,7 +17,8 @@ Return blocked if no text to summarize was supplied. Otherwise return success. ```configs provider=opencode-go model=deepseek-v4.1-flash -retries=0 +retries=2 +timeout=1h ``` ```edges diff --git a/examples/task.md b/examples/task.md index 6b87ff18..6a191035 100644 --- a/examples/task.md +++ b/examples/task.md @@ -25,7 +25,8 @@ shell ```configs provider=opencode-go model=deepseek-v4.1-flash -retries=0 +retries=2 +timeout=1h ``` ```edges diff --git a/examples/three-stage.md b/examples/three-stage.md index 97b7d5a9..6da44cf7 100644 --- a/examples/three-stage.md +++ b/examples/three-stage.md @@ -18,7 +18,8 @@ grep ```configs provider=opencode-go model=deepseek-v4.1-flash -retries=0 +retries=2 +timeout=1h ``` ```edges @@ -49,7 +50,8 @@ edit ```configs provider=opencode-go model=deepseek-v4.1-flash -retries=0 +retries=2 +timeout=1h ``` ```edges @@ -81,7 +83,8 @@ shell ```configs provider=opencode-go model=deepseek-v4.1-flash -retries=0 +retries=2 +timeout=1h ``` ```edges diff --git a/internal/author/skill/references/format.md b/internal/author/skill/references/format.md index e662dc0f..2f7c152c 100644 --- a/internal/author/skill/references/format.md +++ b/internal/author/skill/references/format.md @@ -24,7 +24,8 @@ Perform the task supplied in input.prompt. Return evidence, not invented success type=model provider=opencode-go model=deepseek-v4.1-flash -retries=0 +retries=2 +timeout=1h ``` ```tools @@ -48,8 +49,22 @@ Use the exact provider/model IDs validated during server setup. New drafts/templ inherit that selection; do not assume the host's login matches the container's. Other OpenCode provider/model IDs are accepted syntactically; actual availability, authentication and inference require `pipelines provider-check` on the execution -computer/container. Retries are -integers 0–100. Explicit empty `tools` denies all; allowed tools are `read`, `glob`, +computer/container. Optional `retries` defaults to 2 (three total attempts), +and accepts integers 0–100; explicit 0 disables execution retries. Optional +`timeout` defaults to `1h` per attempt and accepts positive durations such as +`30m` or `90s`. Automatic retries only cover retryable transport failures and +timeouts; admitted tool work is never replayed automatically. Optional +`budget_usd=1.50` and `budget_tokens=50000` independently cap reported usage. +They accumulate across attempts, format repair sessions, and repeated visits +to this stage in the same run. Omit either budget to leave that resource unlimited. +Both budgets must be positive; tokens include input, output, reasoning and cache +reads/writes. Accounting failures stop budget-limited stages. In-flight requests +may exceed limits before usage is reported; these are not provider-side hard caps. +The run has no additional wall-clock deadline; step/visit caps and cancellation +still apply. A native ntfy stage accepts `timeout` but never automatically retries +delivery and has no model budget. + +Explicit empty `tools` denies all; allowed tools are `read`, `glob`, `grep`, `webfetch`, `websearch`, `edit`, `shell`, one per line. No wildcards, delegation, or browser tools. Tools are per-stage and automatically enabled by starting a run. `edit` supports any file type and creating files/directories in the diff --git a/internal/opencode/budget.go b/internal/opencode/budget.go new file mode 100644 index 00000000..4e01f589 --- /dev/null +++ b/internal/opencode/budget.go @@ -0,0 +1,16 @@ +package opencode + +import ( + "context" + "fmt" +) + +// BudgetMessages uses the same owned endpoint and credentials, without sharing +// mutable request diagnostics or generation state with the main connection. +func (c *Connection) BudgetMessages(ctx context.Context, id string) ([]byte, error) { + if !sessionID.MatchString(id) { + return nil, fmt.Errorf("invalid session ID") + } + reader := &Connection{Backend: c.Backend, Executable: c.Executable, Directory: c.Directory, URL: c.URL, password: c.password} + return reader.Messages(ctx, id) +} diff --git a/internal/pipeline/definition.go b/internal/pipeline/definition.go index 5d922394..8be1f92a 100644 --- a/internal/pipeline/definition.go +++ b/internal/pipeline/definition.go @@ -33,7 +33,7 @@ func Template(names []string, linear bool) string { if linear && i+1 < len(names) { next = names[i+1] } - fmt.Fprintf(&b, "# stage %s\n\n```output\nDescribe the output map.\n```\n\n```prompt\nDescribe the task.\n```\n\n```tools\n```\n\n```edges\nsuccess -> %s\nerror -> %s\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n", name, next, protocol.FailureEnd) + fmt.Fprintf(&b, "# stage %s\n\n```output\nDescribe the output map.\n```\n\n```prompt\nDescribe the task.\n```\n\n```tools\n```\n\n```edges\nsuccess -> %s\nerror -> %s\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=2\ntimeout=1h\n```\n\n", name, next, protocol.FailureEnd) } return b.String() } diff --git a/internal/pipeline/definition_test.go b/internal/pipeline/definition_test.go index 0600be4a..40047504 100644 --- a/internal/pipeline/definition_test.go +++ b/internal/pipeline/definition_test.go @@ -32,11 +32,11 @@ func TestValidation(t *testing.T) { "duplicate stage": valid + valid, "unknown target": strings.Replace(valid, "success -> reviewer", "success -> missing", 1), "duplicate outcome": strings.Replace(valid, "success -> reviewer", "success -> reviewer\nsuccess -> #", 1), - "unknown config": strings.Replace(valid, "retries=0", "oops=2", 1), - "duplicate config": strings.Replace(valid, "retries=0", "retries=0\nretries=1", 1), + "unknown config": strings.Replace(valid, "retries=2", "oops=2", 1), + "duplicate config": strings.Replace(valid, "retries=2", "retries=2\nretries=1", 1), "invalid provider": strings.Replace(valid, "provider=opencode-go", "provider=not valid", 1), "invalid model": strings.Replace(valid, "model=deepseek-v4.1-flash", "model=not valid", 1), - "bad retries": strings.Replace(valid, "retries=0", "retries=-1", 1), + "bad retries": strings.Replace(valid, "retries=2", "retries=-1", 1), "multiple targets": strings.Replace(valid, "success -> reviewer", "success -> reviewer, implementer", 1), "unreachable": Template([]string{"a", "b"}, false), "no termination": strings.NewReplacer("-> #success", "-> a", "-> #failure", "-> a").Replace(Template([]string{"a"}, true)), diff --git a/internal/pipeline/runs.go b/internal/pipeline/runs.go index 792e1d9f..6116b8e8 100644 --- a/internal/pipeline/runs.go +++ b/internal/pipeline/runs.go @@ -97,11 +97,22 @@ func (s *Store) CreateRun(ctx context.Context, d *protocol.PipelineDetail, key, } const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage, -strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',r.finished_at),''),r.diagnostic +strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',r.finished_at),''),r.diagnostic, +CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END, +(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed') FROM pipeline_runs r JOIN pipelines p ON p.id=r.pipeline_id JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version` func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error { - return row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic) + var output string + if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps); err != nil { + return err + } + var values map[string]any + if err := decodeValues(output, &values); err != nil { + return err + } + r.ResultPreview = protocol.OutputPreview(values) + return nil } func (s *Store) ListRuns(ctx context.Context) ([]protocol.RunSummary, error) { diff --git a/internal/protocol/result.go b/internal/protocol/result.go new file mode 100644 index 00000000..b720e1bf --- /dev/null +++ b/internal/protocol/result.go @@ -0,0 +1,58 @@ +package protocol + +import ( + "encoding/json" + "sort" + "strings" + "unicode" +) + +// OutputPreview quotes retained output, never infers accomplishments from a +// successful process or invents a schema for technology-neutral workflows. +func OutputPreview(output map[string]any) string { + keys := []string{"summary", "result", "report", "message", "accomplished", "artifacts", "evidence", "reason", "error"} + seen := map[string]bool{} + remaining := make([]string, 0, len(output)) + for key := range output { + remaining = append(remaining, key) + } + sort.Strings(remaining) + keys = append(keys, remaining...) + for _, key := range keys { + if seen[key] || key == "input" { + continue + } + seen[key] = true + value, exists := output[key] + if !exists || value == nil { + continue + } + text, ok := value.(string) + if !ok { + encoded, err := json.Marshal(value) + if err != nil { + continue + } + text = string(encoded) + } + text = strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return ' ' + } + return r + }, text) + text = strings.Join(strings.Fields(text), " ") + if text == "" || text == "[]" || text == "{}" { + continue + } + if key != "summary" && key != "result" && key != "report" && key != "message" { + text = key + ": " + text + } + runes := []rune(text) + if len(runes) > 280 { + text = string(runes[:279]) + "…" + } + return text + } + return "" +} diff --git a/internal/protocol/result_test.go b/internal/protocol/result_test.go new file mode 100644 index 00000000..59e5259a --- /dev/null +++ b/internal/protocol/result_test.go @@ -0,0 +1,28 @@ +package protocol + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestOutputPreviewUsesRetainedValuesWithoutAssumingSchema(t *testing.T) { + for _, test := range []struct { + output map[string]any + want string + }{ + {map[string]any{"summary": "Fixed layout\nand added regression checks", "evidence": "details"}, "Fixed layout and added regression checks"}, + {map[string]any{"artifacts": []string{"report.md"}}, `artifacts: ["report.md"]`}, + {map[string]any{"z": true, "a": json.Number("9007199254740993")}, "a: 9007199254740993"}, + {map[string]any{"input": map[string]any{"prompt": "private task"}}, ""}, + {nil, ""}, + } { + if got := OutputPreview(test.output); got != test.want { + t.Fatal(got, test.want) + } + } + preview := OutputPreview(map[string]any{"summary": strings.Repeat("宽", 500)}) + if len([]rune(preview)) != 280 || !strings.HasSuffix(preview, "…") { + t.Fatal("preview is not bounded by characters") + } +} diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index cbe0c0a9..fa9d35d9 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -23,18 +23,20 @@ type SessionEvidence struct { } type RunSummary struct { - ID int64 `json:"id"` - PipelineID int64 `json:"pipeline_id"` - Name string `json:"name"` - Version int `json:"version"` - ContentHash string `json:"content_hash"` - Status string `json:"status"` - Executing bool `json:"executing"` - Outcome string `json:"outcome"` - CurrentStage string `json:"current_stage"` - CreatedAt string `json:"created_at"` - FinishedAt string `json:"finished_at"` - Diagnostic string `json:"diagnostic"` + ID int64 `json:"id"` + PipelineID int64 `json:"pipeline_id"` + Name string `json:"name"` + Version int `json:"version"` + ContentHash string `json:"content_hash"` + Status string `json:"status"` + Executing bool `json:"executing"` + Outcome string `json:"outcome"` + CurrentStage string `json:"current_stage"` + CreatedAt string `json:"created_at"` + FinishedAt string `json:"finished_at"` + Diagnostic string `json:"diagnostic"` + ResultPreview string `json:"result_preview,omitempty"` + CompletedSteps int `json:"completed_steps"` } type RunStep struct { diff --git a/internal/server/runner.go b/internal/server/runner.go index d45630c7..fc582a94 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -2,6 +2,7 @@ package server import ( "context" + "errors" "fmt" "log" "maps" @@ -157,7 +158,7 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru return nil, err } if created { - ctx, cancel := context.WithTimeout(r.ctx, 30*time.Minute) + ctx, cancel := context.WithCancel(r.ctx) r.active[id] = cancel r.wg.Add(1) go func() { @@ -242,6 +243,7 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, input := stage.Values(request.Input) key := d.Graph.Stages[0].Key visits := map[string]int{} + budgets := map[string]*stage.Budget{} finish := func(status, outcome, diagnostic string) { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -305,10 +307,27 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, } var result stage.Result var err error + stageCtx := ctx + var modelConfig stage.ModelConfig + if stage.EffectiveType(def.Type) == stage.ModelType { + spec, specErr := (stage.ModelStage{}).Spec(def) + if specErr != nil { + finish("failed", "error", specErr.Error()) + return + } + modelConfig = spec.Config + if budgets[key] == nil { + budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens) + } + stageCtx = stage.WithBudget(ctx, budgets[key]) + } if r.execute != nil { - result, err = r.execute(ctx, def, input) + timeout, _ := stage.StageTimeout(def) + executeCtx, cancel := context.WithTimeout(stageCtx, timeout) + result, err = r.execute(executeCtx, def, input) + cancel() } else if useAgent { - result, err = (stage.ModelStage{Client: client}).Run(ctx, def, input) + result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input) } else if stage.EffectiveType(def.Type) == stage.ModelType { plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations} plain.OnGenerationSession = func(session, backend, reference string, active bool) error { @@ -316,12 +335,15 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, defer cancel() return r.store.RunSession(c, id, sequence, session, active, backend, reference) } - result, err = (stage.ModelStage{Client: plain}).Run(ctx, def, input) + result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input) client.Transcript = plain.FormatAudit } else { result, err = stage.DefaultRegistry().Run(ctx, def, input) } step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript)} + if budget := budgets[key]; budget != nil { + step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries) + } if err != nil { step.Status = "failed" step.Diagnostic = err.Error() @@ -343,7 +365,7 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, } } input = stage.Values(step.Output) - if useAgent { + if useAgent || errors.Is(err, stage.ErrStageCleanup) { sessions, checkErr := r.store.ActiveAgentSessions(context.Background()) if checkErr != nil || len(sessions) > 0 { finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work") diff --git a/internal/server/stage_limits_test.go b/internal/server/stage_limits_test.go new file mode 100644 index 00000000..a4c896de --- /dev/null +++ b/internal/server/stage_limits_test.go @@ -0,0 +1,156 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/cli" + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func TestStageBudgetAcrossVisitsThroughCLIAndWebsocket(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dir := t.TempDir() + executable := filepath.Join(dir, "opencode") + script := fmt.Sprintf(`#!/bin/sh +if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi +case "$3" in + /api/session) + n=0; if [ -f %q/count ]; then n=$(cat %q/count); fi + n=$((n+1)); echo "$n" > %q/count + printf '{"data":{"id":"ses_budget_%%s"}}' "$n" ;; + */generate) + id=${3#/api/session/}; id=${id%%/generate} + touch %q/"$id" + echo '{"data":{"text":"{\"outcome\":\"success\",\"output\":{\"summary\":\"done\"}}"}}' ;; + */interrupt) echo '{}' ;; + /api/session/ses_budget_*) + id=${3#/api/session/}; cost=0; tokens=0 + if [ -f %q/"$id" ]; then cost=0.4; tokens=3; fi + printf '{"data":{"id":"%%s","cost":%%s,"tokens":{"input":%%s,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}' "$id" "$cost" "$tokens" ;; + *) exit 99 ;; +esac +`, dir, dir, dir, dir, dir) + if err := os.WriteFile(executable, []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PIPELINES_OPENCODE", executable) + store, err := pipeline.Open(filepath.Join(dir, "runs.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "success -> #success", "success -> worker", 1) + source = strings.Replace(source, "timeout=1h", "timeout=1h\nbudget_usd=1\nbudget_tokens=5", 1) + if _, err := store.Submit(ctx, "budget", source); err != nil { + t.Fatal(err) + } + handler := Handler(ctx, store) + defer func() { cancel(); WaitRuns(handler) }() + srv := httptest.NewServer(handler) + defer srv.Close() + url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" + var out bytes.Buffer + app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} + if err := app.Run(ctx, []string{"--url", url, "run", "budget", "task", "--yes", "--json", "--wait", "--timeout", "10s", "--workspace", dir}); err == nil || !strings.Contains(err.Error(), "run ended failed") { + t.Fatal(err, out.String()) + } + var response protocol.Message + if err := json.Unmarshal(out.Bytes(), &response); err != nil { + t.Fatal(err, out.String()) + } + detail := response.RunDetail + if detail == nil || detail.ResultPreview != "done" || detail.CompletedSteps != 1 { + t.Fatal("partial results missing from run summary", out.String()) + } + if detail == nil || detail.Status != "failed" || len(detail.Steps) != 2 || detail.Steps[0].SessionID == detail.Steps[1].SessionID || detail.Steps[1].AgentActive || !strings.Contains(detail.Steps[1].Diagnostic, "stage budget exceeded") { + t.Fatal("budget run/audit incorrect", out.String()) + } + var audit struct { + Limits struct { + Timeout string + Retries int + Budget stage.BudgetSnapshot + } `json:"stage_limits"` + } + if err := json.Unmarshal([]byte(detail.Steps[1].Transcript), &audit); err != nil { + t.Fatal(err) + } + if audit.Limits.Timeout != time.Hour.String() || audit.Limits.Retries != 2 || audit.Limits.Budget.Attempts != 2 || audit.Limits.Budget.UsedTokens != 6 { + t.Fatal("limits not persisted across visits", audit) + } + count, err := os.ReadFile(filepath.Join(dir, "count")) + if err != nil || strings.TrimSpace(string(count)) != "2" { + t.Fatal("budget failed to block further work", string(count), err) + } + stored, err := store.GetRun(ctx, detail.ID) + if err != nil || stored.Steps[1].Transcript != detail.Steps[1].Transcript { + t.Fatal("evidence was not retained", err) + } +} + +func TestRunnerStageTimeoutAndNoShortOuterDeadline(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "runs.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"worker"}, true) + if _, err := store.Submit(ctx, "default", source); err != nil { + t.Fatal(err) + } + items, _ := store.List(ctx) + r, err := newRunner(ctx, store) + if err != nil { + t.Fatal(err) + } + r.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) { + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) < time.Hour-time.Second { + t.Error("outer run deadline truncated default stage timeout") + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + if _, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}}); err != nil { + t.Fatal(err) + } + r.wg.Wait() + source = strings.Replace(source, "timeout=1h", "timeout=20ms", 1) + if _, err := store.Submit(ctx, "short", source); err != nil { + t.Fatal(err) + } + items, _ = store.List(ctx) + var id int64 + for _, item := range items { + if item.Name == "short" { + id = item.ID + } + } + r.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) { + <-ctx.Done() + return stage.Result{}, ctx.Err() + } + run, err := r.start(protocol.Message{PipelineID: id, Version: 1, RequestKey: "short", Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + detail, err := store.GetRun(ctx, run.ID) + if err != nil || detail.Status != "failed" || !strings.Contains(detail.Steps[0].Diagnostic, "deadline exceeded") { + t.Fatal("per-stage timeout not persisted", detail, err) + } +} diff --git a/internal/stage/agent.go b/internal/stage/agent.go index 3def175e..8396e0ba 100644 --- a/internal/stage/agent.go +++ b/internal/stage/agent.go @@ -14,12 +14,13 @@ import ( // before any prompt is admitted, so restart recovery can interrupt the agent. type AgentClient struct { OpenCodeClient - WritePaths []string - Tools []string // nil is the legacy manual approval bundle; empty denies all - OnSession func(string, bool) error - SessionID string - Transcript string - Policy ToolPolicy + WritePaths []string + Tools []string // nil is the legacy manual approval bundle; empty denies all + OnSession func(string, bool) error + SessionID string + Transcript string + Policy ToolPolicy + WorkStarted bool } func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error { @@ -46,6 +47,8 @@ func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error { } func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) { + c.WorkStarted = false + c.SessionID = "" if c.Operations != nil { c.Operations.Provider, c.Operations.Model = config.Provider, config.Model } @@ -83,18 +86,21 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s return "", fmt.Errorf("retained OpenCode session %s; ownership could not be recorded: %w", c.SessionID, err) } } + var watch *budgetWatch defer func() { + watch.stopWatch(ctx, &err) + defer watch.finish(ctx, &err) cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) defer cancel() if err != nil { if stopErr := c.Interrupt(cleanup, c.SessionID); stopErr != nil { - err = fmt.Errorf("agent interrupt failed; retained session %s needs inspection: %w", c.SessionID, errors.Join(err, stopErr)) + err = fmt.Errorf("agent interrupt failed; retained session %s needs inspection: %w", c.SessionID, errors.Join(err, stopErr, ErrStageCleanup)) return } } if c.OnSession != nil { if persistErr := c.OnSession(c.SessionID, false); persistErr != nil { - err = errors.Join(err, persistErr) + err = errors.Join(err, persistErr, ErrStageCleanup) text = "" return } @@ -121,6 +127,11 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s if c.Policy.Shell != nil { prompt += "\nPermitted shell commands: " + strings.Join(c.Policy.Shell, "; ") } + ctx, watch, err = c.watchBudget(ctx, config.Budget, c.SessionID) + if err != nil { + return "", err + } + c.WorkStarted = true if _, err = c.call(ctx, "post", "/api/session/"+c.SessionID+"/prompt", generationBody("text", prompt, config, c.Backend)); err != nil { return "", err } diff --git a/internal/stage/budget.go b/internal/stage/budget.go new file mode 100644 index 00000000..3c20fcd9 --- /dev/null +++ b/internal/stage/budget.go @@ -0,0 +1,292 @@ +package stage + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "math" + "strconv" + "sync" + "time" +) + +const DefaultRetries = 2 +const DefaultTimeout = time.Hour + +var ErrBudgetExceeded = errors.New("stage budget exceeded") +var ErrBudgetAccounting = errors.New("stage budget accounting unavailable") +var ErrStageCleanup = errors.New("stage cleanup not confirmed") + +func StageTimeout(def Definition) (time.Duration, error) { + if def.Configs["timeout"] == "" { + return DefaultTimeout, nil + } + duration, err := time.ParseDuration(def.Configs["timeout"]) + if err != nil || duration <= 0 { + return 0, fmt.Errorf("timeout must be a positive duration, such as 1h or 30m") + } + return duration, nil +} + +func parseBudget(def Definition) (float64, int64, error) { + var usd float64 + var tokens int64 + var err error + if value := def.Configs["budget_usd"]; value != "" { + usd, err = strconv.ParseFloat(value, 64) + if err != nil || usd <= 0 || math.IsNaN(usd) || math.IsInf(usd, 0) { + return 0, 0, fmt.Errorf("budget_usd must be a positive finite USD amount") + } + } + if value := def.Configs["budget_tokens"]; value != "" { + tokens, err = strconv.ParseInt(value, 10, 64) + if err != nil || tokens <= 0 { + return 0, 0, fmt.Errorf("budget_tokens must be a positive integer") + } + } + return usd, tokens, nil +} + +type BudgetSnapshot struct { + LimitUSD float64 `json:"limit_usd,omitempty"` + LimitTokens int64 `json:"limit_tokens,omitempty"` + UsedUSD float64 `json:"used_usd"` + UsedTokens int64 `json:"used_tokens"` + USDKnown bool `json:"usd_known"` + TokensKnown bool `json:"tokens_known"` + Attempts int `json:"attempts"` +} + +type reportedUsage struct { + USD *float64 `json:"cost"` + Tokens *struct { + Input, Output, Reasoning *int64 + Cache *struct{ Read, Write *int64 } + } `json:"tokens"` +} + +type Budget struct { + mu sync.Mutex + state BudgetSnapshot + records map[string]BudgetSnapshot + pollInterval time.Duration +} + +func NewBudget(usd float64, tokens int64) *Budget { + return &Budget{state: BudgetSnapshot{LimitUSD: usd, LimitTokens: tokens}, records: map[string]BudgetSnapshot{}, pollInterval: time.Second} +} +func (b *Budget) Enabled() bool { return b != nil && (b.state.LimitUSD > 0 || b.state.LimitTokens > 0) } +func (b *Budget) Snapshot() BudgetSnapshot { b.mu.Lock(); defer b.mu.Unlock(); return b.state } +func (b *Budget) Attempt() { b.mu.Lock(); b.state.Attempts++; b.mu.Unlock() } +func (b *Budget) checkLocked() error { + if b.state.LimitUSD > 0 && b.state.UsedUSD >= b.state.LimitUSD { + return fmt.Errorf("%w: USD %.6f / %.6f", ErrBudgetExceeded, b.state.UsedUSD, b.state.LimitUSD) + } + if b.state.LimitTokens > 0 && b.state.UsedTokens >= b.state.LimitTokens { + return fmt.Errorf("%w: tokens %d / %d", ErrBudgetExceeded, b.state.UsedTokens, b.state.LimitTokens) + } + return nil +} +func (b *Budget) Check() error { b.mu.Lock(); defer b.mu.Unlock(); return b.checkLocked() } + +func (b *Budget) observe(key string, usage reportedUsage) error { + b.mu.Lock() + defer b.mu.Unlock() + old := b.records[key] + defer func() { b.records[key] = old }() + if usage.USD != nil { + if *usage.USD < 0 || math.IsNaN(*usage.USD) || math.IsInf(*usage.USD, 0) { + return fmt.Errorf("%w: invalid cost", ErrBudgetAccounting) + } + value := max(old.UsedUSD, *usage.USD) + total := b.state.UsedUSD + (value - old.UsedUSD) + if math.IsNaN(total) || math.IsInf(total, 0) { + return fmt.Errorf("%w: cost overflow", ErrBudgetAccounting) + } + b.state.UsedUSD = total + old.UsedUSD, old.USDKnown, b.state.USDKnown = value, true, true + } else if b.state.LimitUSD > 0 { + return fmt.Errorf("%w: cost was not reported", ErrBudgetAccounting) + } + t := usage.Tokens + if t != nil && t.Input != nil && t.Output != nil && t.Reasoning != nil && t.Cache != nil && t.Cache.Read != nil && t.Cache.Write != nil { + var total int64 + for _, value := range []int64{*t.Input, *t.Output, *t.Reasoning, *t.Cache.Read, *t.Cache.Write} { + if value < 0 || value > math.MaxInt64-total { + return fmt.Errorf("%w: invalid token count", ErrBudgetAccounting) + } + total += value + } + total = max(total, old.UsedTokens) + if total-old.UsedTokens > math.MaxInt64-b.state.UsedTokens { + return fmt.Errorf("%w: token count overflow", ErrBudgetAccounting) + } + b.state.UsedTokens += total - old.UsedTokens + old.UsedTokens, old.TokensKnown, b.state.TokensKnown = total, true, true + } else if b.state.LimitTokens > 0 { + return fmt.Errorf("%w: complete token usage was not reported", ErrBudgetAccounting) + } + return b.checkLocked() +} + +type budgetContextKey struct{} + +func WithBudget(ctx context.Context, budget *Budget) context.Context { + return context.WithValue(ctx, budgetContextKey{}, budget) +} +func budgetFromContext(ctx context.Context) *Budget { + b, _ := ctx.Value(budgetContextKey{}).(*Budget) + return b +} + +type budgetWatch struct { + read func(context.Context, bool) error + cancel context.CancelCauseFunc + stop chan struct{} + done chan struct{} +} + +func (c OpenCodeClient) BudgetCapable() bool { return true } + +func (c OpenCodeClient) watchBudget(ctx context.Context, budget *Budget, id string) (context.Context, *budgetWatch, error) { + if !budget.Enabled() { + return ctx, nil, nil + } + if err := budget.Check(); err != nil { + return ctx, nil, err + } + // Poll through an independent reader. Native calls have per-request mutable + // diagnostic fields, which must not race the long-running prompt request. + reader := c + reader.Operations = nil + w := &budgetWatch{stop: make(chan struct{}), done: make(chan struct{})} + w.read = func(parent context.Context, final bool) error { + readCtx, cancel := context.WithTimeout(parent, 2*time.Second) + defer cancel() + var data []byte + var err error + if reader.connection != nil && reader.Backend == "v1" { + data, err = reader.connection.BudgetMessages(readCtx, id) + } else { + data, err = reader.call(readCtx, "get", "/api/session/"+id, nil) + } + if err != nil { + return fmt.Errorf("%w: usage read failed: %v", ErrBudgetAccounting, err) + } + return budget.readUsage(id, data, reader.Backend == "v1", final) + } + if err := w.read(ctx, false); err != nil { + return ctx, nil, err + } + workCtx, cancel := context.WithCancelCause(ctx) + w.cancel = cancel + go func() { + defer close(w.done) + ticker := time.NewTicker(budget.pollInterval) + defer ticker.Stop() + for { + select { + case <-w.stop: + return + case <-workCtx.Done(): + return + case <-ticker.C: + if err := w.read(workCtx, false); err != nil { + if workCtx.Err() == nil { + cancel(err) + } + return + } + } + } + }() + return workCtx, w, nil +} + +func (w *budgetWatch) stopWatch(ctx context.Context, err *error) { + if w == nil { + return + } + close(w.stop) + <-w.done + if cause := context.Cause(ctx); errors.Is(cause, ErrBudgetExceeded) || errors.Is(cause, ErrBudgetAccounting) { + *err = errors.Join(*err, cause) + } +} +func (w *budgetWatch) finish(ctx context.Context, err *error) { + if w == nil { + return + } + defer w.cancel(nil) + if usageErr := w.read(context.WithoutCancel(ctx), true); usageErr != nil { + *err = errors.Join(*err, usageErr) + } +} + +func (b *Budget) readUsage(id string, data []byte, native bool, final ...bool) error { + if !native { + var response struct { + Data struct { + ID string `json:"id"` + reportedUsage + } `json:"data"` + } + if json.Unmarshal(data, &response) != nil || response.Data.ID != id { + return fmt.Errorf("%w: invalid or mismatched usage response", ErrBudgetAccounting) + } + return b.observe(id, response.Data.reportedUsage) + } + var messages []struct { + Info struct { + ID, Role, SessionID string + Time struct{ Completed int64 } + reportedUsage + } + } + if len(bytes.TrimSpace(data)) == 0 || bytes.TrimSpace(data)[0] != '[' || json.Unmarshal(data, &messages) != nil { + return fmt.Errorf("%w: invalid native usage response", ErrBudgetAccounting) + } + var accountingErr error + completed := 0 + for _, m := range messages { + if m.Info.Role != "assistant" { + continue + } + if m.Info.Time.Completed != 0 { + completed++ + } + if m.Info.Time.Completed == 0 && (m.Info.USD == nil || m.Info.Tokens == nil) { + continue + } + if m.Info.ID == "" || (m.Info.SessionID != "" && m.Info.SessionID != id) { + return fmt.Errorf("%w: invalid native message identity", ErrBudgetAccounting) + } + if err := b.observe(id+"/"+m.Info.ID, m.Info.reportedUsage); err != nil { + if errors.Is(err, ErrBudgetAccounting) && accountingErr == nil { + accountingErr = err + } + } + } + if accountingErr != nil { + return accountingErr + } + if len(final) > 0 && final[0] && completed == 0 { + return fmt.Errorf("%w: no completed assistant usage was reported", ErrBudgetAccounting) + } + return b.Check() +} + +func BudgetTranscript(original string, budget *Budget, timeout time.Duration, retries int) string { + var data map[string]json.RawMessage + if json.Unmarshal([]byte(original), &data) != nil || data == nil { + data = map[string]json.RawMessage{} + if original != "" { + data["original_transcript"], _ = json.Marshal(original) + } + } + data["stage_limits"], _ = json.Marshal(map[string]any{"timeout": timeout.String(), "retries": retries, "budget": budget.Snapshot()}) + encoded, _ := json.Marshal(data) + return string(encoded) +} diff --git a/internal/stage/budget_test.go b/internal/stage/budget_test.go new file mode 100644 index 00000000..6a50a97f --- /dev/null +++ b/internal/stage/budget_test.go @@ -0,0 +1,315 @@ +package stage + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "math" + "strings" + "sync/atomic" + "testing" + "time" +) + +func usageResponse(id string, usd float64, tokens int64) []byte { + return []byte(fmt.Sprintf(`{"data":{"id":%q,"cost":%g,"tokens":{"input":%d,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}`, id, usd, tokens)) +} + +func TestStageLimitDefaultsAndValidation(t *testing.T) { + def := modelDefinition() + delete(def.Configs, "retries") + spec, err := (ModelStage{}).Spec(def) + if err != nil || spec.Config.Retries != 2 || spec.Config.Timeout != time.Hour || spec.Config.BudgetUSD != 0 || spec.Config.BudgetTokens != 0 { + t.Fatal("wrong defaults", spec, err) + } + def.Configs["timeout"], def.Configs["budget_usd"], def.Configs["budget_tokens"] = "30m", "1.25", "50000" + spec, err = (ModelStage{}).Spec(def) + if err != nil || spec.Config.Timeout != 30*time.Minute || spec.Config.BudgetUSD != 1.25 || spec.Config.BudgetTokens != 50000 { + t.Fatal("limits not parsed", spec, err) + } + for _, value := range []struct{ key, value string }{{"timeout", "0s"}, {"timeout", "-1h"}, {"timeout", "1"}, {"budget_usd", "NaN"}, {"budget_usd", "Inf"}, {"budget_usd", "0"}, {"budget_usd", "-1"}, {"budget_tokens", "0"}, {"budget_tokens", "-1"}, {"budget_tokens", "1.5"}, {"budget_tokens", "9223372036854775808"}} { + copy := def + copy.Configs = maps.Clone(def.Configs) + copy.Configs[value.key] = value.value + if err := (ModelStage{}).Validate(copy); err == nil { + t.Fatal("accepted invalid limit", value) + } + } +} + +func TestDefaultRetriesAndTimeoutResetEachAttempt(t *testing.T) { + def := modelDefinition() + delete(def.Configs, "retries") + calls := 0 + client := generatorFunc(func(ctx context.Context, _ ModelConfig, _ string) (string, error) { + calls++ + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) < time.Hour-time.Second { + t.Fatal("default attempt timeout is not one hour") + } + return "", &openCodeError{status: 503} + }) + if _, err := (ModelStage{Client: client}).Run(context.Background(), def, nil); err == nil || calls != 3 { + t.Fatal("default retry count wrong", calls, err) + } + def.Configs["timeout"] = "20ms" + calls = 0 + client = generatorFunc(func(ctx context.Context, _ ModelConfig, _ string) (string, error) { + calls++ + deadline, _ := ctx.Deadline() + if time.Until(deadline) <= 0 { + t.Fatal("attempt inherited expired timeout") + } + if calls == 1 { + <-ctx.Done() + return "", ctx.Err() + } + return `{"outcome":"success","output":{}}`, nil + }) + if _, err := (ModelStage{Client: client}).Run(context.Background(), def, nil); err != nil || calls != 2 { + t.Fatal("timeout did not reset for retry", calls, err) + } +} + +func TestBudgetsAccumulateAcrossRetriesAndBlockFurtherGeneration(t *testing.T) { + for _, test := range []struct{ name, key, value string }{{"USD", "budget_usd", "0.5"}, {"tokens", "budget_tokens", "5"}} { + t.Run(test.name, func(t *testing.T) { + def := modelDefinition() + def.Configs["retries"] = "5" + def.Configs[test.key] = test.value + var sessions, generations atomic.Int64 + client := &OpenCodeClient{command: func(ctx context.Context, _ string, args ...string) ([]byte, error) { + path := args[2] + if path == "/api/session" { + return []byte(fmt.Sprintf(`{"data":{"id":"ses_attempt_%d"}}`, sessions.Add(1))), nil + } + if args[1] == "get" { + cost, tokens := 0.0, int64(0) + if generations.Load() >= sessions.Load() { + cost, tokens = .4, 3 + } + return usageResponse(strings.TrimPrefix(path, "/api/session/"), cost, tokens), nil + } + if strings.HasSuffix(path, "/generate") { + generations.Add(1) + return nil, &openCodeError{status: 503} + } + return []byte(`{}`), nil + }} + _, err := (ModelStage{Client: client}).Run(context.Background(), def, nil) + if !errors.Is(err, ErrBudgetExceeded) || generations.Load() != 2 { + t.Fatal("retries bypassed shared budget", generations.Load(), err) + } + }) + } +} + +func TestBudgetAccountingFailsClosedBeforeGeneration(t *testing.T) { + def := modelDefinition() + def.Configs["budget_usd"], def.Configs["budget_tokens"] = "1", "10" + var generated atomic.Int64 + client := &OpenCodeClient{command: func(_ context.Context, _ string, args ...string) ([]byte, error) { + if args[2] == "/api/session" { + return []byte(`{"data":{"id":"ses_missing"}}`), nil + } + if args[1] == "get" { + return []byte(`{"data":{"id":"ses_missing"}}`), nil + } + if strings.HasSuffix(args[2], "/generate") { + generated.Add(1) + } + return []byte(`{}`), nil + }} + if _, err := (ModelStage{Client: client}).Run(context.Background(), def, nil); !errors.Is(err, ErrBudgetAccounting) || generated.Load() != 0 { + t.Fatal("unknown usage treated as zero", err, generated.Load()) + } + calls := 0 + unsupported := generatorFunc(func(context.Context, ModelConfig, string) (string, error) { calls++; return "", nil }) + if _, err := (ModelStage{Client: unsupported}).Run(context.Background(), def, nil); !errors.Is(err, ErrBudgetAccounting) || calls != 0 { + t.Fatal("unsupported generator bypassed limits", err, calls) + } +} + +func TestLiveAgentBudgetInterruptsAndDoesNotReplay(t *testing.T) { + def := modelDefinition() + def.Configs["budget_tokens"] = "10" + def.Configs["retries"] = "2" + def.Blocks["tools"] = "read" + budget := NewBudget(0, 10) + budget.pollInterval = time.Millisecond + var prompts, interrupts atomic.Int64 + var inactive atomic.Bool + client := &AgentClient{OpenCodeClient: OpenCodeClient{command: func(ctx context.Context, _ string, args ...string) ([]byte, error) { + path := args[2] + if path == "/api/session" { + return []byte(`{"data":{"id":"ses_agent_budget"}}`), nil + } + if args[1] == "get" { + tokens := int64(0) + if prompts.Load() > 0 { + tokens = 12 + } + return usageResponse("ses_agent_budget", .1, tokens), nil + } + if strings.HasSuffix(path, "/prompt") { + prompts.Add(1) + return []byte(`{}`), nil + } + if strings.HasSuffix(path, "/wait") { + <-ctx.Done() + return nil, ctx.Err() + } + if strings.HasSuffix(path, "/interrupt") { + interrupts.Add(1) + return []byte(`{}`), nil + } + return nil, fmt.Errorf("unexpected operation %s", path) + }}, OnSession: func(_ string, active bool) error { + if !active { + inactive.Store(true) + } + return nil + }} + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _, err := (ModelStage{Client: client}).Run(WithBudget(ctx, budget), def, nil) + if !errors.Is(err, ErrBudgetExceeded) || prompts.Load() != 1 || interrupts.Load() != 1 || !inactive.Load() || budget.Snapshot().UsedTokens != 12 { + t.Fatal("live limit did not stop and retain agent safely", err, prompts.Load(), interrupts.Load(), inactive.Load(), budget.Snapshot()) + } +} + +func TestBudgetLedgerDeduplicatesPollsAndIncludesCacheAndReasoning(t *testing.T) { + b := NewBudget(10, 100) + data := []byte(`{"data":{"id":"ses_ledger","cost":0.2,"tokens":{"input":3,"output":4,"reasoning":5,"cache":{"read":6,"write":7}}}}`) + for i := 0; i < 5; i++ { + if err := b.readUsage("ses_ledger", data, false); err != nil { + t.Fatal(err) + } + } + if s := b.Snapshot(); s.UsedTokens != 25 || s.UsedUSD != .2 { + t.Fatal("polls double counted or cache/reasoning omitted", s) + } + if err := b.readUsage("ses_other", data, false); !errors.Is(err, ErrBudgetAccounting) { + t.Fatal("accepted another session's accounting", err) + } + for _, invalid := range []string{`{"data":{"id":"ses_ledger","cost":-1}}`, `{"data":{"id":"ses_ledger","cost":0.1,"tokens":{"input":-1,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}`} { + if err := b.readUsage("ses_ledger", []byte(invalid), false); !errors.Is(err, ErrBudgetAccounting) { + t.Fatal("invalid usage accepted", err) + } + } + var original map[string]json.RawMessage + if err := json.Unmarshal([]byte(BudgetTranscript(`{"large":9007199254740993}`, b, time.Hour, 2)), &original); err != nil || string(original["large"]) != "9007199254740993" { + t.Fatal("limit audit changed original evidence", err, string(original["large"])) + } +} + +func TestNativeBudgetCountsMessagesAcrossCompaction(t *testing.T) { + b := NewBudget(1, 100) + message := func(id string, cost float64, tokens int64) string { + return fmt.Sprintf(`{"info":{"id":%q,"sessionID":"ses_native","role":"assistant","time":{"completed":1},"cost":%g,"tokens":{"input":%d,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}`, id, cost, tokens) + } + first := []byte("[" + message("msg_1", .2, 10) + "," + message("msg_2", .3, 20) + "]") + if err := b.readUsage("ses_native", first, true); err != nil { + t.Fatal(err) + } + if err := b.readUsage("ses_native", []byte("["+message("msg_2", .3, 20)+","+message("msg_3", .4, 30)+"]"), true); err != nil { + t.Fatal(err) + } + s := b.Snapshot() + if s.UsedTokens != 60 || math.Abs(s.UsedUSD-.9) > 1e-9 { + t.Fatal("removed messages reset accumulated usage", s) + } + if err := b.readUsage("ses_native", []byte(`[{"info":{"id":"msg_missing","role":"assistant","time":{"completed":1}}}]`), true); !errors.Is(err, ErrBudgetAccounting) { + t.Fatal("missing completed-message accounting accepted", err) + } +} + +func TestNativeFinalUsageCannotBeMissing(t *testing.T) { + b := NewBudget(1, 100) + for _, data := range []string{`[]`, `[{"info":{"id":"msg_inflight","role":"assistant","time":{},"cost":0,"tokens":{"input":0,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}]`} { + if err := b.readUsage("ses_native", []byte(data), true, true); !errors.Is(err, ErrBudgetAccounting) { + t.Fatal("missing final accounting accepted", err) + } + } +} + +func TestAgentRetriesBeforeAdmissionButNeverReplaysWork(t *testing.T) { + def := modelDefinition() + delete(def.Configs, "retries") + def.Blocks["tools"] = "read" + sessions, prompts := 0, 0 + client := &AgentClient{OpenCodeClient: OpenCodeClient{command: func(_ context.Context, _ string, args ...string) ([]byte, error) { + if args[2] == "/api/session" { + sessions++ + if sessions == 1 { + return nil, &openCodeError{status: 503} + } + return []byte(`{"data":{"id":"ses_admitted"}}`), nil + } + if strings.HasSuffix(args[2], "/prompt") { + prompts++ + return nil, &openCodeError{status: 503} + } + return []byte(`{}`), nil + }}} + _, err := (ModelStage{Client: client}).Run(context.Background(), def, nil) + if err == nil || sessions != 2 || prompts != 1 { + t.Fatal("admitted work replayed or safe retry skipped", sessions, prompts, err) + } +} + +func TestCleanupFailurePreventsTextGenerationRetry(t *testing.T) { + def := modelDefinition() + delete(def.Configs, "retries") + generated := 0 + client := &OpenCodeClient{command: func(_ context.Context, _ string, args ...string) ([]byte, error) { + if args[2] == "/api/session" { + return []byte(`{"data":{"id":"ses_cleanup"}}`), nil + } + if strings.HasSuffix(args[2], "/generate") { + generated++ + return nil, &openCodeError{status: 503} + } + return nil, errors.New("interrupt unavailable") + }} + if _, err := (ModelStage{Client: client}).Run(context.Background(), def, nil); !errors.Is(err, ErrStageCleanup) || generated != 1 { + t.Fatal("retried before cleanup confirmed", generated, err) + } +} + +func TestFormatRepairUsesTheSameBudget(t *testing.T) { + def := modelDefinition() + def.Configs["budget_tokens"] = "5" + budget := NewBudget(0, 5) + sessions, generated := 0, 0 + client := &OpenCodeClient{command: func(_ context.Context, _ string, args ...string) ([]byte, error) { + path := args[2] + if path == "/api/session" { + sessions++ + return []byte(fmt.Sprintf(`{"data":{"id":"ses_repair_%d"}}`, sessions)), nil + } + if args[1] == "get" { + tokens := int64(0) + if generated >= sessions { + tokens = 3 + } + return usageResponse(strings.TrimPrefix(path, "/api/session/"), .1, tokens), nil + } + if strings.HasSuffix(path, "/generate") { + generated++ + text := `{"outcome":"success" "output":{}}` + if generated > 1 { + text = `{"outcome":"success","output":{}}` + } + data, _ := json.Marshal(map[string]any{"data": map[string]string{"text": text}}) + return data, nil + } + return []byte(`{}`), nil + }} + _, err := (ModelStage{Client: client}).Run(WithBudget(context.Background(), budget), def, nil) + if !errors.Is(err, ErrBudgetExceeded) || sessions != 2 || budget.Snapshot().UsedTokens != 6 { + t.Fatal("repair bypassed execution budget", sessions, budget.Snapshot(), err) + } +} diff --git a/internal/stage/model.go b/internal/stage/model.go index 2eeb7bfa..6e51d39d 100644 --- a/internal/stage/model.go +++ b/internal/stage/model.go @@ -2,6 +2,7 @@ package stage import ( "context" + "errors" "fmt" "regexp" "strconv" @@ -26,6 +27,10 @@ type ModelConfig struct { Provider string Model string Retries int + Timeout time.Duration + BudgetUSD float64 + BudgetTokens int64 + Budget *Budget ResultSchema map[string]any } @@ -73,14 +78,14 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { if _, err := StagePermissions(def); err != nil { return ModelSpec{}, err } - for _, key := range []string{"provider", "model", "retries"} { + for _, key := range []string{"provider", "model"} { if def.Configs[key] == "" { return ModelSpec{}, fmt.Errorf("missing config %s", key) } } for key := range def.Configs { switch key { - case "provider", "model", "retries": + case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens": default: return ModelSpec{}, fmt.Errorf("unknown config %q", key) } @@ -88,13 +93,25 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { if !ValidModel(def.Configs["provider"], def.Configs["model"]) { return ModelSpec{}, fmt.Errorf("invalid provider/model identifier; use the exact IDs validated during setup") } - retries, err := strconv.Atoi(def.Configs["retries"]) + retries := DefaultRetries + var err error + if value := def.Configs["retries"]; value != "" { + retries, err = strconv.Atoi(value) + } if err != nil || retries < 0 || retries > 100 { return ModelSpec{}, fmt.Errorf("retries must be an integer from 0 to 100") } + timeout, err := StageTimeout(def) + if err != nil { + return ModelSpec{}, err + } + usd, tokens, err := parseBudget(def) + if err != nil { + return ModelSpec{}, err + } return ModelSpec{ Output: def.Blocks["output"], Prompt: def.Blocks["prompt"], - Config: ModelConfig{Provider: def.Configs["provider"], Model: def.Configs["model"], Retries: retries}, + Config: ModelConfig{Provider: def.Configs["provider"], Model: def.Configs["model"], Retries: retries, Timeout: timeout, BudgetUSD: usd, BudgetTokens: tokens}, }, nil } @@ -111,8 +128,10 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu if err != nil { return Result{}, err } - ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) - defer cancel() + spec.Config.Budget = budgetFromContext(ctx) + if spec.Config.Budget == nil { + spec.Config.Budget = NewBudget(spec.Config.BudgetUSD, spec.Config.BudgetTokens) + } client := m.Client if client == nil { tools, _, _ := ToolNames(def) @@ -121,35 +140,59 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu } client = OpenCodeClient{} } - // Replaying an agent can repeat file mutations. Graph feedback edges, not - // transport/result retries, own additional tool-enabled attempts. + if spec.Config.Budget.Enabled() { + capable, ok := client.(interface{ BudgetCapable() bool }) + if !ok || !capable.BudgetCapable() { + return Result{}, fmt.Errorf("%w: generator cannot enforce usage limits", ErrBudgetAccounting) + } + } + // Replaying admitted tool work can repeat mutations. Only pre-admission + // transport errors may automatically retry an audited agent. if agent, ok := client.(*AgentClient); ok { if tools, explicit, _ := ToolNames(def); explicit { agent.Tools = tools } agent.Policy, _ = StagePermissions(def) - spec.Config.Retries = 0 } for attempt := 0; attempt <= spec.Config.Retries; attempt++ { if err := ctx.Err(); err != nil { return Result{}, err } - var text string - text, err = client.Generate(ctx, spec.Config, prompt) - if err == nil { - var result Result - result, err = decodeModelResult(stageResultText(text), def.Edges) - if err == nil { - return result, nil - } - // Formatting retries are isolated from execution retries. Never replay - // stage instructions or tool actions to fix malformed final JSON. - return repairModelResult(ctx, client, spec.Config, text, def.Edges, err) + if err := spec.Config.Budget.Check(); err != nil { + return Result{}, err } + spec.Config.Budget.Attempt() + attemptCtx, cancel := context.WithTimeout(ctx, spec.Config.Timeout) + text, generateErr := client.Generate(attemptCtx, spec.Config, prompt) + if generateErr == nil && attemptCtx.Err() != nil { + generateErr = attemptCtx.Err() + } + if generateErr == nil { + result, resultErr := decodeModelResult(stageResultText(text), def.Edges) + if resultErr != nil { + result, resultErr = repairModelResult(attemptCtx, client, spec.Config, text, def.Edges, resultErr) + } + if attemptCtx.Err() != nil { + resultErr = errors.Join(resultErr, attemptCtx.Err()) + } + cancel() + return result, resultErr + } + err = generateErr + if attemptCtx.Err() != nil && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) { + err = errors.Join(err, attemptCtx.Err()) + } + cancel() if ctx.Err() != nil { return Result{}, ctx.Err() } - if attempt == spec.Config.Retries || !retryableModelError(err) { + if errors.Is(err, ErrBudgetExceeded) || errors.Is(err, ErrBudgetAccounting) || errors.Is(err, ErrStageCleanup) { + break + } + if agent, ok := client.(*AgentClient); ok && agent.WorkStarted { + break + } + if attempt == spec.Config.Retries || (!retryableModelError(err) && !errors.Is(err, context.DeadlineExceeded)) { break } delay := time.Duration(1< 32<<10 { diff --git a/internal/tui/dashboard.go b/internal/tui/dashboard.go index 5ec81fcf..6677b8c8 100644 --- a/internal/tui/dashboard.go +++ b/internal/tui/dashboard.go @@ -47,12 +47,15 @@ func (m Model) homeLines() []string { panelWidth = (m.width - 2) / 2 } runs := []string{bar("Latest runs", fmt.Sprint(len(m.runs)), max(1, panelWidth-4), true), - lipgloss.NewStyle().Foreground(themeMuted).Render("Pipeline · Version · Outcome · Duration")} + lipgloss.NewStyle().Foreground(themeMuted).Render("Outcome · Result · Completed work")} if len(m.runs) == 0 { runs = append(runs, "No runs yet.") } for _, r := range m.runs[:min(8, len(m.runs))] { - runs = append(runs, lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("#%d %s", r.ID, r.Name))+fmt.Sprintf(" · v%d · %s · %s", r.Version, m.progress(r.Status), runDuration(r.CreatedAt, r.FinishedAt)), lipgloss.NewStyle().Foreground(themeMuted).Render(" Started: "+dateLabel(r.CreatedAt))) + runs = append(runs, lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("#%d %s", r.ID, r.Name))+fmt.Sprintf(" · v%d · %s · %s", r.Version, m.progress(r.Status), runDuration(r.CreatedAt, r.FinishedAt)), " "+resultLabel(r)) + if r.CompletedSteps > 0 { + runs = append(runs, fmt.Sprintf(" %d completed visits", r.CompletedSteps)) + } if r.CurrentStage != "" && (r.Status == "running" || r.Status == "queued") { runs = append(runs, " Stage: "+r.CurrentStage) } @@ -70,7 +73,13 @@ func (m Model) homeLines() []string { for _, p := range m.pipelines { lines = append(lines, lipgloss.NewStyle().Bold(true).Render(p.Name)+lipgloss.NewStyle().Foreground(themeMuted).Render(fmt.Sprintf(" v%d · %d stages", p.Version, p.Stages)), - lipgloss.NewStyle().Foreground(themeMuted).Render(" Created: "+dateLabel(p.CreatedAt)), lipgloss.NewStyle().Foreground(themeMuted).Render(" Updated: "+dateLabel(p.UpdatedAt))) + lipgloss.NewStyle().Foreground(themeMuted).Render(" Updated: "+dateLabel(p.UpdatedAt))) + if r := m.latestPipelineRun(p.ID, 0); r != nil { + lines = append(lines, fmt.Sprintf(" Last run #%d · v%d · %s", r.ID, r.Version, m.progress(r.Status)), " "+resultLabel(*r)) + } else { + lines = append(lines, " Not run yet · no recorded results") + } + lines = append(lines, lipgloss.NewStyle().Foreground(themeMuted).Render(" Created: "+dateLabel(p.CreatedAt))) if m.usage != nil { for _, u := range m.usage.Pipelines { if u.PipelineID == p.ID { diff --git a/internal/tui/model.go b/internal/tui/model.go index df4452fc..26f9717f 100644 --- a/internal/tui/model.go +++ b/internal/tui/model.go @@ -338,8 +338,8 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { accepted := m.runOpen && msg.RunDetail != nil && ((msg.Type == protocol.Run && msg.RunDetail.ID == m.requestedRunID) || (msg.Type == protocol.RunStarted && msg.ID == m.runStartRequest)) if accepted { if m.runDetail == nil { - m.runGraphFocus = true - m.runDetailFocus = false + m.runGraphFocus = false + m.runDetailFocus = true m.runGraphTop, m.runGraphPan = 0, 0 } if msg.Type == protocol.RunStarted { diff --git a/internal/tui/panels.go b/internal/tui/panels.go index 7258ad8b..ea6ae505 100644 --- a/internal/tui/panels.go +++ b/internal/tui/panels.go @@ -10,6 +10,7 @@ import ( "github.com/charmbracelet/x/ansi" "git.a-wan.net/alan/pipelines/internal/protocol" + stageconfig "git.a-wan.net/alan/pipelines/internal/stage" ) type paneGeometry struct { @@ -17,7 +18,7 @@ type paneGeometry struct { } func (m Model) pipelineGeometry(height int) paneGeometry { - remaining := max(0, height-3) + remaining := max(0, height-3-len(m.pipelineResultLines())) if remaining >= 15 { graph := remaining * 3 / 4 if m.stageFocus { @@ -122,7 +123,9 @@ func (m Model) pipelineToolbar() []string { meta += " · historical, inactive" } } - return []string{buttons, meta, ""} + lines := []string{buttons} + lines = append(lines, m.pipelineResultLines()...) + return append(lines, meta, "") } func (m Model) pipelineView(height int) string { @@ -284,6 +287,19 @@ func (m Model) stageFields() []stageField { for _, k := range keys { configs = append(configs, lipgloss.NewStyle().Foreground(themeMuted).Render(k+" ")+stage.Configs[k]) } + if stage.Configs["timeout"] == "" { + configs = append(configs, fmt.Sprintf("timeout %s (default per attempt)", stageconfig.DefaultTimeout)) + } + if stage.Type == "" || stage.Type == stageconfig.ModelType { + if stage.Configs["retries"] == "" { + configs = append(configs, fmt.Sprintf("retries %d (default)", stageconfig.DefaultRetries)) + } + for _, key := range []string{"budget_usd", "budget_tokens"} { + if stage.Configs[key] == "" { + configs = append(configs, key+" unlimited") + } + } + } var edges []graphEdge maxLabel := 0 for _, edge := range graphEdges(m.detail.Graph) { diff --git a/internal/tui/results.go b/internal/tui/results.go new file mode 100644 index 00000000..77f574b8 --- /dev/null +++ b/internal/tui/results.go @@ -0,0 +1,131 @@ +package tui + +import ( + "fmt" + "strings" + + "git.a-wan.net/alan/pipelines/internal/protocol" + "github.com/charmbracelet/x/ansi" +) + +func resultLabel(r protocol.RunSummary) string { + preview := ansi.Strip(r.ResultPreview) + if r.Status == "succeeded" { + if preview != "" { + return "Result: " + preview + } + return "Finished · no result recorded" + } + if r.Diagnostic != "" { + return "Stopped: " + ansi.Strip(r.Diagnostic) + } + if preview != "" { + return "Latest output: " + preview + } + if r.Status == "running" || r.Status == "queued" { + return "In progress · awaiting stage output" + } + return "No result recorded" +} + +func (m Model) latestPipelineRun(id int64, version int) *protocol.RunSummary { + var latest *protocol.RunSummary + for i := range m.runs { + r := &m.runs[i] + if r.PipelineID == id && (version == 0 || r.Version == version) && (latest == nil || r.ID > latest.ID) { + latest = r + } + } + return latest +} + +func (m Model) pipelineResultLines() []string { + if m.detail == nil { + return nil + } + r := m.latestPipelineRun(m.detail.ID, m.detail.Version) + if r == nil { + return []string{"Results · This version has not run yet. No accomplishments recorded."} + } + lines := []string{fmt.Sprintf("Results · run #%d · %s · %d completed visits · %s", r.ID, ansi.Strip(m.progress(r.Status)), r.CompletedSteps, runDuration(r.CreatedAt, r.FinishedAt)), resultLabel(*r)} + if r.Status != "succeeded" && r.ResultPreview != "" && r.Diagnostic != "" { + lines = append(lines, "Partial output: "+ansi.Strip(r.ResultPreview)) + } + return lines +} + +func runResultLines(r *protocol.RunDetail) []string { + lines := []string{} + if r.Diagnostic != "" { + lines = append(lines, "Stopped: "+ansi.Strip(r.Diagnostic)) + } + if r.Status == "succeeded" { + if preview := protocol.OutputPreview(r.Output); preview != "" { + lines = append(lines, "Result: "+ansi.Strip(preview)) + } else { + lines = append(lines, "Finished · no result recorded") + } + } else { + if r.Status == "running" || r.Status == "queued" { + lines = append(lines, "In progress · final result pending") + } + for i := len(r.Steps) - 1; i >= 0; i-- { + if r.Steps[i].Status == "completed" { + if preview := protocol.OutputPreview(r.Steps[i].Output); preview != "" { + lines = append(lines, "Latest completed output ("+r.Steps[i].Stage+"): "+ansi.Strip(preview)) + } + break + } + } + } + completed := 0 + failed := 0 + for _, s := range r.Steps { + if s.Status == "completed" { + completed++ + } + if s.Status == "failed" { + failed++ + } + } + lines = append(lines, fmt.Sprintf("Progress: %d completed visits · %d failed · %d recorded", completed, failed, len(r.Steps))) + if r.CurrentStage != "" && (r.Status == "running" || r.Status == "queued") { + lines = append(lines, "Working on: "+r.CurrentStage) + } + return lines +} + +func completedWorkLines(r *protocol.RunDetail) []string { + var lines []string + for _, s := range r.Steps { + if s.Status != "completed" { + continue + } + line := fmt.Sprintf("%d. %s · %s", s.Sequence, s.Stage, s.Outcome) + if preview := protocol.OutputPreview(s.Output); preview != "" { + line += " — " + ansi.Strip(preview) + } else { + line += " — no output recorded" + } + lines = append(lines, line) + } + if len(lines) == 0 { + lines = append(lines, "No completed stage output yet.") + } + return append([]string{"Completed stage outputs (reported, not independently verified):"}, lines...) +} + +func resultOutputLines(r *protocol.RunDetail) []string { + if len(r.Output) > 0 { + return valueLines(r.Output) + } + for i := len(r.Steps) - 1; i >= 0; i-- { + if r.Steps[i].Status == "completed" && len(r.Steps[i].Output) > 0 { + return append([]string{"Partial output · " + r.Steps[i].Stage + " (not a final result)", ""}, valueLines(r.Steps[i].Output)...) + } + } + if strings.TrimSpace(r.Diagnostic) != "" { + return []string{r.Diagnostic} + } + return []string{"No output recorded yet."} +} diff --git a/internal/tui/results_test.go b/internal/tui/results_test.go new file mode 100644 index 00000000..0cfd2ff5 --- /dev/null +++ b/internal/tui/results_test.go @@ -0,0 +1,76 @@ +package tui + +import ( + "context" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/protocol" + "github.com/charmbracelet/x/ansi" +) + +func TestHomeAndPipelineShowRecordedResultsForExactVersion(t *testing.T) { + m := New(context.Background(), "") + m.width, m.height = 140, 48 + m.loaded = true + m.pipelines = []protocol.PipelineSummary{{ID: 1, Name: "Maintenance", Version: 2, Stages: 2}} + m.runs = []protocol.RunSummary{ + {ID: 12, PipelineID: 1, Version: 2, Name: "Maintenance", Status: "succeeded", ResultPreview: "Added regression coverage", CompletedSteps: 2}, + {ID: 11, PipelineID: 1, Version: 1, Status: "succeeded", ResultPreview: "Created the initial report", CompletedSteps: 1}, + } + view := ansi.Strip(m.View()) + for _, want := range []string{"Result: Added regression coverage", "2 completed visits", "Last run #12"} { + if !strings.Contains(view, want) { + t.Fatal("home missing result", want, view) + } + } + m.detail = &protocol.PipelineDetail{PipelineSummary: m.pipelines[0], LatestVersion: 2, Graph: protocol.Graph{Stages: []protocol.Stage{{Key: "worker", Edges: map[string]string{"success": protocol.SuccessEnd}}}}} + view = ansi.Strip(m.pipelineView(40)) + if !strings.Contains(view, "Added regression coverage") || strings.Index(view, "Results") > strings.Index(view, "Graph") { + t.Fatal("pipeline does not lead with results", view) + } + m.detail.Version = 1 + view = ansi.Strip(m.pipelineView(40)) + if !strings.Contains(view, "Created the initial report") || strings.Contains(view, "Added regression coverage") { + t.Fatal("pipeline mixed version results", view) + } + m.detail.Version = 3 + if view = ansi.Strip(m.pipelineView(40)); !strings.Contains(view, "has not run yet") || strings.Contains(view, "Added regression coverage") { + t.Fatal("unexecuted version invented results", view) + } +} + +func TestRunLeadsWithResultAndPreservesPartialAccomplishments(t *testing.T) { + m := New(context.Background(), "") + m.width, m.height = 120, 48 + m.runDetailFocus = true + r := &protocol.RunDetail{RunSummary: protocol.RunSummary{ID: 1, Name: "Maintenance", Status: "failed", Diagnostic: "Verification timed out"}, Steps: []protocol.RunStep{ + {Sequence: 1, Stage: "implement", Status: "completed", Outcome: "success", Output: map[string]any{"summary": "Updated the layout", "artifacts": []string{"screen.go"}}}, + {Sequence: 2, Stage: "verify", Status: "failed", Diagnostic: "Verification timed out"}, + }} + m.runDetail = r + view := ansi.Strip(m.runsView(44)) + for _, want := range []string{"Stopped: Verification timed out", "Latest completed output (implement): Updated the layout", "Partial output", "screen.go"} { + if !strings.Contains(view, want) { + t.Fatal("partial result hidden", want, view) + } + } + if strings.Index(view, "Updated the layout") > strings.Index(view, "Run graph") { + t.Fatal("graph precedes result") + } + lines := ansi.Strip(strings.Join(m.runSectionLines(100), "\n")) + if !strings.Contains(lines, "1. implement · success — Updated the layout") || strings.Index(lines, "[o]") > strings.Index(lines, "[i]") { + t.Fatal("recorded accomplishments hidden behind inputs", lines) + } + r.Status = "succeeded" + r.Diagnostic = "" + r.Output = map[string]any{"summary": "Layout verified", "report": "Checks passed", "artifacts": []string{"screen.go"}} + if lines = ansi.Strip(strings.Join(m.runSectionLines(100), "\n")); !strings.Contains(lines, "Result: Layout verified") { + t.Fatal("final result not prominent", lines) + } + r.Output = nil + r.Steps = nil + if lines = ansi.Strip(strings.Join(m.runSectionLines(100), "\n")); !strings.Contains(lines, "no result recorded") || strings.Contains(lines, "Layout verified") { + t.Fatal("empty success fabricated work", lines) + } +} diff --git a/internal/tui/run_detail.go b/internal/tui/run_detail.go index df58c2d1..1ea8ee3c 100644 --- a/internal/tui/run_detail.go +++ b/internal/tui/run_detail.go @@ -9,6 +9,7 @@ import ( "time" "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" "github.com/charmbracelet/x/ansi" @@ -299,6 +300,11 @@ func evidenceLines(step protocol.RunStep) []string { return append(lines, "No captured tool evidence yet.") } var audit struct { + Limits *struct { + Timeout string `json:"timeout"` + Retries int `json:"retries"` + Budget stage.BudgetSnapshot `json:"budget"` + } `json:"stage_limits"` Data []json.RawMessage `json:"data"` FormatRepairs []struct { Method string `json:"method"` @@ -347,6 +353,24 @@ func evidenceLines(step protocol.RunStep) []string { for _, repair := range audit.FormatRepairs { lines = append(lines, fmt.Sprintf("JSON format repair: %s · accepted: %t", repair.Method, repair.Accepted), repair.Diagnostic) } + if limits := audit.Limits; limits != nil { + b := limits.Budget + lines = append(lines, fmt.Sprintf("Timeout per attempt: %s · Retries: %d · Attempts: %d", limits.Timeout, limits.Retries, b.Attempts)) + if b.LimitUSD > 0 { + used := "unknown" + if b.USDKnown { + used = fmt.Sprintf("$%.6f", b.UsedUSD) + } + lines = append(lines, fmt.Sprintf("USD budget: %s / $%.6f", used, b.LimitUSD)) + } + if b.LimitTokens > 0 { + used := "unknown" + if b.TokensKnown { + used = fmt.Sprint(b.UsedTokens) + } + lines = append(lines, fmt.Sprintf("Token budget: %s / %d", used, b.LimitTokens)) + } + } count := 0 for i := len(audit.Data) - 1; i >= 0; i-- { var message struct { @@ -390,10 +414,7 @@ func (m Model) runSectionLines(width int) []string { r := m.runDetail titles := []string{"Summary", "Input", "Output", "Tool evidence", "Metadata"} shortcuts := []string{"s", "i", "o", "t", "m"} - summary := []string{"Status: " + m.progress(r.Status), "Outcome: " + r.Outcome, fmt.Sprintf("Stages: %d · Current: %s", len(r.Steps), r.CurrentStage), "Started: " + dateLabel(r.CreatedAt), "Finished: " + dateLabel(r.FinishedAt), "Duration: " + runDuration(r.CreatedAt, r.FinishedAt)} - if r.Diagnostic != "" { - summary = append(summary, lipgloss.NewStyle().Foreground(themeError).Render(r.Diagnostic)) - } + summary := append([]string{"Status: " + m.progress(r.Status)}, runResultLines(r)...) if m.runError != "" { summary = append(summary, "Request error: "+m.runError) } @@ -401,17 +422,22 @@ func (m Model) runSectionLines(width int) []string { if len(r.WritePaths) > 0 { editScope = strings.Join(r.WritePaths, ", ") } - content := [][]string{summary, valueLines(r.Input), valueLines(r.Output), {"Select a stage to inspect its tool calls."}, {"Workspace: " + r.Workspace, "Source hash: " + r.ContentHash, fmt.Sprintf("Shell/edit enabled: %t", r.AllowTools), "Stage tools: defined by the pinned pipeline version", "Edit scope: " + editScope}} + output := resultOutputLines(r) + output = append(output, "", strings.Join(completedWorkLines(r), "\n")) + content := [][]string{summary, valueLines(r.Input), output, {"Select a stage to inspect its tool calls."}, {"Started: " + dateLabel(r.CreatedAt), "Finished: " + dateLabel(r.FinishedAt), "Duration: " + runDuration(r.CreatedAt, r.FinishedAt), "Workspace: " + r.Workspace, "Source hash: " + r.ContentHash, fmt.Sprintf("Shell/edit enabled: %t", r.AllowTools), "Stage tools: defined by the pinned pipeline version", "Edit scope: " + editScope}} if r.RetryOf > 0 { content[4] = append(content[4], fmt.Sprintf("Retry of run #%d", r.RetryOf)) } if m.runStepIndex > 0 && m.runStepIndex <= len(r.Steps) { step := r.Steps[m.runStepIndex-1] - summary = []string{"Status: " + m.progress(m.stepProgressStatus(step)), "Route: " + outcomeLabel(step.Outcome, step.Outcome) + " → " + step.Target, "Started: " + dateLabel(step.StartedAt), "Finished: " + dateLabel(step.FinishedAt), "Duration: " + runDuration(step.StartedAt, step.FinishedAt)} + summary = []string{"Status: " + m.progress(m.stepProgressStatus(step)), "Route: " + outcomeLabel(step.Outcome, step.Outcome) + " → " + step.Target} + if preview := protocol.OutputPreview(step.Output); preview != "" { + summary = append(summary, "Reported output: "+ansi.Strip(preview)) + } if step.Diagnostic != "" { summary = append(summary, lipgloss.NewStyle().Foreground(themeError).Render(step.Diagnostic)) } - content = [][]string{summary, valueLines(step.Input), valueLines(step.Output), evidenceLines(step), {"Session: " + step.SessionID, fmt.Sprintf("Agent active: %t", step.AgentActive), fmt.Sprintf("Attempt sequence: %d", step.Sequence)}} + content = [][]string{summary, valueLines(step.Input), valueLines(step.Output), evidenceLines(step), {"Started: " + dateLabel(step.StartedAt), "Finished: " + dateLabel(step.FinishedAt), "Duration: " + runDuration(step.StartedAt, step.FinishedAt), "Session: " + step.SessionID, fmt.Sprintf("Agent active: %t", step.AgentActive), fmt.Sprintf("Attempt sequence: %d", step.Sequence)}} for _, receipt := range step.Notifications { line := "Notification: " + receipt.Destination + " · " + receipt.Status if receipt.Filename != "" { @@ -430,7 +456,8 @@ func (m Model) runSectionLines(width int) []string { } } var lines []string - for i, title := range titles { + for _, i := range []int{0, 2, 1, 3, 4} { + title := titles[i] marker := "▾" if m.runSectionClosed(i) { marker = "▸" @@ -529,11 +556,12 @@ func (m Model) runsDetailsView(height int) string { rows = append(rows, "Search: "+m.runSearch.View()) } runs := m.visibleRuns() - start, end := selectionWindow(len(runs), m.runIndex, max(1, height-4-len(rows))) + start, end := selectionWindow(len(runs), m.runIndex, max(1, (height-4-len(rows))/2)) for i := start; i < end; i++ { r := runs[i] text := fmt.Sprintf("#%d %s · v%d · %s · %s", r.ID, r.Name, r.Version, m.progress(r.Status), dateLabel(r.FinishedAt)) rows = append(rows, menuOption(text, i == m.runIndex)) + rows = append(rows, " "+ansi.Truncate(resultLabel(r), max(1, m.width-6), "…")) } if len(runs) == 0 { rows = append(rows, "No runs yet.") diff --git a/internal/tui/run_graph.go b/internal/tui/run_graph.go index 711a48fc..26df5bb3 100644 --- a/internal/tui/run_graph.go +++ b/internal/tui/run_graph.go @@ -97,7 +97,7 @@ func (m Model) runsView(height int) string { } graph := m.runGraphPanel(graphHeight, top, pan) details := m.runsDetailsView(detailHeight) - return fitBlock(strings.Split(graph+"\n"+details, "\n"), m.width, height) + return fitBlock(strings.Split(details+"\n"+graph, "\n"), m.width, height) } func (m Model) runPaneHeights(height int) (int, int) { diff --git a/internal/tui/run_graph_test.go b/internal/tui/run_graph_test.go index 3c36c3ef..47fa5184 100644 --- a/internal/tui/run_graph_test.go +++ b/internal/tui/run_graph_test.go @@ -103,8 +103,8 @@ func TestRunPathUsesSharedRendererAndRecordedRoutesWithRepeatedVisits(t *testing t.Fatal("run graph diverged from common renderer") } view := ansi.Strip(m.runsView(36)) - if strings.Index(view, "Run graph") < 0 || strings.Index(view, "Run graph") > strings.Index(view, "Run overview") || !strings.Contains(view, "3. plan (running)") { - t.Fatal("run graph missing above details or current visit not visible", view) + if strings.Index(view, "Run graph") < 0 || strings.Index(view, "Run graph") < strings.Index(view, "Run overview") || !strings.Contains(view, "3. plan (running)") { + t.Fatal("run graph missing below results or current visit not visible", view) } refreshed := *m.runDetail refreshed.Steps = append([]protocol.RunStep(nil), m.runDetail.Steps...) diff --git a/internal/tui/run_pane_size_test.go b/internal/tui/run_pane_size_test.go index 9a4200e4..b11bc435 100644 --- a/internal/tui/run_pane_size_test.go +++ b/internal/tui/run_pane_size_test.go @@ -18,8 +18,13 @@ func TestRunPanesExpandWithTabFocus(t *testing.T) { updated, _ := m.Update(receivedMsg(protocol.Message{Type: protocol.Run, RunDetail: run})) m = updated.(Model) graph, details := m.runPaneHeights(44) - if !m.runGraphFocus || graph <= details || graph+details != 44 { - t.Fatal("new run did not prioritize graph", graph, details) + if !m.runDetailFocus || m.runGraphFocus || details <= graph || graph+details != 44 { + t.Fatal("new run did not prioritize results", graph, details) + } + m = runKey(m, "tab") + graph, details = m.runPaneHeights(44) + if !m.runGraphFocus || graph <= details { + t.Fatal("graph focus did not expand") } graphPage := m.runPageSize() m = runKey(m, "tab") diff --git a/internal/tui/runs_test.go b/internal/tui/runs_test.go index e3a44dd5..8fff4eb9 100644 --- a/internal/tui/runs_test.go +++ b/internal/tui/runs_test.go @@ -195,8 +195,8 @@ func TestRunSectionsOrderTitleAndNumberSelection(t *testing.T) { m := runInspectionModel() m = runKey(m, "1") text := ansi.Strip(strings.Join(m.runSectionLines(70), "\n")) - if strings.Index(text, "Input") > strings.Index(text, "Output") { - t.Fatal("input must precede output") + if strings.Index(text, "Output") > strings.Index(text, "Input") { + t.Fatal("output must precede input") } if m.runSectionClosed(2) || !m.runSectionClosed(1) { t.Fatal("output should remain visible; input initially collapsed")