Add stage execution limits and result-first workflow views
This commit is contained in:
@@ -149,7 +149,8 @@ Turn input.prompt into a focused plan and acceptance checks.
|
||||
```configs
|
||||
provider=opencode-go
|
||||
model=deepseek-v4.1-flash
|
||||
retries=0
|
||||
retries=2
|
||||
timeout=1h
|
||||
```
|
||||
|
||||
```edges
|
||||
|
||||
+43
@@ -309,6 +309,49 @@ Quit with `:quit`, not `q`; Ctrl+C remains an interrupt.
|
||||
JSON results go to stdout, errors to stderr (JSON error envelopes when requested),
|
||||
and failures exit nonzero. Export/template `--source` deliberately prints raw
|
||||
Markdown. `--timeout` bounds each server request and the optional wait duration.
|
||||
It does not change the stage execution timeout.
|
||||
|
||||
### Stage limits
|
||||
|
||||
Model-stage `configs` support these optional settings:
|
||||
|
||||
```text
|
||||
retries=2
|
||||
timeout=1h
|
||||
budget_usd=1.50
|
||||
budget_tokens=50000
|
||||
```
|
||||
|
||||
Retries default to **2** (three total attempts); timeout defaults to **one hour
|
||||
per attempt**. Omitted budgets are unlimited; configured budgets must be positive.
|
||||
Each budget covers all attempts, format-only repairs, and repeated visits to that
|
||||
stage in one run. A new run starts new accounting. Tokens include input, output,
|
||||
reasoning, and cache reads/writes. USD amounts use OpenCode's reported cost.
|
||||
Missing or invalid required accounting stops the stage rather than counting zero.
|
||||
Usage is polled during execution and read again afterward; an in-flight request
|
||||
may exceed a limit before usage arrives. These are not provider-side hard caps.
|
||||
|
||||
Automatic retries cover retryable transport failures and per-attempt timeouts.
|
||||
They do not replay admitted tool work, invalid results, or budget failures.
|
||||
Cleanup must be confirmed before another attempt. Existing explicit `retries=0`
|
||||
settings remain unchanged. The old 30-minute run deadline no longer truncates
|
||||
stage timeouts; the 30-step and five-visits-per-stage limits still apply.
|
||||
Native ntfy stages also accept `timeout` (default `1h`), but never automatically
|
||||
retry potentially delivered notifications and do not accept model budgets.
|
||||
|
||||
Pinned definitions contain the configured limits. Run transcripts retain limits,
|
||||
attempt counts and reported budget usage, also shown in the TUI's Evidence pane.
|
||||
|
||||
### Result-first views
|
||||
|
||||
Home shows retained result previews and completed visits alongside each run and
|
||||
pipeline. Pipeline pages show the latest result for the **selected version**,
|
||||
not a result from a different published definition. Run opens focused on results,
|
||||
with output before input and the graph below. Failed or ongoing runs preserve
|
||||
partial output and completed stage reports without labeling them final success.
|
||||
These previews quote recorded output; they do not independently verify model claims.
|
||||
Tab still expands the focused graph, stage list, or result pane; `n`/`p` still
|
||||
navigate recorded visits, including repeated stages.
|
||||
|
||||
## Verification scope
|
||||
|
||||
|
||||
@@ -20,7 +20,8 @@ webfetch
|
||||
```configs
|
||||
provider=opencode-go
|
||||
model=deepseek-v4.1-flash
|
||||
retries=0
|
||||
retries=2
|
||||
timeout=1h
|
||||
```
|
||||
|
||||
```edges
|
||||
@@ -48,7 +49,8 @@ Return success for a supported report or blocked when the evidence is insufficie
|
||||
```configs
|
||||
provider=opencode-go
|
||||
model=deepseek-v4.1-flash
|
||||
retries=0
|
||||
retries=2
|
||||
timeout=1h
|
||||
```
|
||||
|
||||
```edges
|
||||
|
||||
+2
-1
@@ -24,7 +24,8 @@ grep
|
||||
```configs
|
||||
provider=opencode-go
|
||||
model=deepseek-v4.1-flash
|
||||
retries=0
|
||||
retries=2
|
||||
timeout=1h
|
||||
```
|
||||
|
||||
```edges
|
||||
|
||||
+2
-1
@@ -17,7 +17,8 @@ Return blocked if no text to summarize was supplied. Otherwise return success.
|
||||
```configs
|
||||
provider=opencode-go
|
||||
model=deepseek-v4.1-flash
|
||||
retries=0
|
||||
retries=2
|
||||
timeout=1h
|
||||
```
|
||||
|
||||
```edges
|
||||
|
||||
+2
-1
@@ -25,7 +25,8 @@ shell
|
||||
```configs
|
||||
provider=opencode-go
|
||||
model=deepseek-v4.1-flash
|
||||
retries=0
|
||||
retries=2
|
||||
timeout=1h
|
||||
```
|
||||
|
||||
```edges
|
||||
|
||||
@@ -18,7 +18,8 @@ grep
|
||||
```configs
|
||||
provider=opencode-go
|
||||
model=deepseek-v4.1-flash
|
||||
retries=0
|
||||
retries=2
|
||||
timeout=1h
|
||||
```
|
||||
|
||||
```edges
|
||||
@@ -49,7 +50,8 @@ edit
|
||||
```configs
|
||||
provider=opencode-go
|
||||
model=deepseek-v4.1-flash
|
||||
retries=0
|
||||
retries=2
|
||||
timeout=1h
|
||||
```
|
||||
|
||||
```edges
|
||||
@@ -81,7 +83,8 @@ shell
|
||||
```configs
|
||||
provider=opencode-go
|
||||
model=deepseek-v4.1-flash
|
||||
retries=0
|
||||
retries=2
|
||||
timeout=1h
|
||||
```
|
||||
|
||||
```edges
|
||||
|
||||
@@ -24,7 +24,8 @@ Perform the task supplied in input.prompt. Return evidence, not invented success
|
||||
type=model
|
||||
provider=opencode-go
|
||||
model=deepseek-v4.1-flash
|
||||
retries=0
|
||||
retries=2
|
||||
timeout=1h
|
||||
```
|
||||
|
||||
```tools
|
||||
@@ -48,8 +49,22 @@ Use the exact provider/model IDs validated during server setup. New drafts/templ
|
||||
inherit that selection; do not assume the host's login matches the container's.
|
||||
Other OpenCode provider/model IDs are accepted syntactically; actual availability,
|
||||
authentication and inference require `pipelines provider-check` on the execution
|
||||
computer/container. Retries are
|
||||
integers 0–100. Explicit empty `tools` denies all; allowed tools are `read`, `glob`,
|
||||
computer/container. Optional `retries` defaults to 2 (three total attempts),
|
||||
and accepts integers 0–100; explicit 0 disables execution retries. Optional
|
||||
`timeout` defaults to `1h` per attempt and accepts positive durations such as
|
||||
`30m` or `90s`. Automatic retries only cover retryable transport failures and
|
||||
timeouts; admitted tool work is never replayed automatically. Optional
|
||||
`budget_usd=1.50` and `budget_tokens=50000` independently cap reported usage.
|
||||
They accumulate across attempts, format repair sessions, and repeated visits
|
||||
to this stage in the same run. Omit either budget to leave that resource unlimited.
|
||||
Both budgets must be positive; tokens include input, output, reasoning and cache
|
||||
reads/writes. Accounting failures stop budget-limited stages. In-flight requests
|
||||
may exceed limits before usage is reported; these are not provider-side hard caps.
|
||||
The run has no additional wall-clock deadline; step/visit caps and cancellation
|
||||
still apply. A native ntfy stage accepts `timeout` but never automatically retries
|
||||
delivery and has no model budget.
|
||||
|
||||
Explicit empty `tools` denies all; allowed tools are `read`, `glob`,
|
||||
`grep`, `webfetch`, `websearch`, `edit`, `shell`, one per line. No wildcards,
|
||||
delegation, or browser tools. Tools are per-stage and automatically enabled by
|
||||
starting a run. `edit` supports any file type and creating files/directories in the
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
package opencode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// BudgetMessages uses the same owned endpoint and credentials, without sharing
|
||||
// mutable request diagnostics or generation state with the main connection.
|
||||
func (c *Connection) BudgetMessages(ctx context.Context, id string) ([]byte, error) {
|
||||
if !sessionID.MatchString(id) {
|
||||
return nil, fmt.Errorf("invalid session ID")
|
||||
}
|
||||
reader := &Connection{Backend: c.Backend, Executable: c.Executable, Directory: c.Directory, URL: c.URL, password: c.password}
|
||||
return reader.Messages(ctx, id)
|
||||
}
|
||||
@@ -33,7 +33,7 @@ func Template(names []string, linear bool) string {
|
||||
if linear && i+1 < len(names) {
|
||||
next = names[i+1]
|
||||
}
|
||||
fmt.Fprintf(&b, "# stage %s\n\n```output\nDescribe the output map.\n```\n\n```prompt\nDescribe the task.\n```\n\n```tools\n```\n\n```edges\nsuccess -> %s\nerror -> %s\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n", name, next, protocol.FailureEnd)
|
||||
fmt.Fprintf(&b, "# stage %s\n\n```output\nDescribe the output map.\n```\n\n```prompt\nDescribe the task.\n```\n\n```tools\n```\n\n```edges\nsuccess -> %s\nerror -> %s\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=2\ntimeout=1h\n```\n\n", name, next, protocol.FailureEnd)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
@@ -32,11 +32,11 @@ func TestValidation(t *testing.T) {
|
||||
"duplicate stage": valid + valid,
|
||||
"unknown target": strings.Replace(valid, "success -> reviewer", "success -> missing", 1),
|
||||
"duplicate outcome": strings.Replace(valid, "success -> reviewer", "success -> reviewer\nsuccess -> #", 1),
|
||||
"unknown config": strings.Replace(valid, "retries=0", "oops=2", 1),
|
||||
"duplicate config": strings.Replace(valid, "retries=0", "retries=0\nretries=1", 1),
|
||||
"unknown config": strings.Replace(valid, "retries=2", "oops=2", 1),
|
||||
"duplicate config": strings.Replace(valid, "retries=2", "retries=2\nretries=1", 1),
|
||||
"invalid provider": strings.Replace(valid, "provider=opencode-go", "provider=not valid", 1),
|
||||
"invalid model": strings.Replace(valid, "model=deepseek-v4.1-flash", "model=not valid", 1),
|
||||
"bad retries": strings.Replace(valid, "retries=0", "retries=-1", 1),
|
||||
"bad retries": strings.Replace(valid, "retries=2", "retries=-1", 1),
|
||||
"multiple targets": strings.Replace(valid, "success -> reviewer", "success -> reviewer, implementer", 1),
|
||||
"unreachable": Template([]string{"a", "b"}, false),
|
||||
"no termination": strings.NewReplacer("-> #success", "-> a", "-> #failure", "-> a").Replace(Template([]string{"a"}, true)),
|
||||
|
||||
@@ -97,11 +97,22 @@ func (s *Store) CreateRun(ctx context.Context, d *protocol.PipelineDetail, key,
|
||||
}
|
||||
|
||||
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
|
||||
strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',r.finished_at),''),r.diagnostic
|
||||
strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',r.finished_at),''),r.diagnostic,
|
||||
CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END,
|
||||
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed')
|
||||
FROM pipeline_runs r JOIN pipelines p ON p.id=r.pipeline_id JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version`
|
||||
|
||||
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
|
||||
return row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic)
|
||||
var output string
|
||||
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps); err != nil {
|
||||
return err
|
||||
}
|
||||
var values map[string]any
|
||||
if err := decodeValues(output, &values); err != nil {
|
||||
return err
|
||||
}
|
||||
r.ResultPreview = protocol.OutputPreview(values)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) ListRuns(ctx context.Context) ([]protocol.RunSummary, error) {
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// OutputPreview quotes retained output, never infers accomplishments from a
|
||||
// successful process or invents a schema for technology-neutral workflows.
|
||||
func OutputPreview(output map[string]any) string {
|
||||
keys := []string{"summary", "result", "report", "message", "accomplished", "artifacts", "evidence", "reason", "error"}
|
||||
seen := map[string]bool{}
|
||||
remaining := make([]string, 0, len(output))
|
||||
for key := range output {
|
||||
remaining = append(remaining, key)
|
||||
}
|
||||
sort.Strings(remaining)
|
||||
keys = append(keys, remaining...)
|
||||
for _, key := range keys {
|
||||
if seen[key] || key == "input" {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
value, exists := output[key]
|
||||
if !exists || value == nil {
|
||||
continue
|
||||
}
|
||||
text, ok := value.(string)
|
||||
if !ok {
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
text = string(encoded)
|
||||
}
|
||||
text = strings.Map(func(r rune) rune {
|
||||
if unicode.IsControl(r) {
|
||||
return ' '
|
||||
}
|
||||
return r
|
||||
}, text)
|
||||
text = strings.Join(strings.Fields(text), " ")
|
||||
if text == "" || text == "[]" || text == "{}" {
|
||||
continue
|
||||
}
|
||||
if key != "summary" && key != "result" && key != "report" && key != "message" {
|
||||
text = key + ": " + text
|
||||
}
|
||||
runes := []rune(text)
|
||||
if len(runes) > 280 {
|
||||
text = string(runes[:279]) + "…"
|
||||
}
|
||||
return text
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOutputPreviewUsesRetainedValuesWithoutAssumingSchema(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
output map[string]any
|
||||
want string
|
||||
}{
|
||||
{map[string]any{"summary": "Fixed layout\nand added regression checks", "evidence": "details"}, "Fixed layout and added regression checks"},
|
||||
{map[string]any{"artifacts": []string{"report.md"}}, `artifacts: ["report.md"]`},
|
||||
{map[string]any{"z": true, "a": json.Number("9007199254740993")}, "a: 9007199254740993"},
|
||||
{map[string]any{"input": map[string]any{"prompt": "private task"}}, ""},
|
||||
{nil, ""},
|
||||
} {
|
||||
if got := OutputPreview(test.output); got != test.want {
|
||||
t.Fatal(got, test.want)
|
||||
}
|
||||
}
|
||||
preview := OutputPreview(map[string]any{"summary": strings.Repeat("宽", 500)})
|
||||
if len([]rune(preview)) != 280 || !strings.HasSuffix(preview, "…") {
|
||||
t.Fatal("preview is not bounded by characters")
|
||||
}
|
||||
}
|
||||
+14
-12
@@ -23,18 +23,20 @@ type SessionEvidence struct {
|
||||
}
|
||||
|
||||
type RunSummary struct {
|
||||
ID int64 `json:"id"`
|
||||
PipelineID int64 `json:"pipeline_id"`
|
||||
Name string `json:"name"`
|
||||
Version int `json:"version"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
Status string `json:"status"`
|
||||
Executing bool `json:"executing"`
|
||||
Outcome string `json:"outcome"`
|
||||
CurrentStage string `json:"current_stage"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
FinishedAt string `json:"finished_at"`
|
||||
Diagnostic string `json:"diagnostic"`
|
||||
ID int64 `json:"id"`
|
||||
PipelineID int64 `json:"pipeline_id"`
|
||||
Name string `json:"name"`
|
||||
Version int `json:"version"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
Status string `json:"status"`
|
||||
Executing bool `json:"executing"`
|
||||
Outcome string `json:"outcome"`
|
||||
CurrentStage string `json:"current_stage"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
FinishedAt string `json:"finished_at"`
|
||||
Diagnostic string `json:"diagnostic"`
|
||||
ResultPreview string `json:"result_preview,omitempty"`
|
||||
CompletedSteps int `json:"completed_steps"`
|
||||
}
|
||||
|
||||
type RunStep struct {
|
||||
|
||||
@@ -2,6 +2,7 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"maps"
|
||||
@@ -157,7 +158,7 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
|
||||
return nil, err
|
||||
}
|
||||
if created {
|
||||
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Minute)
|
||||
ctx, cancel := context.WithCancel(r.ctx)
|
||||
r.active[id] = cancel
|
||||
r.wg.Add(1)
|
||||
go func() {
|
||||
@@ -242,6 +243,7 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail,
|
||||
input := stage.Values(request.Input)
|
||||
key := d.Graph.Stages[0].Key
|
||||
visits := map[string]int{}
|
||||
budgets := map[string]*stage.Budget{}
|
||||
finish := func(status, outcome, diagnostic string) {
|
||||
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
@@ -305,10 +307,27 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail,
|
||||
}
|
||||
var result stage.Result
|
||||
var err error
|
||||
stageCtx := ctx
|
||||
var modelConfig stage.ModelConfig
|
||||
if stage.EffectiveType(def.Type) == stage.ModelType {
|
||||
spec, specErr := (stage.ModelStage{}).Spec(def)
|
||||
if specErr != nil {
|
||||
finish("failed", "error", specErr.Error())
|
||||
return
|
||||
}
|
||||
modelConfig = spec.Config
|
||||
if budgets[key] == nil {
|
||||
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
|
||||
}
|
||||
stageCtx = stage.WithBudget(ctx, budgets[key])
|
||||
}
|
||||
if r.execute != nil {
|
||||
result, err = r.execute(ctx, def, input)
|
||||
timeout, _ := stage.StageTimeout(def)
|
||||
executeCtx, cancel := context.WithTimeout(stageCtx, timeout)
|
||||
result, err = r.execute(executeCtx, def, input)
|
||||
cancel()
|
||||
} else if useAgent {
|
||||
result, err = (stage.ModelStage{Client: client}).Run(ctx, def, input)
|
||||
result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input)
|
||||
} else if stage.EffectiveType(def.Type) == stage.ModelType {
|
||||
plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations}
|
||||
plain.OnGenerationSession = func(session, backend, reference string, active bool) error {
|
||||
@@ -316,12 +335,15 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail,
|
||||
defer cancel()
|
||||
return r.store.RunSession(c, id, sequence, session, active, backend, reference)
|
||||
}
|
||||
result, err = (stage.ModelStage{Client: plain}).Run(ctx, def, input)
|
||||
result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input)
|
||||
client.Transcript = plain.FormatAudit
|
||||
} else {
|
||||
result, err = stage.DefaultRegistry().Run(ctx, def, input)
|
||||
}
|
||||
step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript)}
|
||||
if budget := budgets[key]; budget != nil {
|
||||
step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries)
|
||||
}
|
||||
if err != nil {
|
||||
step.Status = "failed"
|
||||
step.Diagnostic = err.Error()
|
||||
@@ -343,7 +365,7 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail,
|
||||
}
|
||||
}
|
||||
input = stage.Values(step.Output)
|
||||
if useAgent {
|
||||
if useAgent || errors.Is(err, stage.ErrStageCleanup) {
|
||||
sessions, checkErr := r.store.ActiveAgentSessions(context.Background())
|
||||
if checkErr != nil || len(sessions) > 0 {
|
||||
finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work")
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/cli"
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func TestStageBudgetAcrossVisitsThroughCLIAndWebsocket(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
dir := t.TempDir()
|
||||
executable := filepath.Join(dir, "opencode")
|
||||
script := fmt.Sprintf(`#!/bin/sh
|
||||
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
|
||||
case "$3" in
|
||||
/api/session)
|
||||
n=0; if [ -f %q/count ]; then n=$(cat %q/count); fi
|
||||
n=$((n+1)); echo "$n" > %q/count
|
||||
printf '{"data":{"id":"ses_budget_%%s"}}' "$n" ;;
|
||||
*/generate)
|
||||
id=${3#/api/session/}; id=${id%%/generate}
|
||||
touch %q/"$id"
|
||||
echo '{"data":{"text":"{\"outcome\":\"success\",\"output\":{\"summary\":\"done\"}}"}}' ;;
|
||||
*/interrupt) echo '{}' ;;
|
||||
/api/session/ses_budget_*)
|
||||
id=${3#/api/session/}; cost=0; tokens=0
|
||||
if [ -f %q/"$id" ]; then cost=0.4; tokens=3; fi
|
||||
printf '{"data":{"id":"%%s","cost":%%s,"tokens":{"input":%%s,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}' "$id" "$cost" "$tokens" ;;
|
||||
*) exit 99 ;;
|
||||
esac
|
||||
`, dir, dir, dir, dir, dir)
|
||||
if err := os.WriteFile(executable, []byte(script), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PIPELINES_OPENCODE", executable)
|
||||
store, err := pipeline.Open(filepath.Join(dir, "runs.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
source := pipeline.Template([]string{"worker"}, true)
|
||||
source = strings.Replace(source, "success -> #success", "success -> worker", 1)
|
||||
source = strings.Replace(source, "timeout=1h", "timeout=1h\nbudget_usd=1\nbudget_tokens=5", 1)
|
||||
if _, err := store.Submit(ctx, "budget", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := Handler(ctx, store)
|
||||
defer func() { cancel(); WaitRuns(handler) }()
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
||||
var out bytes.Buffer
|
||||
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
|
||||
if err := app.Run(ctx, []string{"--url", url, "run", "budget", "task", "--yes", "--json", "--wait", "--timeout", "10s", "--workspace", dir}); err == nil || !strings.Contains(err.Error(), "run ended failed") {
|
||||
t.Fatal(err, out.String())
|
||||
}
|
||||
var response protocol.Message
|
||||
if err := json.Unmarshal(out.Bytes(), &response); err != nil {
|
||||
t.Fatal(err, out.String())
|
||||
}
|
||||
detail := response.RunDetail
|
||||
if detail == nil || detail.ResultPreview != "done" || detail.CompletedSteps != 1 {
|
||||
t.Fatal("partial results missing from run summary", out.String())
|
||||
}
|
||||
if detail == nil || detail.Status != "failed" || len(detail.Steps) != 2 || detail.Steps[0].SessionID == detail.Steps[1].SessionID || detail.Steps[1].AgentActive || !strings.Contains(detail.Steps[1].Diagnostic, "stage budget exceeded") {
|
||||
t.Fatal("budget run/audit incorrect", out.String())
|
||||
}
|
||||
var audit struct {
|
||||
Limits struct {
|
||||
Timeout string
|
||||
Retries int
|
||||
Budget stage.BudgetSnapshot
|
||||
} `json:"stage_limits"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(detail.Steps[1].Transcript), &audit); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if audit.Limits.Timeout != time.Hour.String() || audit.Limits.Retries != 2 || audit.Limits.Budget.Attempts != 2 || audit.Limits.Budget.UsedTokens != 6 {
|
||||
t.Fatal("limits not persisted across visits", audit)
|
||||
}
|
||||
count, err := os.ReadFile(filepath.Join(dir, "count"))
|
||||
if err != nil || strings.TrimSpace(string(count)) != "2" {
|
||||
t.Fatal("budget failed to block further work", string(count), err)
|
||||
}
|
||||
stored, err := store.GetRun(ctx, detail.ID)
|
||||
if err != nil || stored.Steps[1].Transcript != detail.Steps[1].Transcript {
|
||||
t.Fatal("evidence was not retained", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerStageTimeoutAndNoShortOuterDeadline(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "runs.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
source := pipeline.Template([]string{"worker"}, true)
|
||||
if _, err := store.Submit(ctx, "default", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := store.List(ctx)
|
||||
r, err := newRunner(ctx, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok || time.Until(deadline) < time.Hour-time.Second {
|
||||
t.Error("outer run deadline truncated default stage timeout")
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
if _, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "default", Input: map[string]any{"prompt": "task"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
source = strings.Replace(source, "timeout=1h", "timeout=20ms", 1)
|
||||
if _, err := store.Submit(ctx, "short", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ = store.List(ctx)
|
||||
var id int64
|
||||
for _, item := range items {
|
||||
if item.Name == "short" {
|
||||
id = item.ID
|
||||
}
|
||||
}
|
||||
r.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
|
||||
<-ctx.Done()
|
||||
return stage.Result{}, ctx.Err()
|
||||
}
|
||||
run, err := r.start(protocol.Message{PipelineID: id, Version: 1, RequestKey: "short", Input: map[string]any{"prompt": "task"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
detail, err := store.GetRun(ctx, run.ID)
|
||||
if err != nil || detail.Status != "failed" || !strings.Contains(detail.Steps[0].Diagnostic, "deadline exceeded") {
|
||||
t.Fatal("per-stage timeout not persisted", detail, err)
|
||||
}
|
||||
}
|
||||
+19
-8
@@ -14,12 +14,13 @@ import (
|
||||
// before any prompt is admitted, so restart recovery can interrupt the agent.
|
||||
type AgentClient struct {
|
||||
OpenCodeClient
|
||||
WritePaths []string
|
||||
Tools []string // nil is the legacy manual approval bundle; empty denies all
|
||||
OnSession func(string, bool) error
|
||||
SessionID string
|
||||
Transcript string
|
||||
Policy ToolPolicy
|
||||
WritePaths []string
|
||||
Tools []string // nil is the legacy manual approval bundle; empty denies all
|
||||
OnSession func(string, bool) error
|
||||
SessionID string
|
||||
Transcript string
|
||||
Policy ToolPolicy
|
||||
WorkStarted bool
|
||||
}
|
||||
|
||||
func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error {
|
||||
@@ -46,6 +47,8 @@ func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error {
|
||||
}
|
||||
|
||||
func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) {
|
||||
c.WorkStarted = false
|
||||
c.SessionID = ""
|
||||
if c.Operations != nil {
|
||||
c.Operations.Provider, c.Operations.Model = config.Provider, config.Model
|
||||
}
|
||||
@@ -83,18 +86,21 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
|
||||
return "", fmt.Errorf("retained OpenCode session %s; ownership could not be recorded: %w", c.SessionID, err)
|
||||
}
|
||||
}
|
||||
var watch *budgetWatch
|
||||
defer func() {
|
||||
watch.stopWatch(ctx, &err)
|
||||
defer watch.finish(ctx, &err)
|
||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
|
||||
defer cancel()
|
||||
if err != nil {
|
||||
if stopErr := c.Interrupt(cleanup, c.SessionID); stopErr != nil {
|
||||
err = fmt.Errorf("agent interrupt failed; retained session %s needs inspection: %w", c.SessionID, errors.Join(err, stopErr))
|
||||
err = fmt.Errorf("agent interrupt failed; retained session %s needs inspection: %w", c.SessionID, errors.Join(err, stopErr, ErrStageCleanup))
|
||||
return
|
||||
}
|
||||
}
|
||||
if c.OnSession != nil {
|
||||
if persistErr := c.OnSession(c.SessionID, false); persistErr != nil {
|
||||
err = errors.Join(err, persistErr)
|
||||
err = errors.Join(err, persistErr, ErrStageCleanup)
|
||||
text = ""
|
||||
return
|
||||
}
|
||||
@@ -121,6 +127,11 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
|
||||
if c.Policy.Shell != nil {
|
||||
prompt += "\nPermitted shell commands: " + strings.Join(c.Policy.Shell, "; ")
|
||||
}
|
||||
ctx, watch, err = c.watchBudget(ctx, config.Budget, c.SessionID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
c.WorkStarted = true
|
||||
if _, err = c.call(ctx, "post", "/api/session/"+c.SessionID+"/prompt", generationBody("text", prompt, config, c.Backend)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const DefaultRetries = 2
|
||||
const DefaultTimeout = time.Hour
|
||||
|
||||
var ErrBudgetExceeded = errors.New("stage budget exceeded")
|
||||
var ErrBudgetAccounting = errors.New("stage budget accounting unavailable")
|
||||
var ErrStageCleanup = errors.New("stage cleanup not confirmed")
|
||||
|
||||
func StageTimeout(def Definition) (time.Duration, error) {
|
||||
if def.Configs["timeout"] == "" {
|
||||
return DefaultTimeout, nil
|
||||
}
|
||||
duration, err := time.ParseDuration(def.Configs["timeout"])
|
||||
if err != nil || duration <= 0 {
|
||||
return 0, fmt.Errorf("timeout must be a positive duration, such as 1h or 30m")
|
||||
}
|
||||
return duration, nil
|
||||
}
|
||||
|
||||
func parseBudget(def Definition) (float64, int64, error) {
|
||||
var usd float64
|
||||
var tokens int64
|
||||
var err error
|
||||
if value := def.Configs["budget_usd"]; value != "" {
|
||||
usd, err = strconv.ParseFloat(value, 64)
|
||||
if err != nil || usd <= 0 || math.IsNaN(usd) || math.IsInf(usd, 0) {
|
||||
return 0, 0, fmt.Errorf("budget_usd must be a positive finite USD amount")
|
||||
}
|
||||
}
|
||||
if value := def.Configs["budget_tokens"]; value != "" {
|
||||
tokens, err = strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || tokens <= 0 {
|
||||
return 0, 0, fmt.Errorf("budget_tokens must be a positive integer")
|
||||
}
|
||||
}
|
||||
return usd, tokens, nil
|
||||
}
|
||||
|
||||
type BudgetSnapshot struct {
|
||||
LimitUSD float64 `json:"limit_usd,omitempty"`
|
||||
LimitTokens int64 `json:"limit_tokens,omitempty"`
|
||||
UsedUSD float64 `json:"used_usd"`
|
||||
UsedTokens int64 `json:"used_tokens"`
|
||||
USDKnown bool `json:"usd_known"`
|
||||
TokensKnown bool `json:"tokens_known"`
|
||||
Attempts int `json:"attempts"`
|
||||
}
|
||||
|
||||
type reportedUsage struct {
|
||||
USD *float64 `json:"cost"`
|
||||
Tokens *struct {
|
||||
Input, Output, Reasoning *int64
|
||||
Cache *struct{ Read, Write *int64 }
|
||||
} `json:"tokens"`
|
||||
}
|
||||
|
||||
type Budget struct {
|
||||
mu sync.Mutex
|
||||
state BudgetSnapshot
|
||||
records map[string]BudgetSnapshot
|
||||
pollInterval time.Duration
|
||||
}
|
||||
|
||||
func NewBudget(usd float64, tokens int64) *Budget {
|
||||
return &Budget{state: BudgetSnapshot{LimitUSD: usd, LimitTokens: tokens}, records: map[string]BudgetSnapshot{}, pollInterval: time.Second}
|
||||
}
|
||||
func (b *Budget) Enabled() bool { return b != nil && (b.state.LimitUSD > 0 || b.state.LimitTokens > 0) }
|
||||
func (b *Budget) Snapshot() BudgetSnapshot { b.mu.Lock(); defer b.mu.Unlock(); return b.state }
|
||||
func (b *Budget) Attempt() { b.mu.Lock(); b.state.Attempts++; b.mu.Unlock() }
|
||||
func (b *Budget) checkLocked() error {
|
||||
if b.state.LimitUSD > 0 && b.state.UsedUSD >= b.state.LimitUSD {
|
||||
return fmt.Errorf("%w: USD %.6f / %.6f", ErrBudgetExceeded, b.state.UsedUSD, b.state.LimitUSD)
|
||||
}
|
||||
if b.state.LimitTokens > 0 && b.state.UsedTokens >= b.state.LimitTokens {
|
||||
return fmt.Errorf("%w: tokens %d / %d", ErrBudgetExceeded, b.state.UsedTokens, b.state.LimitTokens)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (b *Budget) Check() error { b.mu.Lock(); defer b.mu.Unlock(); return b.checkLocked() }
|
||||
|
||||
func (b *Budget) observe(key string, usage reportedUsage) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
old := b.records[key]
|
||||
defer func() { b.records[key] = old }()
|
||||
if usage.USD != nil {
|
||||
if *usage.USD < 0 || math.IsNaN(*usage.USD) || math.IsInf(*usage.USD, 0) {
|
||||
return fmt.Errorf("%w: invalid cost", ErrBudgetAccounting)
|
||||
}
|
||||
value := max(old.UsedUSD, *usage.USD)
|
||||
total := b.state.UsedUSD + (value - old.UsedUSD)
|
||||
if math.IsNaN(total) || math.IsInf(total, 0) {
|
||||
return fmt.Errorf("%w: cost overflow", ErrBudgetAccounting)
|
||||
}
|
||||
b.state.UsedUSD = total
|
||||
old.UsedUSD, old.USDKnown, b.state.USDKnown = value, true, true
|
||||
} else if b.state.LimitUSD > 0 {
|
||||
return fmt.Errorf("%w: cost was not reported", ErrBudgetAccounting)
|
||||
}
|
||||
t := usage.Tokens
|
||||
if t != nil && t.Input != nil && t.Output != nil && t.Reasoning != nil && t.Cache != nil && t.Cache.Read != nil && t.Cache.Write != nil {
|
||||
var total int64
|
||||
for _, value := range []int64{*t.Input, *t.Output, *t.Reasoning, *t.Cache.Read, *t.Cache.Write} {
|
||||
if value < 0 || value > math.MaxInt64-total {
|
||||
return fmt.Errorf("%w: invalid token count", ErrBudgetAccounting)
|
||||
}
|
||||
total += value
|
||||
}
|
||||
total = max(total, old.UsedTokens)
|
||||
if total-old.UsedTokens > math.MaxInt64-b.state.UsedTokens {
|
||||
return fmt.Errorf("%w: token count overflow", ErrBudgetAccounting)
|
||||
}
|
||||
b.state.UsedTokens += total - old.UsedTokens
|
||||
old.UsedTokens, old.TokensKnown, b.state.TokensKnown = total, true, true
|
||||
} else if b.state.LimitTokens > 0 {
|
||||
return fmt.Errorf("%w: complete token usage was not reported", ErrBudgetAccounting)
|
||||
}
|
||||
return b.checkLocked()
|
||||
}
|
||||
|
||||
type budgetContextKey struct{}
|
||||
|
||||
func WithBudget(ctx context.Context, budget *Budget) context.Context {
|
||||
return context.WithValue(ctx, budgetContextKey{}, budget)
|
||||
}
|
||||
func budgetFromContext(ctx context.Context) *Budget {
|
||||
b, _ := ctx.Value(budgetContextKey{}).(*Budget)
|
||||
return b
|
||||
}
|
||||
|
||||
type budgetWatch struct {
|
||||
read func(context.Context, bool) error
|
||||
cancel context.CancelCauseFunc
|
||||
stop chan struct{}
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func (c OpenCodeClient) BudgetCapable() bool { return true }
|
||||
|
||||
func (c OpenCodeClient) watchBudget(ctx context.Context, budget *Budget, id string) (context.Context, *budgetWatch, error) {
|
||||
if !budget.Enabled() {
|
||||
return ctx, nil, nil
|
||||
}
|
||||
if err := budget.Check(); err != nil {
|
||||
return ctx, nil, err
|
||||
}
|
||||
// Poll through an independent reader. Native calls have per-request mutable
|
||||
// diagnostic fields, which must not race the long-running prompt request.
|
||||
reader := c
|
||||
reader.Operations = nil
|
||||
w := &budgetWatch{stop: make(chan struct{}), done: make(chan struct{})}
|
||||
w.read = func(parent context.Context, final bool) error {
|
||||
readCtx, cancel := context.WithTimeout(parent, 2*time.Second)
|
||||
defer cancel()
|
||||
var data []byte
|
||||
var err error
|
||||
if reader.connection != nil && reader.Backend == "v1" {
|
||||
data, err = reader.connection.BudgetMessages(readCtx, id)
|
||||
} else {
|
||||
data, err = reader.call(readCtx, "get", "/api/session/"+id, nil)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: usage read failed: %v", ErrBudgetAccounting, err)
|
||||
}
|
||||
return budget.readUsage(id, data, reader.Backend == "v1", final)
|
||||
}
|
||||
if err := w.read(ctx, false); err != nil {
|
||||
return ctx, nil, err
|
||||
}
|
||||
workCtx, cancel := context.WithCancelCause(ctx)
|
||||
w.cancel = cancel
|
||||
go func() {
|
||||
defer close(w.done)
|
||||
ticker := time.NewTicker(budget.pollInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-w.stop:
|
||||
return
|
||||
case <-workCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := w.read(workCtx, false); err != nil {
|
||||
if workCtx.Err() == nil {
|
||||
cancel(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return workCtx, w, nil
|
||||
}
|
||||
|
||||
func (w *budgetWatch) stopWatch(ctx context.Context, err *error) {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
close(w.stop)
|
||||
<-w.done
|
||||
if cause := context.Cause(ctx); errors.Is(cause, ErrBudgetExceeded) || errors.Is(cause, ErrBudgetAccounting) {
|
||||
*err = errors.Join(*err, cause)
|
||||
}
|
||||
}
|
||||
func (w *budgetWatch) finish(ctx context.Context, err *error) {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
defer w.cancel(nil)
|
||||
if usageErr := w.read(context.WithoutCancel(ctx), true); usageErr != nil {
|
||||
*err = errors.Join(*err, usageErr)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Budget) readUsage(id string, data []byte, native bool, final ...bool) error {
|
||||
if !native {
|
||||
var response struct {
|
||||
Data struct {
|
||||
ID string `json:"id"`
|
||||
reportedUsage
|
||||
} `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(data, &response) != nil || response.Data.ID != id {
|
||||
return fmt.Errorf("%w: invalid or mismatched usage response", ErrBudgetAccounting)
|
||||
}
|
||||
return b.observe(id, response.Data.reportedUsage)
|
||||
}
|
||||
var messages []struct {
|
||||
Info struct {
|
||||
ID, Role, SessionID string
|
||||
Time struct{ Completed int64 }
|
||||
reportedUsage
|
||||
}
|
||||
}
|
||||
if len(bytes.TrimSpace(data)) == 0 || bytes.TrimSpace(data)[0] != '[' || json.Unmarshal(data, &messages) != nil {
|
||||
return fmt.Errorf("%w: invalid native usage response", ErrBudgetAccounting)
|
||||
}
|
||||
var accountingErr error
|
||||
completed := 0
|
||||
for _, m := range messages {
|
||||
if m.Info.Role != "assistant" {
|
||||
continue
|
||||
}
|
||||
if m.Info.Time.Completed != 0 {
|
||||
completed++
|
||||
}
|
||||
if m.Info.Time.Completed == 0 && (m.Info.USD == nil || m.Info.Tokens == nil) {
|
||||
continue
|
||||
}
|
||||
if m.Info.ID == "" || (m.Info.SessionID != "" && m.Info.SessionID != id) {
|
||||
return fmt.Errorf("%w: invalid native message identity", ErrBudgetAccounting)
|
||||
}
|
||||
if err := b.observe(id+"/"+m.Info.ID, m.Info.reportedUsage); err != nil {
|
||||
if errors.Is(err, ErrBudgetAccounting) && accountingErr == nil {
|
||||
accountingErr = err
|
||||
}
|
||||
}
|
||||
}
|
||||
if accountingErr != nil {
|
||||
return accountingErr
|
||||
}
|
||||
if len(final) > 0 && final[0] && completed == 0 {
|
||||
return fmt.Errorf("%w: no completed assistant usage was reported", ErrBudgetAccounting)
|
||||
}
|
||||
return b.Check()
|
||||
}
|
||||
|
||||
func BudgetTranscript(original string, budget *Budget, timeout time.Duration, retries int) string {
|
||||
var data map[string]json.RawMessage
|
||||
if json.Unmarshal([]byte(original), &data) != nil || data == nil {
|
||||
data = map[string]json.RawMessage{}
|
||||
if original != "" {
|
||||
data["original_transcript"], _ = json.Marshal(original)
|
||||
}
|
||||
}
|
||||
data["stage_limits"], _ = json.Marshal(map[string]any{"timeout": timeout.String(), "retries": retries, "budget": budget.Snapshot()})
|
||||
encoded, _ := json.Marshal(data)
|
||||
return string(encoded)
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"math"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func usageResponse(id string, usd float64, tokens int64) []byte {
|
||||
return []byte(fmt.Sprintf(`{"data":{"id":%q,"cost":%g,"tokens":{"input":%d,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}`, id, usd, tokens))
|
||||
}
|
||||
|
||||
func TestStageLimitDefaultsAndValidation(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
delete(def.Configs, "retries")
|
||||
spec, err := (ModelStage{}).Spec(def)
|
||||
if err != nil || spec.Config.Retries != 2 || spec.Config.Timeout != time.Hour || spec.Config.BudgetUSD != 0 || spec.Config.BudgetTokens != 0 {
|
||||
t.Fatal("wrong defaults", spec, err)
|
||||
}
|
||||
def.Configs["timeout"], def.Configs["budget_usd"], def.Configs["budget_tokens"] = "30m", "1.25", "50000"
|
||||
spec, err = (ModelStage{}).Spec(def)
|
||||
if err != nil || spec.Config.Timeout != 30*time.Minute || spec.Config.BudgetUSD != 1.25 || spec.Config.BudgetTokens != 50000 {
|
||||
t.Fatal("limits not parsed", spec, err)
|
||||
}
|
||||
for _, value := range []struct{ key, value string }{{"timeout", "0s"}, {"timeout", "-1h"}, {"timeout", "1"}, {"budget_usd", "NaN"}, {"budget_usd", "Inf"}, {"budget_usd", "0"}, {"budget_usd", "-1"}, {"budget_tokens", "0"}, {"budget_tokens", "-1"}, {"budget_tokens", "1.5"}, {"budget_tokens", "9223372036854775808"}} {
|
||||
copy := def
|
||||
copy.Configs = maps.Clone(def.Configs)
|
||||
copy.Configs[value.key] = value.value
|
||||
if err := (ModelStage{}).Validate(copy); err == nil {
|
||||
t.Fatal("accepted invalid limit", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultRetriesAndTimeoutResetEachAttempt(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
delete(def.Configs, "retries")
|
||||
calls := 0
|
||||
client := generatorFunc(func(ctx context.Context, _ ModelConfig, _ string) (string, error) {
|
||||
calls++
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok || time.Until(deadline) < time.Hour-time.Second {
|
||||
t.Fatal("default attempt timeout is not one hour")
|
||||
}
|
||||
return "", &openCodeError{status: 503}
|
||||
})
|
||||
if _, err := (ModelStage{Client: client}).Run(context.Background(), def, nil); err == nil || calls != 3 {
|
||||
t.Fatal("default retry count wrong", calls, err)
|
||||
}
|
||||
def.Configs["timeout"] = "20ms"
|
||||
calls = 0
|
||||
client = generatorFunc(func(ctx context.Context, _ ModelConfig, _ string) (string, error) {
|
||||
calls++
|
||||
deadline, _ := ctx.Deadline()
|
||||
if time.Until(deadline) <= 0 {
|
||||
t.Fatal("attempt inherited expired timeout")
|
||||
}
|
||||
if calls == 1 {
|
||||
<-ctx.Done()
|
||||
return "", ctx.Err()
|
||||
}
|
||||
return `{"outcome":"success","output":{}}`, nil
|
||||
})
|
||||
if _, err := (ModelStage{Client: client}).Run(context.Background(), def, nil); err != nil || calls != 2 {
|
||||
t.Fatal("timeout did not reset for retry", calls, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgetsAccumulateAcrossRetriesAndBlockFurtherGeneration(t *testing.T) {
|
||||
for _, test := range []struct{ name, key, value string }{{"USD", "budget_usd", "0.5"}, {"tokens", "budget_tokens", "5"}} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Configs["retries"] = "5"
|
||||
def.Configs[test.key] = test.value
|
||||
var sessions, generations atomic.Int64
|
||||
client := &OpenCodeClient{command: func(ctx context.Context, _ string, args ...string) ([]byte, error) {
|
||||
path := args[2]
|
||||
if path == "/api/session" {
|
||||
return []byte(fmt.Sprintf(`{"data":{"id":"ses_attempt_%d"}}`, sessions.Add(1))), nil
|
||||
}
|
||||
if args[1] == "get" {
|
||||
cost, tokens := 0.0, int64(0)
|
||||
if generations.Load() >= sessions.Load() {
|
||||
cost, tokens = .4, 3
|
||||
}
|
||||
return usageResponse(strings.TrimPrefix(path, "/api/session/"), cost, tokens), nil
|
||||
}
|
||||
if strings.HasSuffix(path, "/generate") {
|
||||
generations.Add(1)
|
||||
return nil, &openCodeError{status: 503}
|
||||
}
|
||||
return []byte(`{}`), nil
|
||||
}}
|
||||
_, err := (ModelStage{Client: client}).Run(context.Background(), def, nil)
|
||||
if !errors.Is(err, ErrBudgetExceeded) || generations.Load() != 2 {
|
||||
t.Fatal("retries bypassed shared budget", generations.Load(), err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgetAccountingFailsClosedBeforeGeneration(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Configs["budget_usd"], def.Configs["budget_tokens"] = "1", "10"
|
||||
var generated atomic.Int64
|
||||
client := &OpenCodeClient{command: func(_ context.Context, _ string, args ...string) ([]byte, error) {
|
||||
if args[2] == "/api/session" {
|
||||
return []byte(`{"data":{"id":"ses_missing"}}`), nil
|
||||
}
|
||||
if args[1] == "get" {
|
||||
return []byte(`{"data":{"id":"ses_missing"}}`), nil
|
||||
}
|
||||
if strings.HasSuffix(args[2], "/generate") {
|
||||
generated.Add(1)
|
||||
}
|
||||
return []byte(`{}`), nil
|
||||
}}
|
||||
if _, err := (ModelStage{Client: client}).Run(context.Background(), def, nil); !errors.Is(err, ErrBudgetAccounting) || generated.Load() != 0 {
|
||||
t.Fatal("unknown usage treated as zero", err, generated.Load())
|
||||
}
|
||||
calls := 0
|
||||
unsupported := generatorFunc(func(context.Context, ModelConfig, string) (string, error) { calls++; return "", nil })
|
||||
if _, err := (ModelStage{Client: unsupported}).Run(context.Background(), def, nil); !errors.Is(err, ErrBudgetAccounting) || calls != 0 {
|
||||
t.Fatal("unsupported generator bypassed limits", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveAgentBudgetInterruptsAndDoesNotReplay(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Configs["budget_tokens"] = "10"
|
||||
def.Configs["retries"] = "2"
|
||||
def.Blocks["tools"] = "read"
|
||||
budget := NewBudget(0, 10)
|
||||
budget.pollInterval = time.Millisecond
|
||||
var prompts, interrupts atomic.Int64
|
||||
var inactive atomic.Bool
|
||||
client := &AgentClient{OpenCodeClient: OpenCodeClient{command: func(ctx context.Context, _ string, args ...string) ([]byte, error) {
|
||||
path := args[2]
|
||||
if path == "/api/session" {
|
||||
return []byte(`{"data":{"id":"ses_agent_budget"}}`), nil
|
||||
}
|
||||
if args[1] == "get" {
|
||||
tokens := int64(0)
|
||||
if prompts.Load() > 0 {
|
||||
tokens = 12
|
||||
}
|
||||
return usageResponse("ses_agent_budget", .1, tokens), nil
|
||||
}
|
||||
if strings.HasSuffix(path, "/prompt") {
|
||||
prompts.Add(1)
|
||||
return []byte(`{}`), nil
|
||||
}
|
||||
if strings.HasSuffix(path, "/wait") {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if strings.HasSuffix(path, "/interrupt") {
|
||||
interrupts.Add(1)
|
||||
return []byte(`{}`), nil
|
||||
}
|
||||
return nil, fmt.Errorf("unexpected operation %s", path)
|
||||
}}, OnSession: func(_ string, active bool) error {
|
||||
if !active {
|
||||
inactive.Store(true)
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
_, err := (ModelStage{Client: client}).Run(WithBudget(ctx, budget), def, nil)
|
||||
if !errors.Is(err, ErrBudgetExceeded) || prompts.Load() != 1 || interrupts.Load() != 1 || !inactive.Load() || budget.Snapshot().UsedTokens != 12 {
|
||||
t.Fatal("live limit did not stop and retain agent safely", err, prompts.Load(), interrupts.Load(), inactive.Load(), budget.Snapshot())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgetLedgerDeduplicatesPollsAndIncludesCacheAndReasoning(t *testing.T) {
|
||||
b := NewBudget(10, 100)
|
||||
data := []byte(`{"data":{"id":"ses_ledger","cost":0.2,"tokens":{"input":3,"output":4,"reasoning":5,"cache":{"read":6,"write":7}}}}`)
|
||||
for i := 0; i < 5; i++ {
|
||||
if err := b.readUsage("ses_ledger", data, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if s := b.Snapshot(); s.UsedTokens != 25 || s.UsedUSD != .2 {
|
||||
t.Fatal("polls double counted or cache/reasoning omitted", s)
|
||||
}
|
||||
if err := b.readUsage("ses_other", data, false); !errors.Is(err, ErrBudgetAccounting) {
|
||||
t.Fatal("accepted another session's accounting", err)
|
||||
}
|
||||
for _, invalid := range []string{`{"data":{"id":"ses_ledger","cost":-1}}`, `{"data":{"id":"ses_ledger","cost":0.1,"tokens":{"input":-1,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}`} {
|
||||
if err := b.readUsage("ses_ledger", []byte(invalid), false); !errors.Is(err, ErrBudgetAccounting) {
|
||||
t.Fatal("invalid usage accepted", err)
|
||||
}
|
||||
}
|
||||
var original map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(BudgetTranscript(`{"large":9007199254740993}`, b, time.Hour, 2)), &original); err != nil || string(original["large"]) != "9007199254740993" {
|
||||
t.Fatal("limit audit changed original evidence", err, string(original["large"]))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeBudgetCountsMessagesAcrossCompaction(t *testing.T) {
|
||||
b := NewBudget(1, 100)
|
||||
message := func(id string, cost float64, tokens int64) string {
|
||||
return fmt.Sprintf(`{"info":{"id":%q,"sessionID":"ses_native","role":"assistant","time":{"completed":1},"cost":%g,"tokens":{"input":%d,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}`, id, cost, tokens)
|
||||
}
|
||||
first := []byte("[" + message("msg_1", .2, 10) + "," + message("msg_2", .3, 20) + "]")
|
||||
if err := b.readUsage("ses_native", first, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.readUsage("ses_native", []byte("["+message("msg_2", .3, 20)+","+message("msg_3", .4, 30)+"]"), true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := b.Snapshot()
|
||||
if s.UsedTokens != 60 || math.Abs(s.UsedUSD-.9) > 1e-9 {
|
||||
t.Fatal("removed messages reset accumulated usage", s)
|
||||
}
|
||||
if err := b.readUsage("ses_native", []byte(`[{"info":{"id":"msg_missing","role":"assistant","time":{"completed":1}}}]`), true); !errors.Is(err, ErrBudgetAccounting) {
|
||||
t.Fatal("missing completed-message accounting accepted", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeFinalUsageCannotBeMissing(t *testing.T) {
|
||||
b := NewBudget(1, 100)
|
||||
for _, data := range []string{`[]`, `[{"info":{"id":"msg_inflight","role":"assistant","time":{},"cost":0,"tokens":{"input":0,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}]`} {
|
||||
if err := b.readUsage("ses_native", []byte(data), true, true); !errors.Is(err, ErrBudgetAccounting) {
|
||||
t.Fatal("missing final accounting accepted", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentRetriesBeforeAdmissionButNeverReplaysWork(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
delete(def.Configs, "retries")
|
||||
def.Blocks["tools"] = "read"
|
||||
sessions, prompts := 0, 0
|
||||
client := &AgentClient{OpenCodeClient: OpenCodeClient{command: func(_ context.Context, _ string, args ...string) ([]byte, error) {
|
||||
if args[2] == "/api/session" {
|
||||
sessions++
|
||||
if sessions == 1 {
|
||||
return nil, &openCodeError{status: 503}
|
||||
}
|
||||
return []byte(`{"data":{"id":"ses_admitted"}}`), nil
|
||||
}
|
||||
if strings.HasSuffix(args[2], "/prompt") {
|
||||
prompts++
|
||||
return nil, &openCodeError{status: 503}
|
||||
}
|
||||
return []byte(`{}`), nil
|
||||
}}}
|
||||
_, err := (ModelStage{Client: client}).Run(context.Background(), def, nil)
|
||||
if err == nil || sessions != 2 || prompts != 1 {
|
||||
t.Fatal("admitted work replayed or safe retry skipped", sessions, prompts, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupFailurePreventsTextGenerationRetry(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
delete(def.Configs, "retries")
|
||||
generated := 0
|
||||
client := &OpenCodeClient{command: func(_ context.Context, _ string, args ...string) ([]byte, error) {
|
||||
if args[2] == "/api/session" {
|
||||
return []byte(`{"data":{"id":"ses_cleanup"}}`), nil
|
||||
}
|
||||
if strings.HasSuffix(args[2], "/generate") {
|
||||
generated++
|
||||
return nil, &openCodeError{status: 503}
|
||||
}
|
||||
return nil, errors.New("interrupt unavailable")
|
||||
}}
|
||||
if _, err := (ModelStage{Client: client}).Run(context.Background(), def, nil); !errors.Is(err, ErrStageCleanup) || generated != 1 {
|
||||
t.Fatal("retried before cleanup confirmed", generated, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatRepairUsesTheSameBudget(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Configs["budget_tokens"] = "5"
|
||||
budget := NewBudget(0, 5)
|
||||
sessions, generated := 0, 0
|
||||
client := &OpenCodeClient{command: func(_ context.Context, _ string, args ...string) ([]byte, error) {
|
||||
path := args[2]
|
||||
if path == "/api/session" {
|
||||
sessions++
|
||||
return []byte(fmt.Sprintf(`{"data":{"id":"ses_repair_%d"}}`, sessions)), nil
|
||||
}
|
||||
if args[1] == "get" {
|
||||
tokens := int64(0)
|
||||
if generated >= sessions {
|
||||
tokens = 3
|
||||
}
|
||||
return usageResponse(strings.TrimPrefix(path, "/api/session/"), .1, tokens), nil
|
||||
}
|
||||
if strings.HasSuffix(path, "/generate") {
|
||||
generated++
|
||||
text := `{"outcome":"success" "output":{}}`
|
||||
if generated > 1 {
|
||||
text = `{"outcome":"success","output":{}}`
|
||||
}
|
||||
data, _ := json.Marshal(map[string]any{"data": map[string]string{"text": text}})
|
||||
return data, nil
|
||||
}
|
||||
return []byte(`{}`), nil
|
||||
}}
|
||||
_, err := (ModelStage{Client: client}).Run(WithBudget(context.Background(), budget), def, nil)
|
||||
if !errors.Is(err, ErrBudgetExceeded) || sessions != 2 || budget.Snapshot().UsedTokens != 6 {
|
||||
t.Fatal("repair bypassed execution budget", sessions, budget.Snapshot(), err)
|
||||
}
|
||||
}
|
||||
+64
-21
@@ -2,6 +2,7 @@ package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
@@ -26,6 +27,10 @@ type ModelConfig struct {
|
||||
Provider string
|
||||
Model string
|
||||
Retries int
|
||||
Timeout time.Duration
|
||||
BudgetUSD float64
|
||||
BudgetTokens int64
|
||||
Budget *Budget
|
||||
ResultSchema map[string]any
|
||||
}
|
||||
|
||||
@@ -73,14 +78,14 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
|
||||
if _, err := StagePermissions(def); err != nil {
|
||||
return ModelSpec{}, err
|
||||
}
|
||||
for _, key := range []string{"provider", "model", "retries"} {
|
||||
for _, key := range []string{"provider", "model"} {
|
||||
if def.Configs[key] == "" {
|
||||
return ModelSpec{}, fmt.Errorf("missing config %s", key)
|
||||
}
|
||||
}
|
||||
for key := range def.Configs {
|
||||
switch key {
|
||||
case "provider", "model", "retries":
|
||||
case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens":
|
||||
default:
|
||||
return ModelSpec{}, fmt.Errorf("unknown config %q", key)
|
||||
}
|
||||
@@ -88,13 +93,25 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
|
||||
if !ValidModel(def.Configs["provider"], def.Configs["model"]) {
|
||||
return ModelSpec{}, fmt.Errorf("invalid provider/model identifier; use the exact IDs validated during setup")
|
||||
}
|
||||
retries, err := strconv.Atoi(def.Configs["retries"])
|
||||
retries := DefaultRetries
|
||||
var err error
|
||||
if value := def.Configs["retries"]; value != "" {
|
||||
retries, err = strconv.Atoi(value)
|
||||
}
|
||||
if err != nil || retries < 0 || retries > 100 {
|
||||
return ModelSpec{}, fmt.Errorf("retries must be an integer from 0 to 100")
|
||||
}
|
||||
timeout, err := StageTimeout(def)
|
||||
if err != nil {
|
||||
return ModelSpec{}, err
|
||||
}
|
||||
usd, tokens, err := parseBudget(def)
|
||||
if err != nil {
|
||||
return ModelSpec{}, err
|
||||
}
|
||||
return ModelSpec{
|
||||
Output: def.Blocks["output"], Prompt: def.Blocks["prompt"],
|
||||
Config: ModelConfig{Provider: def.Configs["provider"], Model: def.Configs["model"], Retries: retries},
|
||||
Config: ModelConfig{Provider: def.Configs["provider"], Model: def.Configs["model"], Retries: retries, Timeout: timeout, BudgetUSD: usd, BudgetTokens: tokens},
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -111,8 +128,10 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
spec.Config.Budget = budgetFromContext(ctx)
|
||||
if spec.Config.Budget == nil {
|
||||
spec.Config.Budget = NewBudget(spec.Config.BudgetUSD, spec.Config.BudgetTokens)
|
||||
}
|
||||
client := m.Client
|
||||
if client == nil {
|
||||
tools, _, _ := ToolNames(def)
|
||||
@@ -121,35 +140,59 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu
|
||||
}
|
||||
client = OpenCodeClient{}
|
||||
}
|
||||
// Replaying an agent can repeat file mutations. Graph feedback edges, not
|
||||
// transport/result retries, own additional tool-enabled attempts.
|
||||
if spec.Config.Budget.Enabled() {
|
||||
capable, ok := client.(interface{ BudgetCapable() bool })
|
||||
if !ok || !capable.BudgetCapable() {
|
||||
return Result{}, fmt.Errorf("%w: generator cannot enforce usage limits", ErrBudgetAccounting)
|
||||
}
|
||||
}
|
||||
// Replaying admitted tool work can repeat mutations. Only pre-admission
|
||||
// transport errors may automatically retry an audited agent.
|
||||
if agent, ok := client.(*AgentClient); ok {
|
||||
if tools, explicit, _ := ToolNames(def); explicit {
|
||||
agent.Tools = tools
|
||||
}
|
||||
agent.Policy, _ = StagePermissions(def)
|
||||
spec.Config.Retries = 0
|
||||
}
|
||||
for attempt := 0; attempt <= spec.Config.Retries; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
var text string
|
||||
text, err = client.Generate(ctx, spec.Config, prompt)
|
||||
if err == nil {
|
||||
var result Result
|
||||
result, err = decodeModelResult(stageResultText(text), def.Edges)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
// Formatting retries are isolated from execution retries. Never replay
|
||||
// stage instructions or tool actions to fix malformed final JSON.
|
||||
return repairModelResult(ctx, client, spec.Config, text, def.Edges, err)
|
||||
if err := spec.Config.Budget.Check(); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
spec.Config.Budget.Attempt()
|
||||
attemptCtx, cancel := context.WithTimeout(ctx, spec.Config.Timeout)
|
||||
text, generateErr := client.Generate(attemptCtx, spec.Config, prompt)
|
||||
if generateErr == nil && attemptCtx.Err() != nil {
|
||||
generateErr = attemptCtx.Err()
|
||||
}
|
||||
if generateErr == nil {
|
||||
result, resultErr := decodeModelResult(stageResultText(text), def.Edges)
|
||||
if resultErr != nil {
|
||||
result, resultErr = repairModelResult(attemptCtx, client, spec.Config, text, def.Edges, resultErr)
|
||||
}
|
||||
if attemptCtx.Err() != nil {
|
||||
resultErr = errors.Join(resultErr, attemptCtx.Err())
|
||||
}
|
||||
cancel()
|
||||
return result, resultErr
|
||||
}
|
||||
err = generateErr
|
||||
if attemptCtx.Err() != nil && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) {
|
||||
err = errors.Join(err, attemptCtx.Err())
|
||||
}
|
||||
cancel()
|
||||
if ctx.Err() != nil {
|
||||
return Result{}, ctx.Err()
|
||||
}
|
||||
if attempt == spec.Config.Retries || !retryableModelError(err) {
|
||||
if errors.Is(err, ErrBudgetExceeded) || errors.Is(err, ErrBudgetAccounting) || errors.Is(err, ErrStageCleanup) {
|
||||
break
|
||||
}
|
||||
if agent, ok := client.(*AgentClient); ok && agent.WorkStarted {
|
||||
break
|
||||
}
|
||||
if attempt == spec.Config.Retries || (!retryableModelError(err) && !errors.Is(err, context.DeadlineExceeded)) {
|
||||
break
|
||||
}
|
||||
delay := time.Duration(1<<min(attempt, 4)) * 100 * time.Millisecond
|
||||
|
||||
@@ -59,7 +59,7 @@ func TestModelSpecAndValidation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if spec.Output != "Output map." || spec.Prompt != "Do the task." || !reflect.DeepEqual(spec.Config, ModelConfig{Provider: "opencode-go", Model: "deepseek-v4.1-flash", Retries: 0}) {
|
||||
if spec.Output != "Output map." || spec.Prompt != "Do the task." || !reflect.DeepEqual(spec.Config, ModelConfig{Provider: "opencode-go", Model: "deepseek-v4.1-flash", Retries: 0, Timeout: time.Hour}) {
|
||||
t.Fatalf("invalid grouped model spec: %+v", spec)
|
||||
}
|
||||
for _, block := range []string{"output", "prompt", "configs"} {
|
||||
@@ -71,7 +71,7 @@ func TestModelSpecAndValidation(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, key := range []string{"provider", "model", "retries"} {
|
||||
for _, key := range []string{"provider", "model"} {
|
||||
t.Run("missing config "+key, func(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
delete(def.Configs, key)
|
||||
|
||||
@@ -42,10 +42,13 @@ func (NtfyStage) Validate(def Definition) error {
|
||||
}
|
||||
}
|
||||
for key := range def.Configs {
|
||||
if key != "destination" {
|
||||
if key != "destination" && key != "timeout" {
|
||||
return fmt.Errorf("unknown ntfy config %q", key)
|
||||
}
|
||||
}
|
||||
if _, err := StageTimeout(def); err != nil {
|
||||
return err
|
||||
}
|
||||
if !destinationName.MatchString(def.Configs["destination"]) {
|
||||
return fmt.Errorf("ntfy destination must be a lowercase name containing letters, digits, or underscores")
|
||||
}
|
||||
@@ -59,6 +62,9 @@ func (NtfyStage) Run(ctx context.Context, def Definition, input Values) (Result,
|
||||
if err := (NtfyStage{}).Validate(def); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
timeout, _ := StageTimeout(def)
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
title, _ := input["title"].(string)
|
||||
summary, _ := input["summary"].(string)
|
||||
markdown, _ := input["markdown"].(string)
|
||||
|
||||
@@ -203,19 +203,22 @@ func (c OpenCodeClient) Generate(ctx context.Context, config ModelConfig, prompt
|
||||
if err := json.Unmarshal(created, &session); err != nil || !sessionIDPattern.MatchString(session.Data.ID) {
|
||||
return "", fmt.Errorf("OpenCode returned an invalid session ID")
|
||||
}
|
||||
var watch *budgetWatch
|
||||
defer func() {
|
||||
watch.stopWatch(ctx, &err)
|
||||
defer watch.finish(ctx, &err)
|
||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
if err != nil {
|
||||
if stopErr := c.Interrupt(cleanup, session.Data.ID); stopErr != nil {
|
||||
err = fmt.Errorf("retained OpenCode session %s; interrupt failed: %w", session.Data.ID, errors.Join(err, stopErr))
|
||||
err = fmt.Errorf("retained OpenCode session %s; interrupt failed: %w", session.Data.ID, errors.Join(err, stopErr, ErrStageCleanup))
|
||||
return
|
||||
}
|
||||
}
|
||||
if c.OnGenerationSession != nil {
|
||||
if persistErr := c.OnGenerationSession(session.Data.ID, c.Backend, c.Reference, false); persistErr != nil {
|
||||
text = ""
|
||||
err = errors.Join(err, persistErr)
|
||||
err = errors.Join(err, persistErr, ErrStageCleanup)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
@@ -233,6 +236,10 @@ func (c OpenCodeClient) Generate(ctx context.Context, config ModelConfig, prompt
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
ctx, watch, err = c.watchBudget(ctx, config.Budget, session.Data.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
response, err := c.call(ctx, "post", "/api/session/"+session.Data.ID+"/generate", body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -121,7 +121,7 @@ func repairModelResult(ctx context.Context, client ModelGenerator, config ModelC
|
||||
return Result{}, err
|
||||
}
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("%w; isolated format repair failed: %v", parseError, err)
|
||||
return Result{}, fmt.Errorf("%w; isolated format repair failed: %w", parseError, err)
|
||||
}
|
||||
candidate = stageResultText(candidate)
|
||||
if len(candidate) > 32<<10 {
|
||||
|
||||
@@ -47,12 +47,15 @@ func (m Model) homeLines() []string {
|
||||
panelWidth = (m.width - 2) / 2
|
||||
}
|
||||
runs := []string{bar("Latest runs", fmt.Sprint(len(m.runs)), max(1, panelWidth-4), true),
|
||||
lipgloss.NewStyle().Foreground(themeMuted).Render("Pipeline · Version · Outcome · Duration")}
|
||||
lipgloss.NewStyle().Foreground(themeMuted).Render("Outcome · Result · Completed work")}
|
||||
if len(m.runs) == 0 {
|
||||
runs = append(runs, "No runs yet.")
|
||||
}
|
||||
for _, r := range m.runs[:min(8, len(m.runs))] {
|
||||
runs = append(runs, lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("#%d %s", r.ID, r.Name))+fmt.Sprintf(" · v%d · %s · %s", r.Version, m.progress(r.Status), runDuration(r.CreatedAt, r.FinishedAt)), lipgloss.NewStyle().Foreground(themeMuted).Render(" Started: "+dateLabel(r.CreatedAt)))
|
||||
runs = append(runs, lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("#%d %s", r.ID, r.Name))+fmt.Sprintf(" · v%d · %s · %s", r.Version, m.progress(r.Status), runDuration(r.CreatedAt, r.FinishedAt)), " "+resultLabel(r))
|
||||
if r.CompletedSteps > 0 {
|
||||
runs = append(runs, fmt.Sprintf(" %d completed visits", r.CompletedSteps))
|
||||
}
|
||||
if r.CurrentStage != "" && (r.Status == "running" || r.Status == "queued") {
|
||||
runs = append(runs, " Stage: "+r.CurrentStage)
|
||||
}
|
||||
@@ -70,7 +73,13 @@ func (m Model) homeLines() []string {
|
||||
for _, p := range m.pipelines {
|
||||
lines = append(lines,
|
||||
lipgloss.NewStyle().Bold(true).Render(p.Name)+lipgloss.NewStyle().Foreground(themeMuted).Render(fmt.Sprintf(" v%d · %d stages", p.Version, p.Stages)),
|
||||
lipgloss.NewStyle().Foreground(themeMuted).Render(" Created: "+dateLabel(p.CreatedAt)), lipgloss.NewStyle().Foreground(themeMuted).Render(" Updated: "+dateLabel(p.UpdatedAt)))
|
||||
lipgloss.NewStyle().Foreground(themeMuted).Render(" Updated: "+dateLabel(p.UpdatedAt)))
|
||||
if r := m.latestPipelineRun(p.ID, 0); r != nil {
|
||||
lines = append(lines, fmt.Sprintf(" Last run #%d · v%d · %s", r.ID, r.Version, m.progress(r.Status)), " "+resultLabel(*r))
|
||||
} else {
|
||||
lines = append(lines, " Not run yet · no recorded results")
|
||||
}
|
||||
lines = append(lines, lipgloss.NewStyle().Foreground(themeMuted).Render(" Created: "+dateLabel(p.CreatedAt)))
|
||||
if m.usage != nil {
|
||||
for _, u := range m.usage.Pipelines {
|
||||
if u.PipelineID == p.ID {
|
||||
|
||||
@@ -338,8 +338,8 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
accepted := m.runOpen && msg.RunDetail != nil && ((msg.Type == protocol.Run && msg.RunDetail.ID == m.requestedRunID) || (msg.Type == protocol.RunStarted && msg.ID == m.runStartRequest))
|
||||
if accepted {
|
||||
if m.runDetail == nil {
|
||||
m.runGraphFocus = true
|
||||
m.runDetailFocus = false
|
||||
m.runGraphFocus = false
|
||||
m.runDetailFocus = true
|
||||
m.runGraphTop, m.runGraphPan = 0, 0
|
||||
}
|
||||
if msg.Type == protocol.RunStarted {
|
||||
|
||||
+18
-2
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/charmbracelet/x/ansi"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
stageconfig "git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
type paneGeometry struct {
|
||||
@@ -17,7 +18,7 @@ type paneGeometry struct {
|
||||
}
|
||||
|
||||
func (m Model) pipelineGeometry(height int) paneGeometry {
|
||||
remaining := max(0, height-3)
|
||||
remaining := max(0, height-3-len(m.pipelineResultLines()))
|
||||
if remaining >= 15 {
|
||||
graph := remaining * 3 / 4
|
||||
if m.stageFocus {
|
||||
@@ -122,7 +123,9 @@ func (m Model) pipelineToolbar() []string {
|
||||
meta += " · historical, inactive"
|
||||
}
|
||||
}
|
||||
return []string{buttons, meta, ""}
|
||||
lines := []string{buttons}
|
||||
lines = append(lines, m.pipelineResultLines()...)
|
||||
return append(lines, meta, "")
|
||||
}
|
||||
|
||||
func (m Model) pipelineView(height int) string {
|
||||
@@ -284,6 +287,19 @@ func (m Model) stageFields() []stageField {
|
||||
for _, k := range keys {
|
||||
configs = append(configs, lipgloss.NewStyle().Foreground(themeMuted).Render(k+" ")+stage.Configs[k])
|
||||
}
|
||||
if stage.Configs["timeout"] == "" {
|
||||
configs = append(configs, fmt.Sprintf("timeout %s (default per attempt)", stageconfig.DefaultTimeout))
|
||||
}
|
||||
if stage.Type == "" || stage.Type == stageconfig.ModelType {
|
||||
if stage.Configs["retries"] == "" {
|
||||
configs = append(configs, fmt.Sprintf("retries %d (default)", stageconfig.DefaultRetries))
|
||||
}
|
||||
for _, key := range []string{"budget_usd", "budget_tokens"} {
|
||||
if stage.Configs[key] == "" {
|
||||
configs = append(configs, key+" unlimited")
|
||||
}
|
||||
}
|
||||
}
|
||||
var edges []graphEdge
|
||||
maxLabel := 0
|
||||
for _, edge := range graphEdges(m.detail.Graph) {
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package tui
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"github.com/charmbracelet/x/ansi"
|
||||
)
|
||||
|
||||
func resultLabel(r protocol.RunSummary) string {
|
||||
preview := ansi.Strip(r.ResultPreview)
|
||||
if r.Status == "succeeded" {
|
||||
if preview != "" {
|
||||
return "Result: " + preview
|
||||
}
|
||||
return "Finished · no result recorded"
|
||||
}
|
||||
if r.Diagnostic != "" {
|
||||
return "Stopped: " + ansi.Strip(r.Diagnostic)
|
||||
}
|
||||
if preview != "" {
|
||||
return "Latest output: " + preview
|
||||
}
|
||||
if r.Status == "running" || r.Status == "queued" {
|
||||
return "In progress · awaiting stage output"
|
||||
}
|
||||
return "No result recorded"
|
||||
}
|
||||
|
||||
func (m Model) latestPipelineRun(id int64, version int) *protocol.RunSummary {
|
||||
var latest *protocol.RunSummary
|
||||
for i := range m.runs {
|
||||
r := &m.runs[i]
|
||||
if r.PipelineID == id && (version == 0 || r.Version == version) && (latest == nil || r.ID > latest.ID) {
|
||||
latest = r
|
||||
}
|
||||
}
|
||||
return latest
|
||||
}
|
||||
|
||||
func (m Model) pipelineResultLines() []string {
|
||||
if m.detail == nil {
|
||||
return nil
|
||||
}
|
||||
r := m.latestPipelineRun(m.detail.ID, m.detail.Version)
|
||||
if r == nil {
|
||||
return []string{"Results · This version has not run yet. No accomplishments recorded."}
|
||||
}
|
||||
lines := []string{fmt.Sprintf("Results · run #%d · %s · %d completed visits · %s", r.ID, ansi.Strip(m.progress(r.Status)), r.CompletedSteps, runDuration(r.CreatedAt, r.FinishedAt)), resultLabel(*r)}
|
||||
if r.Status != "succeeded" && r.ResultPreview != "" && r.Diagnostic != "" {
|
||||
lines = append(lines, "Partial output: "+ansi.Strip(r.ResultPreview))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func runResultLines(r *protocol.RunDetail) []string {
|
||||
lines := []string{}
|
||||
if r.Diagnostic != "" {
|
||||
lines = append(lines, "Stopped: "+ansi.Strip(r.Diagnostic))
|
||||
}
|
||||
if r.Status == "succeeded" {
|
||||
if preview := protocol.OutputPreview(r.Output); preview != "" {
|
||||
lines = append(lines, "Result: "+ansi.Strip(preview))
|
||||
} else {
|
||||
lines = append(lines, "Finished · no result recorded")
|
||||
}
|
||||
} else {
|
||||
if r.Status == "running" || r.Status == "queued" {
|
||||
lines = append(lines, "In progress · final result pending")
|
||||
}
|
||||
for i := len(r.Steps) - 1; i >= 0; i-- {
|
||||
if r.Steps[i].Status == "completed" {
|
||||
if preview := protocol.OutputPreview(r.Steps[i].Output); preview != "" {
|
||||
lines = append(lines, "Latest completed output ("+r.Steps[i].Stage+"): "+ansi.Strip(preview))
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
completed := 0
|
||||
failed := 0
|
||||
for _, s := range r.Steps {
|
||||
if s.Status == "completed" {
|
||||
completed++
|
||||
}
|
||||
if s.Status == "failed" {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("Progress: %d completed visits · %d failed · %d recorded", completed, failed, len(r.Steps)))
|
||||
if r.CurrentStage != "" && (r.Status == "running" || r.Status == "queued") {
|
||||
lines = append(lines, "Working on: "+r.CurrentStage)
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func completedWorkLines(r *protocol.RunDetail) []string {
|
||||
var lines []string
|
||||
for _, s := range r.Steps {
|
||||
if s.Status != "completed" {
|
||||
continue
|
||||
}
|
||||
line := fmt.Sprintf("%d. %s · %s", s.Sequence, s.Stage, s.Outcome)
|
||||
if preview := protocol.OutputPreview(s.Output); preview != "" {
|
||||
line += " — " + ansi.Strip(preview)
|
||||
} else {
|
||||
line += " — no output recorded"
|
||||
}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
if len(lines) == 0 {
|
||||
lines = append(lines, "No completed stage output yet.")
|
||||
}
|
||||
return append([]string{"Completed stage outputs (reported, not independently verified):"}, lines...)
|
||||
}
|
||||
|
||||
func resultOutputLines(r *protocol.RunDetail) []string {
|
||||
if len(r.Output) > 0 {
|
||||
return valueLines(r.Output)
|
||||
}
|
||||
for i := len(r.Steps) - 1; i >= 0; i-- {
|
||||
if r.Steps[i].Status == "completed" && len(r.Steps[i].Output) > 0 {
|
||||
return append([]string{"Partial output · " + r.Steps[i].Stage + " (not a final result)", ""}, valueLines(r.Steps[i].Output)...)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(r.Diagnostic) != "" {
|
||||
return []string{r.Diagnostic}
|
||||
}
|
||||
return []string{"No output recorded yet."}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package tui
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"github.com/charmbracelet/x/ansi"
|
||||
)
|
||||
|
||||
func TestHomeAndPipelineShowRecordedResultsForExactVersion(t *testing.T) {
|
||||
m := New(context.Background(), "")
|
||||
m.width, m.height = 140, 48
|
||||
m.loaded = true
|
||||
m.pipelines = []protocol.PipelineSummary{{ID: 1, Name: "Maintenance", Version: 2, Stages: 2}}
|
||||
m.runs = []protocol.RunSummary{
|
||||
{ID: 12, PipelineID: 1, Version: 2, Name: "Maintenance", Status: "succeeded", ResultPreview: "Added regression coverage", CompletedSteps: 2},
|
||||
{ID: 11, PipelineID: 1, Version: 1, Status: "succeeded", ResultPreview: "Created the initial report", CompletedSteps: 1},
|
||||
}
|
||||
view := ansi.Strip(m.View())
|
||||
for _, want := range []string{"Result: Added regression coverage", "2 completed visits", "Last run #12"} {
|
||||
if !strings.Contains(view, want) {
|
||||
t.Fatal("home missing result", want, view)
|
||||
}
|
||||
}
|
||||
m.detail = &protocol.PipelineDetail{PipelineSummary: m.pipelines[0], LatestVersion: 2, Graph: protocol.Graph{Stages: []protocol.Stage{{Key: "worker", Edges: map[string]string{"success": protocol.SuccessEnd}}}}}
|
||||
view = ansi.Strip(m.pipelineView(40))
|
||||
if !strings.Contains(view, "Added regression coverage") || strings.Index(view, "Results") > strings.Index(view, "Graph") {
|
||||
t.Fatal("pipeline does not lead with results", view)
|
||||
}
|
||||
m.detail.Version = 1
|
||||
view = ansi.Strip(m.pipelineView(40))
|
||||
if !strings.Contains(view, "Created the initial report") || strings.Contains(view, "Added regression coverage") {
|
||||
t.Fatal("pipeline mixed version results", view)
|
||||
}
|
||||
m.detail.Version = 3
|
||||
if view = ansi.Strip(m.pipelineView(40)); !strings.Contains(view, "has not run yet") || strings.Contains(view, "Added regression coverage") {
|
||||
t.Fatal("unexecuted version invented results", view)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunLeadsWithResultAndPreservesPartialAccomplishments(t *testing.T) {
|
||||
m := New(context.Background(), "")
|
||||
m.width, m.height = 120, 48
|
||||
m.runDetailFocus = true
|
||||
r := &protocol.RunDetail{RunSummary: protocol.RunSummary{ID: 1, Name: "Maintenance", Status: "failed", Diagnostic: "Verification timed out"}, Steps: []protocol.RunStep{
|
||||
{Sequence: 1, Stage: "implement", Status: "completed", Outcome: "success", Output: map[string]any{"summary": "Updated the layout", "artifacts": []string{"screen.go"}}},
|
||||
{Sequence: 2, Stage: "verify", Status: "failed", Diagnostic: "Verification timed out"},
|
||||
}}
|
||||
m.runDetail = r
|
||||
view := ansi.Strip(m.runsView(44))
|
||||
for _, want := range []string{"Stopped: Verification timed out", "Latest completed output (implement): Updated the layout", "Partial output", "screen.go"} {
|
||||
if !strings.Contains(view, want) {
|
||||
t.Fatal("partial result hidden", want, view)
|
||||
}
|
||||
}
|
||||
if strings.Index(view, "Updated the layout") > strings.Index(view, "Run graph") {
|
||||
t.Fatal("graph precedes result")
|
||||
}
|
||||
lines := ansi.Strip(strings.Join(m.runSectionLines(100), "\n"))
|
||||
if !strings.Contains(lines, "1. implement · success — Updated the layout") || strings.Index(lines, "[o]") > strings.Index(lines, "[i]") {
|
||||
t.Fatal("recorded accomplishments hidden behind inputs", lines)
|
||||
}
|
||||
r.Status = "succeeded"
|
||||
r.Diagnostic = ""
|
||||
r.Output = map[string]any{"summary": "Layout verified", "report": "Checks passed", "artifacts": []string{"screen.go"}}
|
||||
if lines = ansi.Strip(strings.Join(m.runSectionLines(100), "\n")); !strings.Contains(lines, "Result: Layout verified") {
|
||||
t.Fatal("final result not prominent", lines)
|
||||
}
|
||||
r.Output = nil
|
||||
r.Steps = nil
|
||||
if lines = ansi.Strip(strings.Join(m.runSectionLines(100), "\n")); !strings.Contains(lines, "no result recorded") || strings.Contains(lines, "Layout verified") {
|
||||
t.Fatal("empty success fabricated work", lines)
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/charmbracelet/x/ansi"
|
||||
@@ -299,6 +300,11 @@ func evidenceLines(step protocol.RunStep) []string {
|
||||
return append(lines, "No captured tool evidence yet.")
|
||||
}
|
||||
var audit struct {
|
||||
Limits *struct {
|
||||
Timeout string `json:"timeout"`
|
||||
Retries int `json:"retries"`
|
||||
Budget stage.BudgetSnapshot `json:"budget"`
|
||||
} `json:"stage_limits"`
|
||||
Data []json.RawMessage `json:"data"`
|
||||
FormatRepairs []struct {
|
||||
Method string `json:"method"`
|
||||
@@ -347,6 +353,24 @@ func evidenceLines(step protocol.RunStep) []string {
|
||||
for _, repair := range audit.FormatRepairs {
|
||||
lines = append(lines, fmt.Sprintf("JSON format repair: %s · accepted: %t", repair.Method, repair.Accepted), repair.Diagnostic)
|
||||
}
|
||||
if limits := audit.Limits; limits != nil {
|
||||
b := limits.Budget
|
||||
lines = append(lines, fmt.Sprintf("Timeout per attempt: %s · Retries: %d · Attempts: %d", limits.Timeout, limits.Retries, b.Attempts))
|
||||
if b.LimitUSD > 0 {
|
||||
used := "unknown"
|
||||
if b.USDKnown {
|
||||
used = fmt.Sprintf("$%.6f", b.UsedUSD)
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("USD budget: %s / $%.6f", used, b.LimitUSD))
|
||||
}
|
||||
if b.LimitTokens > 0 {
|
||||
used := "unknown"
|
||||
if b.TokensKnown {
|
||||
used = fmt.Sprint(b.UsedTokens)
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("Token budget: %s / %d", used, b.LimitTokens))
|
||||
}
|
||||
}
|
||||
count := 0
|
||||
for i := len(audit.Data) - 1; i >= 0; i-- {
|
||||
var message struct {
|
||||
@@ -390,10 +414,7 @@ func (m Model) runSectionLines(width int) []string {
|
||||
r := m.runDetail
|
||||
titles := []string{"Summary", "Input", "Output", "Tool evidence", "Metadata"}
|
||||
shortcuts := []string{"s", "i", "o", "t", "m"}
|
||||
summary := []string{"Status: " + m.progress(r.Status), "Outcome: " + r.Outcome, fmt.Sprintf("Stages: %d · Current: %s", len(r.Steps), r.CurrentStage), "Started: " + dateLabel(r.CreatedAt), "Finished: " + dateLabel(r.FinishedAt), "Duration: " + runDuration(r.CreatedAt, r.FinishedAt)}
|
||||
if r.Diagnostic != "" {
|
||||
summary = append(summary, lipgloss.NewStyle().Foreground(themeError).Render(r.Diagnostic))
|
||||
}
|
||||
summary := append([]string{"Status: " + m.progress(r.Status)}, runResultLines(r)...)
|
||||
if m.runError != "" {
|
||||
summary = append(summary, "Request error: "+m.runError)
|
||||
}
|
||||
@@ -401,17 +422,22 @@ func (m Model) runSectionLines(width int) []string {
|
||||
if len(r.WritePaths) > 0 {
|
||||
editScope = strings.Join(r.WritePaths, ", ")
|
||||
}
|
||||
content := [][]string{summary, valueLines(r.Input), valueLines(r.Output), {"Select a stage to inspect its tool calls."}, {"Workspace: " + r.Workspace, "Source hash: " + r.ContentHash, fmt.Sprintf("Shell/edit enabled: %t", r.AllowTools), "Stage tools: defined by the pinned pipeline version", "Edit scope: " + editScope}}
|
||||
output := resultOutputLines(r)
|
||||
output = append(output, "", strings.Join(completedWorkLines(r), "\n"))
|
||||
content := [][]string{summary, valueLines(r.Input), output, {"Select a stage to inspect its tool calls."}, {"Started: " + dateLabel(r.CreatedAt), "Finished: " + dateLabel(r.FinishedAt), "Duration: " + runDuration(r.CreatedAt, r.FinishedAt), "Workspace: " + r.Workspace, "Source hash: " + r.ContentHash, fmt.Sprintf("Shell/edit enabled: %t", r.AllowTools), "Stage tools: defined by the pinned pipeline version", "Edit scope: " + editScope}}
|
||||
if r.RetryOf > 0 {
|
||||
content[4] = append(content[4], fmt.Sprintf("Retry of run #%d", r.RetryOf))
|
||||
}
|
||||
if m.runStepIndex > 0 && m.runStepIndex <= len(r.Steps) {
|
||||
step := r.Steps[m.runStepIndex-1]
|
||||
summary = []string{"Status: " + m.progress(m.stepProgressStatus(step)), "Route: " + outcomeLabel(step.Outcome, step.Outcome) + " → " + step.Target, "Started: " + dateLabel(step.StartedAt), "Finished: " + dateLabel(step.FinishedAt), "Duration: " + runDuration(step.StartedAt, step.FinishedAt)}
|
||||
summary = []string{"Status: " + m.progress(m.stepProgressStatus(step)), "Route: " + outcomeLabel(step.Outcome, step.Outcome) + " → " + step.Target}
|
||||
if preview := protocol.OutputPreview(step.Output); preview != "" {
|
||||
summary = append(summary, "Reported output: "+ansi.Strip(preview))
|
||||
}
|
||||
if step.Diagnostic != "" {
|
||||
summary = append(summary, lipgloss.NewStyle().Foreground(themeError).Render(step.Diagnostic))
|
||||
}
|
||||
content = [][]string{summary, valueLines(step.Input), valueLines(step.Output), evidenceLines(step), {"Session: " + step.SessionID, fmt.Sprintf("Agent active: %t", step.AgentActive), fmt.Sprintf("Attempt sequence: %d", step.Sequence)}}
|
||||
content = [][]string{summary, valueLines(step.Input), valueLines(step.Output), evidenceLines(step), {"Started: " + dateLabel(step.StartedAt), "Finished: " + dateLabel(step.FinishedAt), "Duration: " + runDuration(step.StartedAt, step.FinishedAt), "Session: " + step.SessionID, fmt.Sprintf("Agent active: %t", step.AgentActive), fmt.Sprintf("Attempt sequence: %d", step.Sequence)}}
|
||||
for _, receipt := range step.Notifications {
|
||||
line := "Notification: " + receipt.Destination + " · " + receipt.Status
|
||||
if receipt.Filename != "" {
|
||||
@@ -430,7 +456,8 @@ func (m Model) runSectionLines(width int) []string {
|
||||
}
|
||||
}
|
||||
var lines []string
|
||||
for i, title := range titles {
|
||||
for _, i := range []int{0, 2, 1, 3, 4} {
|
||||
title := titles[i]
|
||||
marker := "▾"
|
||||
if m.runSectionClosed(i) {
|
||||
marker = "▸"
|
||||
@@ -529,11 +556,12 @@ func (m Model) runsDetailsView(height int) string {
|
||||
rows = append(rows, "Search: "+m.runSearch.View())
|
||||
}
|
||||
runs := m.visibleRuns()
|
||||
start, end := selectionWindow(len(runs), m.runIndex, max(1, height-4-len(rows)))
|
||||
start, end := selectionWindow(len(runs), m.runIndex, max(1, (height-4-len(rows))/2))
|
||||
for i := start; i < end; i++ {
|
||||
r := runs[i]
|
||||
text := fmt.Sprintf("#%d %s · v%d · %s · %s", r.ID, r.Name, r.Version, m.progress(r.Status), dateLabel(r.FinishedAt))
|
||||
rows = append(rows, menuOption(text, i == m.runIndex))
|
||||
rows = append(rows, " "+ansi.Truncate(resultLabel(r), max(1, m.width-6), "…"))
|
||||
}
|
||||
if len(runs) == 0 {
|
||||
rows = append(rows, "No runs yet.")
|
||||
|
||||
@@ -97,7 +97,7 @@ func (m Model) runsView(height int) string {
|
||||
}
|
||||
graph := m.runGraphPanel(graphHeight, top, pan)
|
||||
details := m.runsDetailsView(detailHeight)
|
||||
return fitBlock(strings.Split(graph+"\n"+details, "\n"), m.width, height)
|
||||
return fitBlock(strings.Split(details+"\n"+graph, "\n"), m.width, height)
|
||||
}
|
||||
|
||||
func (m Model) runPaneHeights(height int) (int, int) {
|
||||
|
||||
@@ -103,8 +103,8 @@ func TestRunPathUsesSharedRendererAndRecordedRoutesWithRepeatedVisits(t *testing
|
||||
t.Fatal("run graph diverged from common renderer")
|
||||
}
|
||||
view := ansi.Strip(m.runsView(36))
|
||||
if strings.Index(view, "Run graph") < 0 || strings.Index(view, "Run graph") > strings.Index(view, "Run overview") || !strings.Contains(view, "3. plan (running)") {
|
||||
t.Fatal("run graph missing above details or current visit not visible", view)
|
||||
if strings.Index(view, "Run graph") < 0 || strings.Index(view, "Run graph") < strings.Index(view, "Run overview") || !strings.Contains(view, "3. plan (running)") {
|
||||
t.Fatal("run graph missing below results or current visit not visible", view)
|
||||
}
|
||||
refreshed := *m.runDetail
|
||||
refreshed.Steps = append([]protocol.RunStep(nil), m.runDetail.Steps...)
|
||||
|
||||
@@ -18,8 +18,13 @@ func TestRunPanesExpandWithTabFocus(t *testing.T) {
|
||||
updated, _ := m.Update(receivedMsg(protocol.Message{Type: protocol.Run, RunDetail: run}))
|
||||
m = updated.(Model)
|
||||
graph, details := m.runPaneHeights(44)
|
||||
if !m.runGraphFocus || graph <= details || graph+details != 44 {
|
||||
t.Fatal("new run did not prioritize graph", graph, details)
|
||||
if !m.runDetailFocus || m.runGraphFocus || details <= graph || graph+details != 44 {
|
||||
t.Fatal("new run did not prioritize results", graph, details)
|
||||
}
|
||||
m = runKey(m, "tab")
|
||||
graph, details = m.runPaneHeights(44)
|
||||
if !m.runGraphFocus || graph <= details {
|
||||
t.Fatal("graph focus did not expand")
|
||||
}
|
||||
graphPage := m.runPageSize()
|
||||
m = runKey(m, "tab")
|
||||
|
||||
@@ -195,8 +195,8 @@ func TestRunSectionsOrderTitleAndNumberSelection(t *testing.T) {
|
||||
m := runInspectionModel()
|
||||
m = runKey(m, "1")
|
||||
text := ansi.Strip(strings.Join(m.runSectionLines(70), "\n"))
|
||||
if strings.Index(text, "Input") > strings.Index(text, "Output") {
|
||||
t.Fatal("input must precede output")
|
||||
if strings.Index(text, "Output") > strings.Index(text, "Input") {
|
||||
t.Fatal("output must precede input")
|
||||
}
|
||||
if m.runSectionClosed(2) || !m.runSectionClosed(1) {
|
||||
t.Fatal("output should remain visible; input initially collapsed")
|
||||
|
||||
Reference in New Issue
Block a user