Files
pipelines/internal/server/human_test.go
T
alan a6aff47117 Add a human decision stage that pauses a run for an operator's choice
A new registered stage type `human` (internal/stage/human.go) has no model,
agent, tools or configs: its required `prompt` block carries the operator
instructions and its non-error edges are the selectable options. It parses,
validates, renders, exports and imports like every other type (no parser
special-case, no config keys).

At runtime the runner parks the run instead of executing: the step and the run
persist `waiting` with self-contained {prompt, options} evidence, and the run
returns without starting later stages, capturing file diffs, scanning artifacts,
holding a goroutine/timer or applying a stage timeout. The wait is durable: a
restart leaves the waiting rows untouched (RecoverRuns excludes them) and a
rebuilt runtime over the same database still accepts the decision.

`pipelines decide <run-id> [stage] --option KEY [--feedback TEXT] [--json]` and
the TUI `c` picker resolve the decision from the pinned graph in one store
transaction, record the choice and feedback as the step's evidence and resume
along edges[KEY]. An unknown option is refused with the valid list, a stage that
is not waiting is refused, and a repeated decision for an already-decided visit
does not double-apply. Cancel and retry keep working while waiting and after a
decision; a human stage is never killed by a stage timeout or max_visits, and a
`notify` policy can fire an `on=waiting` event.

Docs: the authoring skill references (format.md, operations.md), docs/cli.md and
README describe the stage, the command and the TUI picker. Tests cover stage
validation, the parsing round-trip, the store transitions, a websocket wait/decide
integration, cancel/retry, waiting notifications and a persistence-layer rebuild
(second Open + newRunner over one database, no process restart).
2026-10-16 16:47:28 +01:00

439 lines
15 KiB
Go

package server
import (
"context"
"encoding/json"
"fmt"
"mime"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
)
const humanPipelineSource = "# stage gate\n\n```output\nDecision evidence.\n```\n\n```prompt\nReview the screenshots and choose where the run goes.\n```\n\n```edges\ngo -> worker\nhalt -> #success\nerror -> #failure\n```\n\n```configs\ntype=human\n```\n\n# stage worker\n\n```output\nWork result.\n```\n\n```prompt\nDo the work.\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=2\ntimeout=1h\n```\n\n"
func successExecute(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
}
// startHumanRun submits the human pipeline (once) and starts one run, returning
// once the run goroutine has parked the run in the waiting state.
func startHumanRun(t *testing.T, ctx context.Context, r *runner, store *pipeline.Store, key string) *protocol.RunDetail {
t.Helper()
items, _ := store.List(ctx)
var pipelineID int64
for _, p := range items {
if p.Name == "human" {
pipelineID = p.ID
}
}
if pipelineID == 0 {
if _, err := store.Submit(ctx, "human", humanPipelineSource); err != nil {
t.Fatal(err)
}
items, _ = store.List(ctx)
for _, p := range items {
if p.Name == "human" {
pipelineID = p.ID
}
}
}
run, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
return run
}
func TestHumanStageWaitsAndDecisionResumes(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "human.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r.wg.Wait()
r.execute = successExecute
run := startHumanRun(t, ctx, r, store, "human-go")
detail, err := r.getRun(run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != protocol.RunStatusWaiting || detail.CurrentStage != "gate" {
t.Fatalf("run did not wait: %+v", detail.RunSummary)
}
if len(detail.Steps) != 1 || detail.Steps[0].Status != protocol.StepStatusWaiting {
t.Fatalf("waiting step not persisted: %+v", detail.Steps)
}
if prompt := protocol.WaitingPrompt(detail.Steps[0].Output); !strings.Contains(prompt, "Review the screenshots") {
t.Fatalf("waiting step omitted its prompt: %+v", detail.Steps[0].Output)
}
if options := protocol.WaitingOptions(detail.Steps[0].Output); strings.Join(options, ",") != "go,halt" {
t.Fatalf("waiting step omitted its options: %+v", options)
}
if detail.Steps[0].FinishedAt != "" {
t.Fatalf("waiting step must not be finished: %+v", detail.Steps[0])
}
// No later step is created while waiting.
if len(detail.Steps) != 1 {
t.Fatalf("a later step ran while waiting: %+v", detail.Steps)
}
// Unknown option is refused with the valid list.
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "launch"}); err == nil || !strings.Contains(err.Error(), "valid options") || !strings.Contains(err.Error(), "go") {
t.Fatalf("unknown option not refused with the list: %v", err)
}
decided, err := r.decide(protocol.Message{RunID: run.ID, Option: "go", Feedback: "looks good"})
if err != nil {
t.Fatal(err)
}
if decided.Status != "running" {
t.Fatalf("decided run did not resume: %+v", decided.RunSummary)
}
r.wg.Wait()
detail, err = store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "succeeded" || len(detail.Steps) != 2 {
t.Fatalf("run did not resume to completion: %+v", detail.RunSummary)
}
gate := detail.Steps[0]
if gate.Status != "completed" || gate.Outcome != "go" || gate.Target != "worker" || gate.Diagnostic != "" {
t.Fatalf("decision evidence not recorded: %+v", gate)
}
if gate.Output["option"] != "go" || gate.Output["feedback"] != "looks good" {
t.Fatalf("decision output missing option/feedback: %+v", gate.Output)
}
if gate.Verdict == nil || gate.Verdict.Option != "go → worker" || gate.Verdict.Reason != "looks good" {
t.Fatalf("derived verdict missing the decision: %+v", gate.Verdict)
}
// A repeated decision for the already-decided visit must be an idempotent
// no-op (applied=false, no error) and must not create another step. Naming
// the stage explicitly keeps this on the decided visit instead of relying on
// current_stage having advanced past it.
repeat, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "go"})
if err != nil {
t.Fatalf("repeat decision for a decided visit was refused: %v", err)
}
if repeat == nil || repeat.Status != "succeeded" {
t.Fatalf("repeat decision changed the run: %+v", repeat)
}
after, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if len(after.Steps) != 2 {
t.Fatalf("repeat decision created another step: %+v", after.Steps)
}
}
func TestHumanDecisionToTerminalFinalizesRun(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "terminal.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r.wg.Wait()
r.execute = successExecute
run := startHumanRun(t, ctx, r, store, "human-halt")
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt", Feedback: "approved"}); err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "succeeded" || len(detail.Steps) != 1 || detail.Steps[0].Target != protocol.SuccessEnd {
t.Fatalf("decision to success did not finalize the run: %+v", detail.RunSummary)
}
}
func TestHumanStageCancelRetryAndRebuild(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
path := filepath.Join(t.TempDir(), "rebuild.db")
store, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = successExecute
run := startHumanRun(t, ctx, r, store, "human-rebuild")
// Cancel works while waiting.
if err := r.cancel(run.ID); err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "canceled" || detail.Steps[0].Status != "canceled" {
t.Fatalf("waiting run not canceled: %+v %+v", detail.RunSummary, detail.Steps)
}
if err := r.cancel(run.ID); err == nil {
t.Fatal("canceling a canceled run succeeded")
}
if err := store.Close(); err != nil {
t.Fatal(err)
}
// Rebuild runtime state from persisted rows: a second Open + newRunner over
// the same database, with no process restart. RecoverRuns must leave a
// waiting run waiting.
store, err = pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
defer store.Close()
r2, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r2.wg.Wait()
r2.execute = successExecute
waiting := startHumanRun(t, ctx, r2, store, "human-rebuild-2")
if err := store.Close(); err != nil {
t.Fatal(err)
}
store, err = pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
r3, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r3.wg.Wait()
r3.execute = successExecute
recovered, err := r3.getRun(waiting.ID)
if err != nil {
t.Fatal(err)
}
if recovered.Status != protocol.RunStatusWaiting || recovered.Steps[0].Status != protocol.StepStatusWaiting {
t.Fatalf("rebuilt runtime lost the wait: %+v %+v", recovered.RunSummary, recovered.Steps)
}
if _, err := r3.decide(protocol.Message{RunID: waiting.ID, Option: "go", Feedback: "after restart"}); err != nil {
t.Fatal(err)
}
r3.wg.Wait()
final, err := store.GetRun(ctx, waiting.ID)
if err != nil {
t.Fatal(err)
}
if final.Status != "succeeded" || final.Steps[0].Outcome != "go" || final.Steps[0].Output["feedback"] != "after restart" {
t.Fatalf("rebuilt runtime refused or mis-recorded the decision: %+v", final)
}
// Retry works after a decision that later fails.
failing := startHumanRun(t, ctx, r3, store, "human-retry")
r3.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{}, fmt.Errorf("provider failed")
}
if _, err := r3.decide(protocol.Message{RunID: failing.ID, Option: "go"}); err != nil {
t.Fatal(err)
}
r3.wg.Wait()
failed, err := store.GetRun(ctx, failing.ID)
if err != nil {
t.Fatal(err)
}
if failed.Status != "failed" {
t.Fatalf("post-decision failure did not fail the run: %+v", failed.RunSummary)
}
r3.execute = successExecute
retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human"})
if err != nil {
t.Fatal(err)
}
r3.wg.Wait()
retriedDetail, err := store.GetRun(ctx, retried.ID)
if err != nil {
t.Fatal(err)
}
if retried.ID == failing.ID || retriedDetail.Status != protocol.RunStatusWaiting {
t.Fatalf("retry after a decision did not start a fresh waiting run: %+v", retriedDetail.RunSummary)
}
}
const humanOnlySource = "# stage gate\n\n```output\nDecision evidence.\n```\n\n```prompt\nApprove the release.\n```\n\n```edges\ngo -> #success\nhalt -> #failure\nerror -> #failure\n```\n\n```configs\ntype=human\n```\n\n"
func TestHumanRunWebSocketWaitAndDecide(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "ws.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "human-ws", humanOnlySource); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
handler := Handler(ctx, store).(*runHandler)
defer func() { cancel(); WaitRuns(handler) }()
srv := httptest.NewServer(handler)
defer srv.Close()
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
if err != nil {
t.Fatal(err)
}
defer conn.CloseNow()
var snapshot protocol.Message
if err := wsjson.Read(ctx, conn, &snapshot); err != nil {
t.Fatal(err)
}
request := func(message protocol.Message) protocol.Message {
t.Helper()
if err := wsjson.Write(ctx, conn, message); err != nil {
t.Fatal(err)
}
var response protocol.Message
if err := wsjson.Read(ctx, conn, &response); err != nil {
t.Fatal(err)
}
return response
}
started := request(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: items[0].ID, Version: 1, RequestKey: "ws-human", Input: map[string]any{"prompt": "ship it"}})
if started.Type != protocol.RunStarted || started.RunDetail == nil {
t.Fatalf("run not started: %+v", started)
}
runID := started.RunDetail.ID
var waiting protocol.Message
for i := 0; i < 50; i++ {
waiting = request(protocol.Message{Type: protocol.GetRun, ID: "poll", RunID: runID})
if waiting.RunDetail != nil && waiting.RunDetail.Status == protocol.RunStatusWaiting {
break
}
if waiting.Type == protocol.Error {
t.Fatal(waiting.Error)
}
time.Sleep(20 * time.Millisecond)
}
if waiting.RunDetail == nil || waiting.RunDetail.Status != protocol.RunStatusWaiting || waiting.RunDetail.CurrentStage != "gate" {
t.Fatalf("run never reached waiting: %+v", waiting.RunDetail)
}
if len(waiting.RunDetail.Steps) != 1 || waiting.RunDetail.Steps[0].Status != protocol.StepStatusWaiting {
t.Fatalf("waiting step not visible over websocket: %+v", waiting.RunDetail.Steps)
}
if prompt := protocol.WaitingPrompt(waiting.RunDetail.Steps[0].Output); !strings.Contains(prompt, "Approve the release") {
t.Fatalf("prompt not visible over websocket: %+v", waiting.RunDetail.Steps[0].Output)
}
if options := protocol.WaitingOptions(waiting.RunDetail.Steps[0].Output); strings.Join(options, ",") != "go,halt" {
t.Fatalf("options not visible over websocket: %+v", options)
}
// Unknown option is refused verbatim over the wire.
bad := request(protocol.Message{Type: protocol.DecideRun, ID: "bad", RunID: runID, Option: "launch"})
if bad.Type != protocol.Error || !strings.Contains(bad.Error, "valid options") {
t.Fatalf("unknown option not refused over websocket: %+v", bad)
}
decided := request(protocol.Message{Type: protocol.DecideRun, ID: "decide", RunID: runID, Option: "go", Feedback: "approved"})
if decided.Type != protocol.Run || decided.RunDetail == nil {
t.Fatalf("decision not accepted over websocket: %+v", decided)
}
var final protocol.Message
for i := 0; i < 50; i++ {
final = request(protocol.Message{Type: protocol.GetRun, ID: "final", RunID: runID})
if final.RunDetail != nil && final.RunDetail.Status != protocol.RunStatusWaiting && final.RunDetail.Status != "running" {
break
}
time.Sleep(20 * time.Millisecond)
}
if final.RunDetail == nil || final.RunDetail.Status != "succeeded" {
t.Fatalf("decided run did not complete over websocket: %+v", final.RunDetail)
}
if final.RunDetail.Steps[0].Outcome != "go" || final.RunDetail.Steps[0].Output["feedback"] != "approved" {
t.Fatalf("decision evidence not persisted over websocket: %+v", final.RunDetail.Steps[0])
}
}
func TestHumanStageWaitingNotification(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "notify.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
filename, _ := new(mime.WordDecoder).DecodeHeader(r.Header.Get("Filename"))
ack := map[string]any{"id": "msg1", "event": "message"}
if filename != "" {
ack["attachment"] = map[string]string{"name": filename, "url": "https://ntfy.example.com/file/" + filename}
}
_ = json.NewEncoder(w).Encode(ack)
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
source := strings.Replace(humanPipelineSource, "```configs\ntype=human\n```", "```notify\non=waiting\n```\n\n```configs\ntype=human\n```", 1)
if _, err := store.Submit(ctx, "human-notify", source); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
var pipelineID int64
for _, p := range items {
if p.Name == "human-notify" {
pipelineID = p.ID
}
}
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r.wg.Wait()
run, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "human-notify", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
if calls.Load() != 1 {
t.Fatalf("waiting notification fired %d times", calls.Load())
}
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if len(detail.Steps[0].Notifications) != 1 || detail.Steps[0].Notifications[0].Status != "sent" {
t.Fatalf("waiting notification receipt missing: %+v", detail.Steps[0].Notifications)
}
}