Files
pipelines/internal/server/human_test.go
T
alan 3bfb3d6b44 Fix decide-after-finish refusal and run size loop data race
A run stays in r.active after finishRunState persists its terminal status
until its goroutine finishes notifying, so a decision in that window was
refused as "already active". decide now refuses only runs whose terminal
state is not yet durable; a finished run gets ResolveDecision's
already-decided / not-waiting answer, and an executing run is still refused.

Tests overwrote runner.artifactsRoot after Handler/newRunner had started
background goroutines (sizeLoop, session export jobs) that read it. The
root is now fixed at construction through newRunnerAt, which
HandlerWithOptions uses for Options.ArtifactsRoot, and tests configure it
there instead of mutating the field afterwards.
2026-11-01 18:58:14 +00:00

494 lines
17 KiB
Go

package server
import (
"context"
"encoding/json"
"fmt"
"mime"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
)
const humanPipelineSource = "# stage gate\n\n```output\nDecision evidence.\n```\n\n```prompt\nReview the screenshots and choose where the run goes.\n```\n\n```edges\ngo -> worker\nhalt -> #success\nerror -> #failure\n```\n\n```configs\ntype=human\n```\n\n# stage worker\n\n```output\nWork result.\n```\n\n```prompt\nDo the work.\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=2\ntimeout=1h\n```\n\n"
func successExecute(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
}
// startHumanRun submits the human pipeline (once) and starts one run, returning
// once the run goroutine has parked the run in the waiting state.
func startHumanRun(t *testing.T, ctx context.Context, r *runner, store *pipeline.Store, key string) *protocol.RunDetail {
t.Helper()
items, _ := store.List(ctx)
var pipelineID int64
for _, p := range items {
if p.Name == "human" {
pipelineID = p.ID
}
}
if pipelineID == 0 {
if _, err := store.Submit(ctx, "human", humanPipelineSource); err != nil {
t.Fatal(err)
}
items, _ = store.List(ctx)
for _, p := range items {
if p.Name == "human" {
pipelineID = p.ID
}
}
}
return startRun(t, r, pipelineID, key)
}
func TestHumanStageWaitsAndDecisionResumes(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "human.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r.wg.Wait()
r.execute = successExecute
run := startHumanRun(t, ctx, r, store, "human-go")
detail, err := r.getRun(run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != protocol.RunStatusWaiting || detail.CurrentStage != "gate" {
t.Fatalf("run did not wait: %+v", detail.RunSummary)
}
if len(detail.Steps) != 1 || detail.Steps[0].Status != protocol.StepStatusWaiting {
t.Fatalf("waiting step not persisted: %+v", detail.Steps)
}
if prompt := protocol.WaitingPrompt(detail.Steps[0].Output); !strings.Contains(prompt, "Review the screenshots") {
t.Fatalf("waiting step omitted its prompt: %+v", detail.Steps[0].Output)
}
if options := protocol.WaitingOptions(detail.Steps[0].Output); strings.Join(options, ",") != "go,halt" {
t.Fatalf("waiting step omitted its options: %+v", options)
}
if detail.Steps[0].FinishedAt != "" {
t.Fatalf("waiting step must not be finished: %+v", detail.Steps[0])
}
// No later step is created while waiting.
if len(detail.Steps) != 1 {
t.Fatalf("a later step ran while waiting: %+v", detail.Steps)
}
// Unknown option is refused with the valid list.
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "launch"}); err == nil || !strings.Contains(err.Error(), "valid options") || !strings.Contains(err.Error(), "go") {
t.Fatalf("unknown option not refused with the list: %v", err)
}
decided, err := r.decide(protocol.Message{RunID: run.ID, Option: "go", Feedback: "looks good"})
if err != nil {
t.Fatal(err)
}
if decided.Status != "running" {
t.Fatalf("decided run did not resume: %+v", decided.RunSummary)
}
r.wg.Wait()
detail, err = store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "succeeded" || len(detail.Steps) != 2 {
t.Fatalf("run did not resume to completion: %+v", detail.RunSummary)
}
gate := detail.Steps[0]
if gate.Status != "completed" || gate.Outcome != "go" || gate.Target != "worker" || gate.Diagnostic != "" {
t.Fatalf("decision evidence not recorded: %+v", gate)
}
if gate.Output["option"] != "go" || gate.Output["feedback"] != "looks good" {
t.Fatalf("decision output missing option/feedback: %+v", gate.Output)
}
if gate.Verdict == nil || gate.Verdict.Option != "go → worker" || gate.Verdict.Reason != "looks good" {
t.Fatalf("derived verdict missing the decision: %+v", gate.Verdict)
}
// A repeated decision for the already-decided visit must be an idempotent
// no-op (applied=false, no error) and must not create another step. Naming
// the stage explicitly keeps this on the decided visit instead of relying on
// current_stage having advanced past it.
repeat, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "go"})
if err != nil {
t.Fatalf("repeat decision for a decided visit was refused: %v", err)
}
if repeat == nil || repeat.Status != "succeeded" {
t.Fatalf("repeat decision changed the run: %+v", repeat)
}
after, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if len(after.Steps) != 2 {
t.Fatalf("repeat decision created another step: %+v", after.Steps)
}
}
func TestHumanDecisionToTerminalFinalizesRun(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "terminal.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r.wg.Wait()
r.execute = successExecute
run := startHumanRun(t, ctx, r, store, "human-halt")
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt", Feedback: "approved"}); err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "succeeded" || len(detail.Steps) != 1 || detail.Steps[0].Target != protocol.SuccessEnd {
t.Fatalf("decision to success did not finalize the run: %+v", detail.RunSummary)
}
}
func TestHumanStageCancelRetryAndRebuild(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
path := filepath.Join(t.TempDir(), "rebuild.db")
store, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = successExecute
run := startHumanRun(t, ctx, r, store, "human-rebuild")
// Cancel works while waiting.
if err := r.cancel(run.ID); err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "canceled" || detail.Steps[0].Status != "canceled" {
t.Fatalf("waiting run not canceled: %+v %+v", detail.RunSummary, detail.Steps)
}
if err := r.cancel(run.ID); err == nil {
t.Fatal("canceling a canceled run succeeded")
}
if err := store.Close(); err != nil {
t.Fatal(err)
}
// Rebuild runtime state from persisted rows: a second Open + newRunner over
// the same database, with no process restart. RecoverRuns must leave a
// waiting run waiting.
store, err = pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
defer store.Close()
r2, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r2.wg.Wait()
r2.execute = successExecute
waiting := startHumanRun(t, ctx, r2, store, "human-rebuild-2")
if err := store.Close(); err != nil {
t.Fatal(err)
}
store, err = pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
r3, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r3.wg.Wait()
r3.execute = successExecute
recovered, err := r3.getRun(waiting.ID)
if err != nil {
t.Fatal(err)
}
if recovered.Status != protocol.RunStatusWaiting || recovered.Steps[0].Status != protocol.StepStatusWaiting {
t.Fatalf("rebuilt runtime lost the wait: %+v %+v", recovered.RunSummary, recovered.Steps)
}
if _, err := r3.decide(protocol.Message{RunID: waiting.ID, Option: "go", Feedback: "after restart"}); err != nil {
t.Fatal(err)
}
r3.wg.Wait()
final, err := store.GetRun(ctx, waiting.ID)
if err != nil {
t.Fatal(err)
}
if final.Status != "succeeded" || final.Steps[0].Outcome != "go" || final.Steps[0].Output["feedback"] != "after restart" {
t.Fatalf("rebuilt runtime refused or mis-recorded the decision: %+v", final)
}
// Retry works after a decision that later fails.
failing := startHumanRun(t, ctx, r3, store, "human-retry")
r3.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{}, fmt.Errorf("provider failed")
}
if _, err := r3.decide(protocol.Message{RunID: failing.ID, Option: "go"}); err != nil {
t.Fatal(err)
}
r3.wg.Wait()
failed, err := store.GetRun(ctx, failing.ID)
if err != nil {
t.Fatal(err)
}
if failed.Status != "failed" {
t.Fatalf("post-decision failure did not fail the run: %+v", failed.RunSummary)
}
r3.execute = successExecute
retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human", RetryMode: "restart"})
if err != nil {
t.Fatal(err)
}
r3.wg.Wait()
retriedDetail, err := store.GetRun(ctx, retried.ID)
if err != nil {
t.Fatal(err)
}
if retried.ID == failing.ID || retriedDetail.Status != protocol.RunStatusWaiting {
t.Fatalf("retry after a decision did not start a fresh waiting run: %+v", retriedDetail.RunSummary)
}
}
const humanOnlySource = "# stage gate\n\n```output\nDecision evidence.\n```\n\n```prompt\nApprove the release.\n```\n\n```edges\ngo -> #success\nhalt -> #failure\nerror -> #failure\n```\n\n```configs\ntype=human\n```\n\n"
func TestHumanRunWebSocketWaitAndDecide(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "ws.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "human-ws", humanOnlySource); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
handler := Handler(ctx, store).(*runHandler)
defer func() { cancel(); WaitRuns(handler) }()
srv := httptest.NewServer(handler)
defer srv.Close()
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
if err != nil {
t.Fatal(err)
}
defer conn.CloseNow()
var snapshot protocol.Message
if err := wsjson.Read(ctx, conn, &snapshot); err != nil {
t.Fatal(err)
}
request := func(message protocol.Message) protocol.Message {
t.Helper()
if err := wsjson.Write(ctx, conn, message); err != nil {
t.Fatal(err)
}
var response protocol.Message
if err := wsjson.Read(ctx, conn, &response); err != nil {
t.Fatal(err)
}
return response
}
started := request(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: items[0].ID, Version: 1, RequestKey: "ws-human", Input: map[string]any{"prompt": "ship it"}})
if started.Type != protocol.RunStarted || started.RunDetail == nil {
t.Fatalf("run not started: %+v", started)
}
runID := started.RunDetail.ID
var waiting protocol.Message
for i := 0; i < 50; i++ {
waiting = request(protocol.Message{Type: protocol.GetRun, ID: "poll", RunID: runID})
if waiting.RunDetail != nil && waiting.RunDetail.Status == protocol.RunStatusWaiting {
break
}
if waiting.Type == protocol.Error {
t.Fatal(waiting.Error)
}
time.Sleep(20 * time.Millisecond)
}
if waiting.RunDetail == nil || waiting.RunDetail.Status != protocol.RunStatusWaiting || waiting.RunDetail.CurrentStage != "gate" {
t.Fatalf("run never reached waiting: %+v", waiting.RunDetail)
}
if len(waiting.RunDetail.Steps) != 1 || waiting.RunDetail.Steps[0].Status != protocol.StepStatusWaiting {
t.Fatalf("waiting step not visible over websocket: %+v", waiting.RunDetail.Steps)
}
if prompt := protocol.WaitingPrompt(waiting.RunDetail.Steps[0].Output); !strings.Contains(prompt, "Approve the release") {
t.Fatalf("prompt not visible over websocket: %+v", waiting.RunDetail.Steps[0].Output)
}
if options := protocol.WaitingOptions(waiting.RunDetail.Steps[0].Output); strings.Join(options, ",") != "go,halt" {
t.Fatalf("options not visible over websocket: %+v", options)
}
// Unknown option is refused verbatim over the wire.
bad := request(protocol.Message{Type: protocol.DecideRun, ID: "bad", RunID: runID, Option: "launch"})
if bad.Type != protocol.Error || !strings.Contains(bad.Error, "valid options") {
t.Fatalf("unknown option not refused over websocket: %+v", bad)
}
decided := request(protocol.Message{Type: protocol.DecideRun, ID: "decide", RunID: runID, Option: "go", Feedback: "approved"})
if decided.Type != protocol.Run || decided.RunDetail == nil {
t.Fatalf("decision not accepted over websocket: %+v", decided)
}
var final protocol.Message
for i := 0; i < 50; i++ {
final = request(protocol.Message{Type: protocol.GetRun, ID: "final", RunID: runID})
if final.RunDetail != nil && final.RunDetail.Status != protocol.RunStatusWaiting && final.RunDetail.Status != "running" {
break
}
time.Sleep(20 * time.Millisecond)
}
if final.RunDetail == nil || final.RunDetail.Status != "succeeded" {
t.Fatalf("decided run did not complete over websocket: %+v", final.RunDetail)
}
if final.RunDetail.Steps[0].Outcome != "go" || final.RunDetail.Steps[0].Output["feedback"] != "approved" {
t.Fatalf("decision evidence not persisted over websocket: %+v", final.RunDetail.Steps[0])
}
}
func TestHumanStageWaitingNotification(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "notify.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
filename, _ := new(mime.WordDecoder).DecodeHeader(r.Header.Get("Filename"))
ack := map[string]any{"id": "msg1", "event": "message"}
if filename != "" {
ack["attachment"] = map[string]string{"name": filename, "url": "https://ntfy.example.com/file/" + filename}
}
_ = json.NewEncoder(w).Encode(ack)
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
source := strings.Replace(humanPipelineSource, "```configs\ntype=human\n```", "```notify\non=waiting\n```\n\n```configs\ntype=human\n```", 1)
if _, err := store.Submit(ctx, "human-notify", source); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
var pipelineID int64
for _, p := range items {
if p.Name == "human-notify" {
pipelineID = p.ID
}
}
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r.wg.Wait()
run, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "human-notify", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
if calls.Load() != 1 {
t.Fatalf("waiting notification fired %d times", calls.Load())
}
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if len(detail.Steps[0].Notifications) != 1 || detail.Steps[0].Notifications[0].Status != "sent" {
t.Fatalf("waiting notification receipt missing: %+v", detail.Steps[0].Notifications)
}
}
// TestDecideOnFinishingRunIsAnsweredNotRefused pins the window between
// finishRunState persisting a terminal status and the launch teardown removing
// the run from r.active: a decision then gets the store's idempotent/not-waiting
// answer, while a run that is still executing is refused as active.
func TestDecideOnFinishingRunIsAnsweredNotRefused(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "human.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r.wg.Wait()
r.execute = successExecute
run := startHumanRun(t, ctx, r, store, "human-finishing")
// A waiting run that is genuinely executing (registered, not finished) is
// refused.
r.mu.Lock()
r.active[run.ID] = activeRun{cancel: func() {}, workspace: run.Workspace}
r.mu.Unlock()
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt"}); err == nil || !strings.Contains(err.Error(), "already active") {
t.Fatalf("decision on an executing run was not refused: %v", err)
}
r.mu.Lock()
delete(r.active, run.ID)
r.mu.Unlock()
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt", Feedback: "stop"}); err != nil {
t.Fatal(err)
}
r.wg.Wait()
if detail, err := store.GetRun(ctx, run.ID); err != nil || detail.Status != "succeeded" {
t.Fatalf("run did not finish: %+v %v", detail, err)
}
// Simulate the finishing window: terminal status persisted, reservation not
// yet released.
r.mu.Lock()
r.active[run.ID] = activeRun{cancel: func() {}, workspace: run.Workspace}
r.mu.Unlock()
repeat, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "halt"})
if err != nil {
t.Fatalf("repeat decision on a finished run was refused: %v", err)
}
if repeat.Status != "succeeded" || repeat.Steps[0].Output["feedback"] != "stop" {
t.Fatalf("repeat decision changed the run: %+v", repeat)
}
if _, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "go"}); err == nil || !strings.Contains(err.Error(), "not waiting") {
t.Fatalf("conflicting decision on a finished run did not report not waiting: %v", err)
}
r.mu.Lock()
delete(r.active, run.ID)
r.mu.Unlock()
}