434 lines
15 KiB
Go
434 lines
15 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"mime"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"github.com/coder/websocket"
|
|
"github.com/coder/websocket/wsjson"
|
|
)
|
|
|
|
const humanPipelineSource = "# stage gate\n\n```output\nDecision evidence.\n```\n\n```prompt\nReview the screenshots and choose where the run goes.\n```\n\n```edges\ngo -> worker\nhalt -> #success\nerror -> #failure\n```\n\n```configs\ntype=human\n```\n\n# stage worker\n\n```output\nWork result.\n```\n\n```prompt\nDo the work.\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=2\ntimeout=1h\n```\n\n"
|
|
|
|
func successExecute(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
|
|
}
|
|
|
|
// startHumanRun submits the human pipeline (once) and starts one run, returning
|
|
// once the run goroutine has parked the run in the waiting state.
|
|
func startHumanRun(t *testing.T, ctx context.Context, r *runner, store *pipeline.Store, key string) *protocol.RunDetail {
|
|
t.Helper()
|
|
items, _ := store.List(ctx)
|
|
var pipelineID int64
|
|
for _, p := range items {
|
|
if p.Name == "human" {
|
|
pipelineID = p.ID
|
|
}
|
|
}
|
|
if pipelineID == 0 {
|
|
if _, err := store.Submit(ctx, "human", humanPipelineSource); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ = store.List(ctx)
|
|
for _, p := range items {
|
|
if p.Name == "human" {
|
|
pipelineID = p.ID
|
|
}
|
|
}
|
|
}
|
|
return startRun(t, r, pipelineID, key)
|
|
}
|
|
|
|
func TestHumanStageWaitsAndDecisionResumes(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "human.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer r.wg.Wait()
|
|
r.execute = successExecute
|
|
run := startHumanRun(t, ctx, r, store, "human-go")
|
|
|
|
detail, err := r.getRun(run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Status != protocol.RunStatusWaiting || detail.CurrentStage != "gate" {
|
|
t.Fatalf("run did not wait: %+v", detail.RunSummary)
|
|
}
|
|
if len(detail.Steps) != 1 || detail.Steps[0].Status != protocol.StepStatusWaiting {
|
|
t.Fatalf("waiting step not persisted: %+v", detail.Steps)
|
|
}
|
|
if prompt := protocol.WaitingPrompt(detail.Steps[0].Output); !strings.Contains(prompt, "Review the screenshots") {
|
|
t.Fatalf("waiting step omitted its prompt: %+v", detail.Steps[0].Output)
|
|
}
|
|
if options := protocol.WaitingOptions(detail.Steps[0].Output); strings.Join(options, ",") != "go,halt" {
|
|
t.Fatalf("waiting step omitted its options: %+v", options)
|
|
}
|
|
if detail.Steps[0].FinishedAt != "" {
|
|
t.Fatalf("waiting step must not be finished: %+v", detail.Steps[0])
|
|
}
|
|
|
|
// No later step is created while waiting.
|
|
if len(detail.Steps) != 1 {
|
|
t.Fatalf("a later step ran while waiting: %+v", detail.Steps)
|
|
}
|
|
|
|
// Unknown option is refused with the valid list.
|
|
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "launch"}); err == nil || !strings.Contains(err.Error(), "valid options") || !strings.Contains(err.Error(), "go") {
|
|
t.Fatalf("unknown option not refused with the list: %v", err)
|
|
}
|
|
|
|
decided, err := r.decide(protocol.Message{RunID: run.ID, Option: "go", Feedback: "looks good"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if decided.Status != "running" {
|
|
t.Fatalf("decided run did not resume: %+v", decided.RunSummary)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err = store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Status != "succeeded" || len(detail.Steps) != 2 {
|
|
t.Fatalf("run did not resume to completion: %+v", detail.RunSummary)
|
|
}
|
|
gate := detail.Steps[0]
|
|
if gate.Status != "completed" || gate.Outcome != "go" || gate.Target != "worker" || gate.Diagnostic != "" {
|
|
t.Fatalf("decision evidence not recorded: %+v", gate)
|
|
}
|
|
if gate.Output["option"] != "go" || gate.Output["feedback"] != "looks good" {
|
|
t.Fatalf("decision output missing option/feedback: %+v", gate.Output)
|
|
}
|
|
if gate.Verdict == nil || gate.Verdict.Option != "go → worker" || gate.Verdict.Reason != "looks good" {
|
|
t.Fatalf("derived verdict missing the decision: %+v", gate.Verdict)
|
|
}
|
|
|
|
// A repeated decision for the already-decided visit must be an idempotent
|
|
// no-op (applied=false, no error) and must not create another step. Naming
|
|
// the stage explicitly keeps this on the decided visit instead of relying on
|
|
// current_stage having advanced past it.
|
|
repeat, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "go"})
|
|
if err != nil {
|
|
t.Fatalf("repeat decision for a decided visit was refused: %v", err)
|
|
}
|
|
if repeat == nil || repeat.Status != "succeeded" {
|
|
t.Fatalf("repeat decision changed the run: %+v", repeat)
|
|
}
|
|
after, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(after.Steps) != 2 {
|
|
t.Fatalf("repeat decision created another step: %+v", after.Steps)
|
|
}
|
|
}
|
|
|
|
func TestHumanDecisionToTerminalFinalizesRun(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "terminal.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer r.wg.Wait()
|
|
r.execute = successExecute
|
|
run := startHumanRun(t, ctx, r, store, "human-halt")
|
|
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt", Feedback: "approved"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Status != "succeeded" || len(detail.Steps) != 1 || detail.Steps[0].Target != protocol.SuccessEnd {
|
|
t.Fatalf("decision to success did not finalize the run: %+v", detail.RunSummary)
|
|
}
|
|
}
|
|
|
|
func TestHumanStageCancelRetryAndRebuild(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
path := filepath.Join(t.TempDir(), "rebuild.db")
|
|
store, err := pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.execute = successExecute
|
|
run := startHumanRun(t, ctx, r, store, "human-rebuild")
|
|
|
|
// Cancel works while waiting.
|
|
if err := r.cancel(run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Status != "canceled" || detail.Steps[0].Status != "canceled" {
|
|
t.Fatalf("waiting run not canceled: %+v %+v", detail.RunSummary, detail.Steps)
|
|
}
|
|
if err := r.cancel(run.ID); err == nil {
|
|
t.Fatal("canceling a canceled run succeeded")
|
|
}
|
|
if err := store.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Rebuild runtime state from persisted rows: a second Open + newRunner over
|
|
// the same database, with no process restart. RecoverRuns must leave a
|
|
// waiting run waiting.
|
|
store, err = pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
r2, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer r2.wg.Wait()
|
|
r2.execute = successExecute
|
|
waiting := startHumanRun(t, ctx, r2, store, "human-rebuild-2")
|
|
|
|
if err := store.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err = pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r3, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer r3.wg.Wait()
|
|
r3.execute = successExecute
|
|
recovered, err := r3.getRun(waiting.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if recovered.Status != protocol.RunStatusWaiting || recovered.Steps[0].Status != protocol.StepStatusWaiting {
|
|
t.Fatalf("rebuilt runtime lost the wait: %+v %+v", recovered.RunSummary, recovered.Steps)
|
|
}
|
|
if _, err := r3.decide(protocol.Message{RunID: waiting.ID, Option: "go", Feedback: "after restart"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r3.wg.Wait()
|
|
final, err := store.GetRun(ctx, waiting.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if final.Status != "succeeded" || final.Steps[0].Outcome != "go" || final.Steps[0].Output["feedback"] != "after restart" {
|
|
t.Fatalf("rebuilt runtime refused or mis-recorded the decision: %+v", final)
|
|
}
|
|
|
|
// Retry works after a decision that later fails.
|
|
failing := startHumanRun(t, ctx, r3, store, "human-retry")
|
|
r3.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{}, fmt.Errorf("provider failed")
|
|
}
|
|
if _, err := r3.decide(protocol.Message{RunID: failing.ID, Option: "go"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r3.wg.Wait()
|
|
failed, err := store.GetRun(ctx, failing.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if failed.Status != "failed" {
|
|
t.Fatalf("post-decision failure did not fail the run: %+v", failed.RunSummary)
|
|
}
|
|
r3.execute = successExecute
|
|
retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human", RetryMode: "restart"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r3.wg.Wait()
|
|
retriedDetail, err := store.GetRun(ctx, retried.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if retried.ID == failing.ID || retriedDetail.Status != protocol.RunStatusWaiting {
|
|
t.Fatalf("retry after a decision did not start a fresh waiting run: %+v", retriedDetail.RunSummary)
|
|
}
|
|
}
|
|
|
|
const humanOnlySource = "# stage gate\n\n```output\nDecision evidence.\n```\n\n```prompt\nApprove the release.\n```\n\n```edges\ngo -> #success\nhalt -> #failure\nerror -> #failure\n```\n\n```configs\ntype=human\n```\n\n"
|
|
|
|
func TestHumanRunWebSocketWaitAndDecide(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "ws.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "human-ws", humanOnlySource); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
handler := Handler(ctx, store).(*runHandler)
|
|
defer func() { cancel(); WaitRuns(handler) }()
|
|
srv := httptest.NewServer(handler)
|
|
defer srv.Close()
|
|
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer conn.CloseNow()
|
|
var snapshot protocol.Message
|
|
if err := wsjson.Read(ctx, conn, &snapshot); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
request := func(message protocol.Message) protocol.Message {
|
|
t.Helper()
|
|
if err := wsjson.Write(ctx, conn, message); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var response protocol.Message
|
|
if err := wsjson.Read(ctx, conn, &response); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return response
|
|
}
|
|
started := request(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: items[0].ID, Version: 1, RequestKey: "ws-human", Input: map[string]any{"prompt": "ship it"}})
|
|
if started.Type != protocol.RunStarted || started.RunDetail == nil {
|
|
t.Fatalf("run not started: %+v", started)
|
|
}
|
|
runID := started.RunDetail.ID
|
|
|
|
var waiting protocol.Message
|
|
for i := 0; i < 50; i++ {
|
|
waiting = request(protocol.Message{Type: protocol.GetRun, ID: "poll", RunID: runID})
|
|
if waiting.RunDetail != nil && waiting.RunDetail.Status == protocol.RunStatusWaiting {
|
|
break
|
|
}
|
|
if waiting.Type == protocol.Error {
|
|
t.Fatal(waiting.Error)
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
if waiting.RunDetail == nil || waiting.RunDetail.Status != protocol.RunStatusWaiting || waiting.RunDetail.CurrentStage != "gate" {
|
|
t.Fatalf("run never reached waiting: %+v", waiting.RunDetail)
|
|
}
|
|
if len(waiting.RunDetail.Steps) != 1 || waiting.RunDetail.Steps[0].Status != protocol.StepStatusWaiting {
|
|
t.Fatalf("waiting step not visible over websocket: %+v", waiting.RunDetail.Steps)
|
|
}
|
|
if prompt := protocol.WaitingPrompt(waiting.RunDetail.Steps[0].Output); !strings.Contains(prompt, "Approve the release") {
|
|
t.Fatalf("prompt not visible over websocket: %+v", waiting.RunDetail.Steps[0].Output)
|
|
}
|
|
if options := protocol.WaitingOptions(waiting.RunDetail.Steps[0].Output); strings.Join(options, ",") != "go,halt" {
|
|
t.Fatalf("options not visible over websocket: %+v", options)
|
|
}
|
|
|
|
// Unknown option is refused verbatim over the wire.
|
|
bad := request(protocol.Message{Type: protocol.DecideRun, ID: "bad", RunID: runID, Option: "launch"})
|
|
if bad.Type != protocol.Error || !strings.Contains(bad.Error, "valid options") {
|
|
t.Fatalf("unknown option not refused over websocket: %+v", bad)
|
|
}
|
|
|
|
decided := request(protocol.Message{Type: protocol.DecideRun, ID: "decide", RunID: runID, Option: "go", Feedback: "approved"})
|
|
if decided.Type != protocol.Run || decided.RunDetail == nil {
|
|
t.Fatalf("decision not accepted over websocket: %+v", decided)
|
|
}
|
|
var final protocol.Message
|
|
for i := 0; i < 50; i++ {
|
|
final = request(protocol.Message{Type: protocol.GetRun, ID: "final", RunID: runID})
|
|
if final.RunDetail != nil && final.RunDetail.Status != protocol.RunStatusWaiting && final.RunDetail.Status != "running" {
|
|
break
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
if final.RunDetail == nil || final.RunDetail.Status != "succeeded" {
|
|
t.Fatalf("decided run did not complete over websocket: %+v", final.RunDetail)
|
|
}
|
|
if final.RunDetail.Steps[0].Outcome != "go" || final.RunDetail.Steps[0].Output["feedback"] != "approved" {
|
|
t.Fatalf("decision evidence not persisted over websocket: %+v", final.RunDetail.Steps[0])
|
|
}
|
|
}
|
|
|
|
func TestHumanStageWaitingNotification(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "notify.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
var calls atomic.Int32
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
filename, _ := new(mime.WordDecoder).DecodeHeader(r.Header.Get("Filename"))
|
|
ack := map[string]any{"id": "msg1", "event": "message"}
|
|
if filename != "" {
|
|
ack["attachment"] = map[string]string{"name": filename, "url": "https://ntfy.example.com/file/" + filename}
|
|
}
|
|
_ = json.NewEncoder(w).Encode(ack)
|
|
}))
|
|
defer srv.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
|
|
|
|
source := strings.Replace(humanPipelineSource, "```configs\ntype=human\n```", "```notify\non=waiting\n```\n\n```configs\ntype=human\n```", 1)
|
|
if _, err := store.Submit(ctx, "human-notify", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
var pipelineID int64
|
|
for _, p := range items {
|
|
if p.Name == "human-notify" {
|
|
pipelineID = p.ID
|
|
}
|
|
}
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer r.wg.Wait()
|
|
run, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "human-notify", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
if calls.Load() != 1 {
|
|
t.Fatalf("waiting notification fired %d times", calls.Load())
|
|
}
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(detail.Steps[0].Notifications) != 1 || detail.Steps[0].Notifications[0].Status != "sent" {
|
|
t.Fatalf("waiting notification receipt missing: %+v", detail.Steps[0].Notifications)
|
|
}
|
|
}
|