3bfb3d6b44
A run stays in r.active after finishRunState persists its terminal status until its goroutine finishes notifying, so a decision in that window was refused as "already active". decide now refuses only runs whose terminal state is not yet durable; a finished run gets ResolveDecision's already-decided / not-waiting answer, and an executing run is still refused. Tests overwrote runner.artifactsRoot after Handler/newRunner had started background goroutines (sizeLoop, session export jobs) that read it. The root is now fixed at construction through newRunnerAt, which HandlerWithOptions uses for Options.ArtifactsRoot, and tests configure it there instead of mutating the field afterwards.
494 lines
17 KiB
Go
494 lines
17 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"mime"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"github.com/coder/websocket"
|
|
"github.com/coder/websocket/wsjson"
|
|
)
|
|
|
|
const humanPipelineSource = "# stage gate\n\n```output\nDecision evidence.\n```\n\n```prompt\nReview the screenshots and choose where the run goes.\n```\n\n```edges\ngo -> worker\nhalt -> #success\nerror -> #failure\n```\n\n```configs\ntype=human\n```\n\n# stage worker\n\n```output\nWork result.\n```\n\n```prompt\nDo the work.\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=2\ntimeout=1h\n```\n\n"
|
|
|
|
func successExecute(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
|
|
}
|
|
|
|
// startHumanRun submits the human pipeline (once) and starts one run, returning
|
|
// once the run goroutine has parked the run in the waiting state.
|
|
func startHumanRun(t *testing.T, ctx context.Context, r *runner, store *pipeline.Store, key string) *protocol.RunDetail {
|
|
t.Helper()
|
|
items, _ := store.List(ctx)
|
|
var pipelineID int64
|
|
for _, p := range items {
|
|
if p.Name == "human" {
|
|
pipelineID = p.ID
|
|
}
|
|
}
|
|
if pipelineID == 0 {
|
|
if _, err := store.Submit(ctx, "human", humanPipelineSource); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ = store.List(ctx)
|
|
for _, p := range items {
|
|
if p.Name == "human" {
|
|
pipelineID = p.ID
|
|
}
|
|
}
|
|
}
|
|
return startRun(t, r, pipelineID, key)
|
|
}
|
|
|
|
func TestHumanStageWaitsAndDecisionResumes(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "human.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer r.wg.Wait()
|
|
r.execute = successExecute
|
|
run := startHumanRun(t, ctx, r, store, "human-go")
|
|
|
|
detail, err := r.getRun(run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Status != protocol.RunStatusWaiting || detail.CurrentStage != "gate" {
|
|
t.Fatalf("run did not wait: %+v", detail.RunSummary)
|
|
}
|
|
if len(detail.Steps) != 1 || detail.Steps[0].Status != protocol.StepStatusWaiting {
|
|
t.Fatalf("waiting step not persisted: %+v", detail.Steps)
|
|
}
|
|
if prompt := protocol.WaitingPrompt(detail.Steps[0].Output); !strings.Contains(prompt, "Review the screenshots") {
|
|
t.Fatalf("waiting step omitted its prompt: %+v", detail.Steps[0].Output)
|
|
}
|
|
if options := protocol.WaitingOptions(detail.Steps[0].Output); strings.Join(options, ",") != "go,halt" {
|
|
t.Fatalf("waiting step omitted its options: %+v", options)
|
|
}
|
|
if detail.Steps[0].FinishedAt != "" {
|
|
t.Fatalf("waiting step must not be finished: %+v", detail.Steps[0])
|
|
}
|
|
|
|
// No later step is created while waiting.
|
|
if len(detail.Steps) != 1 {
|
|
t.Fatalf("a later step ran while waiting: %+v", detail.Steps)
|
|
}
|
|
|
|
// Unknown option is refused with the valid list.
|
|
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "launch"}); err == nil || !strings.Contains(err.Error(), "valid options") || !strings.Contains(err.Error(), "go") {
|
|
t.Fatalf("unknown option not refused with the list: %v", err)
|
|
}
|
|
|
|
decided, err := r.decide(protocol.Message{RunID: run.ID, Option: "go", Feedback: "looks good"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if decided.Status != "running" {
|
|
t.Fatalf("decided run did not resume: %+v", decided.RunSummary)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err = store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Status != "succeeded" || len(detail.Steps) != 2 {
|
|
t.Fatalf("run did not resume to completion: %+v", detail.RunSummary)
|
|
}
|
|
gate := detail.Steps[0]
|
|
if gate.Status != "completed" || gate.Outcome != "go" || gate.Target != "worker" || gate.Diagnostic != "" {
|
|
t.Fatalf("decision evidence not recorded: %+v", gate)
|
|
}
|
|
if gate.Output["option"] != "go" || gate.Output["feedback"] != "looks good" {
|
|
t.Fatalf("decision output missing option/feedback: %+v", gate.Output)
|
|
}
|
|
if gate.Verdict == nil || gate.Verdict.Option != "go → worker" || gate.Verdict.Reason != "looks good" {
|
|
t.Fatalf("derived verdict missing the decision: %+v", gate.Verdict)
|
|
}
|
|
|
|
// A repeated decision for the already-decided visit must be an idempotent
|
|
// no-op (applied=false, no error) and must not create another step. Naming
|
|
// the stage explicitly keeps this on the decided visit instead of relying on
|
|
// current_stage having advanced past it.
|
|
repeat, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "go"})
|
|
if err != nil {
|
|
t.Fatalf("repeat decision for a decided visit was refused: %v", err)
|
|
}
|
|
if repeat == nil || repeat.Status != "succeeded" {
|
|
t.Fatalf("repeat decision changed the run: %+v", repeat)
|
|
}
|
|
after, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(after.Steps) != 2 {
|
|
t.Fatalf("repeat decision created another step: %+v", after.Steps)
|
|
}
|
|
}
|
|
|
|
func TestHumanDecisionToTerminalFinalizesRun(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "terminal.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer r.wg.Wait()
|
|
r.execute = successExecute
|
|
run := startHumanRun(t, ctx, r, store, "human-halt")
|
|
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt", Feedback: "approved"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Status != "succeeded" || len(detail.Steps) != 1 || detail.Steps[0].Target != protocol.SuccessEnd {
|
|
t.Fatalf("decision to success did not finalize the run: %+v", detail.RunSummary)
|
|
}
|
|
}
|
|
|
|
func TestHumanStageCancelRetryAndRebuild(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
path := filepath.Join(t.TempDir(), "rebuild.db")
|
|
store, err := pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.execute = successExecute
|
|
run := startHumanRun(t, ctx, r, store, "human-rebuild")
|
|
|
|
// Cancel works while waiting.
|
|
if err := r.cancel(run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Status != "canceled" || detail.Steps[0].Status != "canceled" {
|
|
t.Fatalf("waiting run not canceled: %+v %+v", detail.RunSummary, detail.Steps)
|
|
}
|
|
if err := r.cancel(run.ID); err == nil {
|
|
t.Fatal("canceling a canceled run succeeded")
|
|
}
|
|
if err := store.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Rebuild runtime state from persisted rows: a second Open + newRunner over
|
|
// the same database, with no process restart. RecoverRuns must leave a
|
|
// waiting run waiting.
|
|
store, err = pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
r2, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer r2.wg.Wait()
|
|
r2.execute = successExecute
|
|
waiting := startHumanRun(t, ctx, r2, store, "human-rebuild-2")
|
|
|
|
if err := store.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err = pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r3, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer r3.wg.Wait()
|
|
r3.execute = successExecute
|
|
recovered, err := r3.getRun(waiting.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if recovered.Status != protocol.RunStatusWaiting || recovered.Steps[0].Status != protocol.StepStatusWaiting {
|
|
t.Fatalf("rebuilt runtime lost the wait: %+v %+v", recovered.RunSummary, recovered.Steps)
|
|
}
|
|
if _, err := r3.decide(protocol.Message{RunID: waiting.ID, Option: "go", Feedback: "after restart"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r3.wg.Wait()
|
|
final, err := store.GetRun(ctx, waiting.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if final.Status != "succeeded" || final.Steps[0].Outcome != "go" || final.Steps[0].Output["feedback"] != "after restart" {
|
|
t.Fatalf("rebuilt runtime refused or mis-recorded the decision: %+v", final)
|
|
}
|
|
|
|
// Retry works after a decision that later fails.
|
|
failing := startHumanRun(t, ctx, r3, store, "human-retry")
|
|
r3.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{}, fmt.Errorf("provider failed")
|
|
}
|
|
if _, err := r3.decide(protocol.Message{RunID: failing.ID, Option: "go"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r3.wg.Wait()
|
|
failed, err := store.GetRun(ctx, failing.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if failed.Status != "failed" {
|
|
t.Fatalf("post-decision failure did not fail the run: %+v", failed.RunSummary)
|
|
}
|
|
r3.execute = successExecute
|
|
retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human", RetryMode: "restart"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r3.wg.Wait()
|
|
retriedDetail, err := store.GetRun(ctx, retried.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if retried.ID == failing.ID || retriedDetail.Status != protocol.RunStatusWaiting {
|
|
t.Fatalf("retry after a decision did not start a fresh waiting run: %+v", retriedDetail.RunSummary)
|
|
}
|
|
}
|
|
|
|
const humanOnlySource = "# stage gate\n\n```output\nDecision evidence.\n```\n\n```prompt\nApprove the release.\n```\n\n```edges\ngo -> #success\nhalt -> #failure\nerror -> #failure\n```\n\n```configs\ntype=human\n```\n\n"
|
|
|
|
func TestHumanRunWebSocketWaitAndDecide(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "ws.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "human-ws", humanOnlySource); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
handler := Handler(ctx, store).(*runHandler)
|
|
defer func() { cancel(); WaitRuns(handler) }()
|
|
srv := httptest.NewServer(handler)
|
|
defer srv.Close()
|
|
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer conn.CloseNow()
|
|
var snapshot protocol.Message
|
|
if err := wsjson.Read(ctx, conn, &snapshot); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
request := func(message protocol.Message) protocol.Message {
|
|
t.Helper()
|
|
if err := wsjson.Write(ctx, conn, message); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var response protocol.Message
|
|
if err := wsjson.Read(ctx, conn, &response); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return response
|
|
}
|
|
started := request(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: items[0].ID, Version: 1, RequestKey: "ws-human", Input: map[string]any{"prompt": "ship it"}})
|
|
if started.Type != protocol.RunStarted || started.RunDetail == nil {
|
|
t.Fatalf("run not started: %+v", started)
|
|
}
|
|
runID := started.RunDetail.ID
|
|
|
|
var waiting protocol.Message
|
|
for i := 0; i < 50; i++ {
|
|
waiting = request(protocol.Message{Type: protocol.GetRun, ID: "poll", RunID: runID})
|
|
if waiting.RunDetail != nil && waiting.RunDetail.Status == protocol.RunStatusWaiting {
|
|
break
|
|
}
|
|
if waiting.Type == protocol.Error {
|
|
t.Fatal(waiting.Error)
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
if waiting.RunDetail == nil || waiting.RunDetail.Status != protocol.RunStatusWaiting || waiting.RunDetail.CurrentStage != "gate" {
|
|
t.Fatalf("run never reached waiting: %+v", waiting.RunDetail)
|
|
}
|
|
if len(waiting.RunDetail.Steps) != 1 || waiting.RunDetail.Steps[0].Status != protocol.StepStatusWaiting {
|
|
t.Fatalf("waiting step not visible over websocket: %+v", waiting.RunDetail.Steps)
|
|
}
|
|
if prompt := protocol.WaitingPrompt(waiting.RunDetail.Steps[0].Output); !strings.Contains(prompt, "Approve the release") {
|
|
t.Fatalf("prompt not visible over websocket: %+v", waiting.RunDetail.Steps[0].Output)
|
|
}
|
|
if options := protocol.WaitingOptions(waiting.RunDetail.Steps[0].Output); strings.Join(options, ",") != "go,halt" {
|
|
t.Fatalf("options not visible over websocket: %+v", options)
|
|
}
|
|
|
|
// Unknown option is refused verbatim over the wire.
|
|
bad := request(protocol.Message{Type: protocol.DecideRun, ID: "bad", RunID: runID, Option: "launch"})
|
|
if bad.Type != protocol.Error || !strings.Contains(bad.Error, "valid options") {
|
|
t.Fatalf("unknown option not refused over websocket: %+v", bad)
|
|
}
|
|
|
|
decided := request(protocol.Message{Type: protocol.DecideRun, ID: "decide", RunID: runID, Option: "go", Feedback: "approved"})
|
|
if decided.Type != protocol.Run || decided.RunDetail == nil {
|
|
t.Fatalf("decision not accepted over websocket: %+v", decided)
|
|
}
|
|
var final protocol.Message
|
|
for i := 0; i < 50; i++ {
|
|
final = request(protocol.Message{Type: protocol.GetRun, ID: "final", RunID: runID})
|
|
if final.RunDetail != nil && final.RunDetail.Status != protocol.RunStatusWaiting && final.RunDetail.Status != "running" {
|
|
break
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
if final.RunDetail == nil || final.RunDetail.Status != "succeeded" {
|
|
t.Fatalf("decided run did not complete over websocket: %+v", final.RunDetail)
|
|
}
|
|
if final.RunDetail.Steps[0].Outcome != "go" || final.RunDetail.Steps[0].Output["feedback"] != "approved" {
|
|
t.Fatalf("decision evidence not persisted over websocket: %+v", final.RunDetail.Steps[0])
|
|
}
|
|
}
|
|
|
|
func TestHumanStageWaitingNotification(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "notify.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
var calls atomic.Int32
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
filename, _ := new(mime.WordDecoder).DecodeHeader(r.Header.Get("Filename"))
|
|
ack := map[string]any{"id": "msg1", "event": "message"}
|
|
if filename != "" {
|
|
ack["attachment"] = map[string]string{"name": filename, "url": "https://ntfy.example.com/file/" + filename}
|
|
}
|
|
_ = json.NewEncoder(w).Encode(ack)
|
|
}))
|
|
defer srv.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
|
|
|
|
source := strings.Replace(humanPipelineSource, "```configs\ntype=human\n```", "```notify\non=waiting\n```\n\n```configs\ntype=human\n```", 1)
|
|
if _, err := store.Submit(ctx, "human-notify", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
var pipelineID int64
|
|
for _, p := range items {
|
|
if p.Name == "human-notify" {
|
|
pipelineID = p.ID
|
|
}
|
|
}
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer r.wg.Wait()
|
|
run, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "human-notify", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
if calls.Load() != 1 {
|
|
t.Fatalf("waiting notification fired %d times", calls.Load())
|
|
}
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(detail.Steps[0].Notifications) != 1 || detail.Steps[0].Notifications[0].Status != "sent" {
|
|
t.Fatalf("waiting notification receipt missing: %+v", detail.Steps[0].Notifications)
|
|
}
|
|
}
|
|
|
|
// TestDecideOnFinishingRunIsAnsweredNotRefused pins the window between
|
|
// finishRunState persisting a terminal status and the launch teardown removing
|
|
// the run from r.active: a decision then gets the store's idempotent/not-waiting
|
|
// answer, while a run that is still executing is refused as active.
|
|
func TestDecideOnFinishingRunIsAnsweredNotRefused(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "human.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer r.wg.Wait()
|
|
r.execute = successExecute
|
|
run := startHumanRun(t, ctx, r, store, "human-finishing")
|
|
|
|
// A waiting run that is genuinely executing (registered, not finished) is
|
|
// refused.
|
|
r.mu.Lock()
|
|
r.active[run.ID] = activeRun{cancel: func() {}, workspace: run.Workspace}
|
|
r.mu.Unlock()
|
|
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt"}); err == nil || !strings.Contains(err.Error(), "already active") {
|
|
t.Fatalf("decision on an executing run was not refused: %v", err)
|
|
}
|
|
r.mu.Lock()
|
|
delete(r.active, run.ID)
|
|
r.mu.Unlock()
|
|
|
|
if _, err := r.decide(protocol.Message{RunID: run.ID, Option: "halt", Feedback: "stop"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
if detail, err := store.GetRun(ctx, run.ID); err != nil || detail.Status != "succeeded" {
|
|
t.Fatalf("run did not finish: %+v %v", detail, err)
|
|
}
|
|
|
|
// Simulate the finishing window: terminal status persisted, reservation not
|
|
// yet released.
|
|
r.mu.Lock()
|
|
r.active[run.ID] = activeRun{cancel: func() {}, workspace: run.Workspace}
|
|
r.mu.Unlock()
|
|
repeat, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "halt"})
|
|
if err != nil {
|
|
t.Fatalf("repeat decision on a finished run was refused: %v", err)
|
|
}
|
|
if repeat.Status != "succeeded" || repeat.Steps[0].Output["feedback"] != "stop" {
|
|
t.Fatalf("repeat decision changed the run: %+v", repeat)
|
|
}
|
|
if _, err := r.decide(protocol.Message{RunID: run.ID, Stage: "gate", Option: "go"}); err == nil || !strings.Contains(err.Error(), "not waiting") {
|
|
t.Fatalf("conflicting decision on a finished run did not report not waiting: %v", err)
|
|
}
|
|
r.mu.Lock()
|
|
delete(r.active, run.ID)
|
|
r.mu.Unlock()
|
|
}
|