package server import ( "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "path/filepath" "strings" "sync/atomic" "testing" "time" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" "git.a-wan.net/alan/pipelines/internal/summarise" ) func TestRunNotificationPublisherFailureRetriesAfterReopen(t *testing.T) { t.Setenv("PIPELINES_WEB_PUBLIC_URL", "https://pipelines.example.com") ctx := context.Background() path := filepath.Join(t.TempDir(), "retry.db") s, err := pipeline.Open(path) if err != nil { t.Fatal(err) } var calls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if got := r.Header.Get("Click"); got != "https://pipelines.example.com/runs/7" { t.Errorf("wrong web retry link: %q", got) } if calls.Add(1) == 1 { http.Error(w, "transient", http.StatusServiceUnavailable) return } _ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"}) })) defer server.Close() t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake") t.Setenv("PIPELINES_NTFY_TOKEN", "") t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "") r := &runner{ctx: ctx, store: s} // Emulate a durable transition intent saved before the runner callback. if err := s.EnqueueRunNotification(ctx, pipeline.RunNotification{RunID: 7, Sequence: 1, Event: "failed", Channel: "system", Title: "Run failed", Body: "Run 7", Click: "pipelines://run/7"}); err != nil { t.Fatal(err) } r.notifyRun(7, "test", "failed", 1, "stage", "") got, err := s.RunNotifications(ctx, 7) if err != nil || len(got) != 1 || got[0].Status != "pending" || got[0].Attempts != 1 || got[0].Diagnostic == "" { t.Fatalf("failed attempt not audited: %+v %v", got, err) } if err := s.Close(); err != nil { t.Fatal(err) } s, err = pipeline.Open(path) if err != nil { t.Fatal(err) } defer s.Close() r = &runner{ctx: ctx, store: s} r.deliverRunNotifications(time.Now().Add(time.Minute)) r.notifyRun(7, "test", "failed", 1, "stage", "") got, err = s.RunNotifications(ctx, 7) if err != nil || len(got) != 1 || got[0].Status != "sent" || got[0].Attempts != 2 || got[0].MessageID != "ack" || calls.Load() != 2 { t.Fatalf("retry/dedupe: %+v calls=%d %v", got, calls.Load(), err) } } func TestTransactionalSuccessNotificationUsesWebLink(t *testing.T) { ctx := context.Background() s, err := pipeline.Open(filepath.Join(t.TempDir(), "web-success.db")) if err != nil { t.Fatal(err) } defer s.Close() if _, err := s.Submit(ctx, "test", pipeline.Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := s.List(ctx) d, err := s.Get(ctx, items[0].ID, 1) if err != nil { t.Fatal(err) } id, _, err := s.CreateRunTitled(ctx, d, "web-success", t.TempDir(), false, nil, nil, pipeline.RunOverrides{NotifyMode: "on"}, summarise.Title{}) if err != nil { t.Fatal(err) } if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil { t.Fatal(err) } var calls atomic.Int32 endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { calls.Add(1) if got, want := req.Header.Get("Click"), "https://pipelines.example.com/runs/"+fmt.Sprint(id); got != want { t.Errorf("click = %q, want %q", got, want) } _ = json.NewEncoder(w).Encode(map[string]string{"id": "web-success", "event": "message"}) })) defer endpoint.Close() t.Setenv("PIPELINES_WEB_PUBLIC_URL", "https://pipelines.example.com") t.Setenv("PIPELINES_NTFY_URL", endpoint.URL+"/fake") t.Setenv("PIPELINES_NTFY_TOKEN", "") t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "") r := &runner{ctx: ctx, store: s} r.notifyRun(id, "test", "succeeded", 0, "", "") r.deliverRunNotifications(time.Now()) got, err := s.RunNotifications(ctx, id) if err != nil || len(got) != 1 || got[0].Status != "sent" || calls.Load() != 1 { t.Fatalf("notification: %+v calls=%d err=%v", got, calls.Load(), err) } } func TestFinishRunPersistenceFailureDoesNotAnnounceSuccess(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() s, err := pipeline.Open(filepath.Join(t.TempDir(), "not-finished.db")) if err != nil { t.Fatal(err) } defer s.Close() source := strings.Replace(pipeline.Template([]string{"one"}, true), "# stage one", "# stage one\n```notify\non=success\n```", 1) if _, err := s.Submit(ctx, "test", source); err != nil { t.Fatal(err) } items, err := s.List(ctx) if err != nil { t.Fatal(err) } d, err := s.Get(ctx, items[0].ID, 1) if err != nil { t.Fatal(err) } id, _, err := s.CreateRunTitled(ctx, d, "not-finished", "workspace", false, nil, nil, pipeline.RunOverrides{NotifyMode: "on"}, summarise.Title{}) if err != nil { t.Fatal(err) } var calls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1) _ = json.NewEncoder(w).Encode(map[string]string{"id": "wrong-success", "event": "message"}) })) defer server.Close() t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake") t.Setenv("PIPELINES_NTFY_TOKEN", "") t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "") r := &runner{ctx: ctx, store: s, events: newRunEventHub()} if err := s.BeginStep(ctx, id, 1, "one", nil); err != nil { t.Fatal(err) } state := &runState{memory: map[string]memoryValue{}} if _, _, _, _, err := r.finishStage(id, "test", "workspace", d.Graph.Stages[0], state, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: protocol.SuccessEnd, Output: map[string]any{"markdown": "result"}}); err != nil { t.Fatal(err) } if calls.Load() != 0 { t.Fatal("terminal stage success published before FinishRun") } sub := r.events.subscribe("test", "workspace", nil, func(protocol.Message) error { return nil }) defer sub.close() // Marshal failure occurs before FinishRun can commit its proposed success. r.finishRunState(id, "test", "workspace", "succeeded", "success", "", 0, 0, "", stage.Values{"invalid": make(chan int)}) got, err := s.GetRun(ctx, id) if err != nil { t.Fatal(err) } if got.Status != "running" || len(got.Notifications) != 0 || calls.Load() != 0 { t.Fatalf("unpersisted success announced: %+v calls=%d", got, calls.Load()) } sub.mu.Lock() defer sub.mu.Unlock() if len(sub.pending) != 0 { t.Fatalf("unpersisted terminal event emitted: %+v", sub.pending) } } func TestWaitingHookSharesTransactionalRunIntent(t *testing.T) { for _, failFirst := range []bool{false, true} { t.Run(map[bool]string{false: "covered", true: "retry"}[failFirst], func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() s, err := pipeline.Open(filepath.Join(t.TempDir(), "waiting-hook.db")) if err != nil { t.Fatal(err) } defer s.Close() var calls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if calls.Add(1) == 1 && failFirst { http.Error(w, "transient", http.StatusServiceUnavailable) return } _ = json.NewEncoder(w).Encode(map[string]string{"id": "waiting-ack", "event": "message"}) })) defer server.Close() t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake") t.Setenv("PIPELINES_NTFY_TOKEN", "") t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "") source := strings.Replace(humanPipelineSource, "# stage gate", "# stage gate\n```notify\non=waiting\n```", 1) if _, err := s.Submit(ctx, "waiting", source); err != nil { t.Fatal(err) } items, err := s.List(ctx) if err != nil { t.Fatal(err) } r, err := newRunner(ctx, s) if err != nil { t.Fatal(err) } defer func() { cancel(); r.schedulerWG.Wait() }() run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "waiting-hook", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } r.wg.Wait() r.deliverRunNotifications(time.Now().Add(time.Minute)) detail, err := s.GetRun(ctx, run.ID) if err != nil { t.Fatal(err) } wantCalls := int32(1) if failFirst { wantCalls = 2 } if calls.Load() != wantCalls || detail.Status != "waiting" || len(detail.Notifications) != 1 { t.Fatalf("waiting hook duplicated/lost: calls=%d detail=%+v", calls.Load(), detail) } status := detail.Notifications[0].Status if status != "sent" && status != "covered" { t.Fatalf("missing acknowledgement: %+v", detail.Notifications) } }) } } func TestRunNotificationModesAndFinalStageDedupe(t *testing.T) { var calls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1) _ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"}) })) defer server.Close() t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake") t.Setenv("PIPELINES_NTFY_TOKEN", "") t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "") for _, tc := range []struct { mode string hook bool want int32 }{ {"on", false, 1}, {"auto", false, 0}, {"off", false, 0}, {"on", true, 1}, {"auto", true, 1}, {"off", true, 0}, } { t.Run(tc.mode+map[bool]string{true: "-hook", false: "-no-hook"}[tc.hook], func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() s, err := pipeline.Open(filepath.Join(t.TempDir(), "modes.db")) if err != nil { t.Fatal(err) } defer s.Close() source := pipeline.Template([]string{"one"}, true) if tc.hook { source = strings.Replace(source, "# stage one", "# stage one\n```notify\non=success\n```", 1) } if _, err := s.Submit(ctx, "mode-test", source); err != nil { t.Fatal(err) } items, _ := s.List(ctx) r, err := newRunner(ctx, s) if err != nil { t.Fatal(err) } r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success"}, nil } before := calls.Load() run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mode", NotifyMode: tc.mode, Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } r.wg.Wait() detail, err := s.GetRun(ctx, run.ID) if err != nil || detail.Status != "succeeded" || detail.NotifyMode != tc.mode || calls.Load()-before != tc.want { t.Fatalf("mode contract: %+v calls=%d %v", detail, calls.Load()-before, err) } if tc.mode == "on" && (len(detail.Notifications) != 1 || (detail.Notifications[0].Status != "sent" && detail.Notifications[0].Status != "covered")) { t.Fatalf("missing final audit: %+v", detail.Notifications) } cancel() r.schedulerWG.Wait() }) } } func TestRunNotificationAttentionModesAndRetryBound(t *testing.T) { ctx := context.Background() s, err := pipeline.Open(filepath.Join(t.TempDir(), "attention.db")) if err != nil { t.Fatal(err) } defer s.Close() var calls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1) http.Error(w, "unavailable", http.StatusServiceUnavailable) })) defer server.Close() t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake") t.Setenv("PIPELINES_NTFY_TOKEN", "") t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "") r := &runner{ctx: ctx, store: s} for i, mode := range []string{"auto", "on", "off"} { id := int64(i + 1) r.setNotifyMode(id, mode) for sequence, status := range []string{"waiting", "failed"} { r.notifyRun(id, "test", status, sequence, "stage", "") } } if calls.Load() != 4 { t.Fatalf("attention modes: %d calls, want 4", calls.Load()) } r.deliverRunNotifications(time.Now().Add(time.Minute)) r.deliverRunNotifications(time.Now().Add(2 * time.Minute)) r.deliverRunNotifications(time.Now().Add(3 * time.Minute)) if calls.Load() != 12 { t.Fatalf("retry bound: %d calls, want 12", calls.Load()) } for _, id := range []int64{1, 2, 3} { got, err := s.RunNotifications(ctx, id) if err != nil { t.Fatal(err) } if id == 3 && len(got) != 0 { t.Fatalf("off created attempts: %+v", got) } for _, n := range got { if n.Status != "failed" || n.Attempts != 3 { t.Fatalf("missing exhausted audit: %+v", n) } } } } func TestRunNotificationWorkerResumesPendingAfterRestart(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() path := filepath.Join(t.TempDir(), "worker.db") s, err := pipeline.Open(path) if err != nil { t.Fatal(err) } if err := s.EnqueueRunNotification(ctx, pipeline.RunNotification{RunID: 7, Sequence: 1, Event: "failed", Channel: "system", Title: "Run failed", Body: "pending"}); err != nil { t.Fatal(err) } if err := s.Close(); err != nil { t.Fatal(err) } s, err = pipeline.Open(path) if err != nil { t.Fatal(err) } defer s.Close() delivered := make(chan struct{}, 2) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _ = json.NewEncoder(w).Encode(map[string]string{"id": "recovered", "event": "message"}) delivered <- struct{}{} })) defer server.Close() t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake") t.Setenv("PIPELINES_NTFY_TOKEN", "") t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "") r, err := newRunner(ctx, s) if err != nil { t.Fatal(err) } defer func() { cancel(); r.schedulerWG.Wait() }() select { case <-delivered: case <-time.After(5 * time.Second): t.Fatal("pending outbox was not recovered") } deadline := time.Now().Add(5 * time.Second) for { got, err := s.RunNotifications(ctx, 7) if err != nil { t.Fatal(err) } if len(got) == 1 && got[0].Status == "sent" && got[0].MessageID == "recovered" { break } if time.Now().After(deadline) { t.Fatalf("recovery audit: %+v", got) } time.Sleep(10 * time.Millisecond) } }