Files
pipelines/internal/server/run_notification_outbox_test.go
T
alan b109ab3e06 Notify on run end only for runs marked Notify me
Auto runs no longer alert on failure or success; only notify_mode on does,
unless the run_finished setting (now 'Notify when any run ends', default off)
opts into every run end. Waiting still notifies auto and on runs. The setting
is stored under a new key so a value saved under the old meaning is ignored.
2026-10-31 16:52:04 +00:00

417 lines
15 KiB
Go

package server
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"git.a-wan.net/alan/pipelines/internal/summarise"
)
func TestRunNotificationPublisherFailureRetriesAfterReopen(t *testing.T) {
t.Setenv("PIPELINES_WEB_PUBLIC_URL", "https://pipelines.example.com")
ctx := context.Background()
path := filepath.Join(t.TempDir(), "retry.db")
s, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got, actions := r.Header.Get("Click"), r.Header.Get("Actions"); got != "pipelines://run/7" || actions != "view, Open in browser, https://pipelines.example.com/runs/7" {
t.Errorf("wrong retry links: %q %q", got, actions)
}
if calls.Add(1) == 1 {
http.Error(w, "transient", http.StatusServiceUnavailable)
return
}
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r := &runner{ctx: ctx, store: s}
// The run was started with "Notify me", so its failure is announced.
r.setNotifyMode(7, "on")
// Emulate a durable transition intent saved before the runner callback.
if err := s.EnqueueRunNotification(ctx, pipeline.RunNotification{RunID: 7, Sequence: 1, Event: "failed", Channel: "system", Title: "Run failed", Body: "Run 7", Click: "pipelines://run/7"}); err != nil {
t.Fatal(err)
}
r.notifyRun(7, "test", "failed", 1, "stage", "")
got, err := s.RunNotifications(ctx, 7)
if err != nil || len(got) != 1 || got[0].Status != "pending" || got[0].Attempts != 1 || got[0].Diagnostic == "" {
t.Fatalf("failed attempt not audited: %+v %v", got, err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
r = &runner{ctx: ctx, store: s}
r.setNotifyMode(7, "on")
r.deliverRunNotifications(time.Now().Add(time.Minute))
r.notifyRun(7, "test", "failed", 1, "stage", "")
got, err = s.RunNotifications(ctx, 7)
if err != nil || len(got) != 1 || got[0].Status != "sent" || got[0].Attempts != 2 || got[0].MessageID != "ack" || calls.Load() != 2 {
t.Fatalf("retry/dedupe: %+v calls=%d %v", got, calls.Load(), err)
}
}
func TestTransactionalSuccessNotificationUsesWebLink(t *testing.T) {
ctx := context.Background()
s, err := pipeline.Open(filepath.Join(t.TempDir(), "web-success.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
if _, err := s.Submit(ctx, "test", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
d, err := s.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
id, _, err := s.CreateRunTitled(ctx, d, "web-success", t.TempDir(), false, nil, nil, pipeline.RunOverrides{NotifyMode: "on"}, summarise.Title{})
if err != nil {
t.Fatal(err)
}
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
t.Fatal(err)
}
var calls atomic.Int32
endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
calls.Add(1)
if got, want := req.Header.Get("Click"), "pipelines://run/"+fmt.Sprint(id); got != want {
t.Errorf("click = %q, want %q", got, want)
}
if got, want := req.Header.Get("Actions"), "view, Open in browser, https://pipelines.example.com/runs/"+fmt.Sprint(id); got != want {
t.Errorf("actions = %q, want %q", got, want)
}
_ = json.NewEncoder(w).Encode(map[string]string{"id": "web-success", "event": "message"})
}))
defer endpoint.Close()
t.Setenv("PIPELINES_WEB_PUBLIC_URL", "https://pipelines.example.com")
t.Setenv("PIPELINES_NTFY_URL", endpoint.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r := &runner{ctx: ctx, store: s}
r.notifyRun(id, "test", "succeeded", 0, "", "")
r.deliverRunNotifications(time.Now())
got, err := s.RunNotifications(ctx, id)
if err != nil || len(got) != 1 || got[0].Status != "sent" || calls.Load() != 1 {
t.Fatalf("notification: %+v calls=%d err=%v", got, calls.Load(), err)
}
}
func TestFinishRunPersistenceFailureDoesNotAnnounceSuccess(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
s, err := pipeline.Open(filepath.Join(t.TempDir(), "not-finished.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
source := strings.Replace(pipeline.Template([]string{"one"}, true), "# stage one", "# stage one\n```notify\non=success\n```", 1)
if _, err := s.Submit(ctx, "test", source); err != nil {
t.Fatal(err)
}
items, err := s.List(ctx)
if err != nil {
t.Fatal(err)
}
d, err := s.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
id, _, err := s.CreateRunTitled(ctx, d, "not-finished", "workspace", false, nil, nil, pipeline.RunOverrides{NotifyMode: "on"}, summarise.Title{})
if err != nil {
t.Fatal(err)
}
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
_ = json.NewEncoder(w).Encode(map[string]string{"id": "wrong-success", "event": "message"})
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r := &runner{ctx: ctx, store: s, events: newRunEventHub()}
if err := s.BeginStep(ctx, id, 1, "one", nil); err != nil {
t.Fatal(err)
}
state := &runState{memory: map[string]memoryValue{}}
if _, _, _, _, err := r.finishStage(id, "test", "workspace", d.Graph.Stages[0], state, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: protocol.SuccessEnd, Output: map[string]any{"markdown": "result"}}); err != nil {
t.Fatal(err)
}
if calls.Load() != 0 {
t.Fatal("terminal stage success published before FinishRun")
}
sub := r.events.subscribe("test", "workspace", nil, func(protocol.Message) error { return nil })
defer sub.close()
// Marshal failure occurs before FinishRun can commit its proposed success.
r.finishRunState(id, "test", "workspace", "succeeded", "success", "", 0, 0, "", stage.Values{"invalid": make(chan int)})
got, err := s.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
if got.Status != "running" || len(got.Notifications) != 0 || calls.Load() != 0 {
t.Fatalf("unpersisted success announced: %+v calls=%d", got, calls.Load())
}
sub.mu.Lock()
defer sub.mu.Unlock()
if len(sub.pending) != 0 {
t.Fatalf("unpersisted terminal event emitted: %+v", sub.pending)
}
}
func TestWaitingHookSharesTransactionalRunIntent(t *testing.T) {
for _, failFirst := range []bool{false, true} {
t.Run(map[bool]string{false: "covered", true: "retry"}[failFirst], func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
s, err := pipeline.Open(filepath.Join(t.TempDir(), "waiting-hook.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if calls.Add(1) == 1 && failFirst {
http.Error(w, "transient", http.StatusServiceUnavailable)
return
}
_ = json.NewEncoder(w).Encode(map[string]string{"id": "waiting-ack", "event": "message"})
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
source := strings.Replace(humanPipelineSource, "# stage gate", "# stage gate\n```notify\non=waiting\n```", 1)
if _, err := s.Submit(ctx, "waiting", source); err != nil {
t.Fatal(err)
}
items, err := s.List(ctx)
if err != nil {
t.Fatal(err)
}
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.schedulerWG.Wait() }()
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "waiting-hook", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
r.deliverRunNotifications(time.Now().Add(time.Minute))
detail, err := s.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
wantCalls := int32(1)
if failFirst {
wantCalls = 2
}
if calls.Load() != wantCalls || detail.Status != "waiting" || len(detail.Notifications) != 1 {
t.Fatalf("waiting hook duplicated/lost: calls=%d detail=%+v", calls.Load(), detail)
}
status := detail.Notifications[0].Status
if status != "sent" && status != "covered" {
t.Fatalf("missing acknowledgement: %+v", detail.Notifications)
}
})
}
}
func TestRunNotificationModesAndFinalStageDedupe(t *testing.T) {
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
for _, tc := range []struct {
mode string
hook bool
// any turns the global run_finished setting on (it defaults off), so
// every run end notifies, not only runs marked "on".
any bool
want int32
}{
{"on", false, false, 1}, {"auto", false, false, 0}, {"auto", false, true, 1}, {"on", false, true, 1}, {"off", false, false, 0}, {"off", false, true, 0},
// A stage hook is the pipeline's own opt-in: it still publishes for an
// auto run, and never doubles up with the run alert.
{"on", true, false, 1}, {"auto", true, false, 1}, {"auto", true, true, 1}, {"off", true, false, 0},
} {
t.Run(tc.mode+map[bool]string{true: "-hook", false: "-no-hook"}[tc.hook]+map[bool]string{true: "-any", false: ""}[tc.any], func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
s, err := pipeline.Open(filepath.Join(t.TempDir(), "modes.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
if tc.any {
if _, err := s.UpdateNotificationSettings(ctx, map[string]any{protocol.NotifyRunFinished: true}); err != nil {
t.Fatal(err)
}
}
source := pipeline.Template([]string{"one"}, true)
if tc.hook {
source = strings.Replace(source, "# stage one", "# stage one\n```notify\non=success\n```", 1)
}
if _, err := s.Submit(ctx, "mode-test", source); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success"}, nil
}
before := calls.Load()
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mode", NotifyMode: tc.mode, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := s.GetRun(ctx, run.ID)
if err != nil || detail.Status != "succeeded" || detail.NotifyMode != tc.mode || calls.Load()-before != tc.want {
t.Fatalf("mode contract: %+v calls=%d %v", detail, calls.Load()-before, err)
}
if tc.want == 1 && !(tc.hook && tc.mode == "auto" && !tc.any) && (len(detail.Notifications) != 1 || (detail.Notifications[0].Status != "sent" && detail.Notifications[0].Status != "covered")) {
t.Fatalf("missing final audit: %+v", detail.Notifications)
}
cancel()
r.schedulerWG.Wait()
})
}
}
func TestRunNotificationAttentionModesAndRetryBound(t *testing.T) {
ctx := context.Background()
s, err := pipeline.Open(filepath.Join(t.TempDir(), "attention.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
http.Error(w, "unavailable", http.StatusServiceUnavailable)
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r := &runner{ctx: ctx, store: s}
for i, mode := range []string{"auto", "on", "off"} {
id := int64(i + 1)
r.setNotifyMode(id, mode)
for sequence, status := range []string{"waiting", "failed"} {
r.notifyRun(id, "test", status, sequence, "stage", "")
}
}
// auto waiting, on waiting and on failed; an auto run's failure is silent
// because run_finished defaults off.
if calls.Load() != 3 {
t.Fatalf("attention modes: %d calls, want 3", calls.Load())
}
r.deliverRunNotifications(time.Now().Add(time.Minute))
r.deliverRunNotifications(time.Now().Add(2 * time.Minute))
r.deliverRunNotifications(time.Now().Add(3 * time.Minute))
if calls.Load() != 9 {
t.Fatalf("retry bound: %d calls, want 9", calls.Load())
}
for _, id := range []int64{1, 2, 3} {
got, err := s.RunNotifications(ctx, id)
if err != nil {
t.Fatal(err)
}
if id == 3 && len(got) != 0 {
t.Fatalf("off created attempts: %+v", got)
}
for _, n := range got {
if n.Status != "failed" || n.Attempts != 3 {
t.Fatalf("missing exhausted audit: %+v", n)
}
}
}
}
func TestRunNotificationWorkerResumesPendingAfterRestart(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
path := filepath.Join(t.TempDir(), "worker.db")
s, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
if err := s.EnqueueRunNotification(ctx, pipeline.RunNotification{RunID: 7, Sequence: 1, Event: "failed", Channel: "system", Title: "Run failed", Body: "pending"}); err != nil {
t.Fatal(err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
delivered := make(chan struct{}, 2)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]string{"id": "recovered", "event": "message"})
delivered <- struct{}{}
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.schedulerWG.Wait() }()
select {
case <-delivered:
case <-time.After(5 * time.Second):
t.Fatal("pending outbox was not recovered")
}
deadline := time.Now().Add(5 * time.Second)
for {
got, err := s.RunNotifications(ctx, 7)
if err != nil {
t.Fatal(err)
}
if len(got) == 1 && got[0].Status == "sent" && got[0].MessageID == "recovered" {
break
}
if time.Now().After(deadline) {
t.Fatalf("recovery audit: %+v", got)
}
time.Sleep(10 * time.Millisecond)
}
}