40ba2ac117
Run notifications now always click through to pipelines://run/<id>; a valid PIPELINES_WEB_PUBLIC_URL adds an "Open in browser" ntfy view action instead of replacing the click. Queued rows derive links from the run id at delivery, so legacy rows holding a web URL still open the app. Unresolved chat questions/forms and permission requests publish one async, bounded notification to the system destination (pipelines://chat/<id> plus the browser action), deduplicated per form/request id across reconnects and replays. stage.PublishNtfy gains optional, strictly validated view actions. Server tests now clear the operator's real ntfy environment.
401 lines
14 KiB
Go
401 lines
14 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"git.a-wan.net/alan/pipelines/internal/summarise"
|
|
)
|
|
|
|
func TestRunNotificationPublisherFailureRetriesAfterReopen(t *testing.T) {
|
|
t.Setenv("PIPELINES_WEB_PUBLIC_URL", "https://pipelines.example.com")
|
|
ctx := context.Background()
|
|
path := filepath.Join(t.TempDir(), "retry.db")
|
|
s, err := pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if got, actions := r.Header.Get("Click"), r.Header.Get("Actions"); got != "pipelines://run/7" || actions != "view, Open in browser, https://pipelines.example.com/runs/7" {
|
|
t.Errorf("wrong retry links: %q %q", got, actions)
|
|
}
|
|
if calls.Add(1) == 1 {
|
|
http.Error(w, "transient", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
r := &runner{ctx: ctx, store: s}
|
|
// Emulate a durable transition intent saved before the runner callback.
|
|
if err := s.EnqueueRunNotification(ctx, pipeline.RunNotification{RunID: 7, Sequence: 1, Event: "failed", Channel: "system", Title: "Run failed", Body: "Run 7", Click: "pipelines://run/7"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.notifyRun(7, "test", "failed", 1, "stage", "")
|
|
got, err := s.RunNotifications(ctx, 7)
|
|
if err != nil || len(got) != 1 || got[0].Status != "pending" || got[0].Attempts != 1 || got[0].Diagnostic == "" {
|
|
t.Fatalf("failed attempt not audited: %+v %v", got, err)
|
|
}
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err = pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
r = &runner{ctx: ctx, store: s}
|
|
r.deliverRunNotifications(time.Now().Add(time.Minute))
|
|
r.notifyRun(7, "test", "failed", 1, "stage", "")
|
|
got, err = s.RunNotifications(ctx, 7)
|
|
if err != nil || len(got) != 1 || got[0].Status != "sent" || got[0].Attempts != 2 || got[0].MessageID != "ack" || calls.Load() != 2 {
|
|
t.Fatalf("retry/dedupe: %+v calls=%d %v", got, calls.Load(), err)
|
|
}
|
|
}
|
|
|
|
func TestTransactionalSuccessNotificationUsesWebLink(t *testing.T) {
|
|
ctx := context.Background()
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "web-success.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
if _, err := s.Submit(ctx, "test", pipeline.Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := s.List(ctx)
|
|
d, err := s.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
id, _, err := s.CreateRunTitled(ctx, d, "web-success", t.TempDir(), false, nil, nil, pipeline.RunOverrides{NotifyMode: "on"}, summarise.Title{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var calls atomic.Int32
|
|
endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
|
calls.Add(1)
|
|
if got, want := req.Header.Get("Click"), "pipelines://run/"+fmt.Sprint(id); got != want {
|
|
t.Errorf("click = %q, want %q", got, want)
|
|
}
|
|
if got, want := req.Header.Get("Actions"), "view, Open in browser, https://pipelines.example.com/runs/"+fmt.Sprint(id); got != want {
|
|
t.Errorf("actions = %q, want %q", got, want)
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "web-success", "event": "message"})
|
|
}))
|
|
defer endpoint.Close()
|
|
t.Setenv("PIPELINES_WEB_PUBLIC_URL", "https://pipelines.example.com")
|
|
t.Setenv("PIPELINES_NTFY_URL", endpoint.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
r := &runner{ctx: ctx, store: s}
|
|
r.notifyRun(id, "test", "succeeded", 0, "", "")
|
|
r.deliverRunNotifications(time.Now())
|
|
got, err := s.RunNotifications(ctx, id)
|
|
if err != nil || len(got) != 1 || got[0].Status != "sent" || calls.Load() != 1 {
|
|
t.Fatalf("notification: %+v calls=%d err=%v", got, calls.Load(), err)
|
|
}
|
|
}
|
|
|
|
func TestFinishRunPersistenceFailureDoesNotAnnounceSuccess(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "not-finished.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
source := strings.Replace(pipeline.Template([]string{"one"}, true), "# stage one", "# stage one\n```notify\non=success\n```", 1)
|
|
if _, err := s.Submit(ctx, "test", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := s.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d, err := s.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
id, _, err := s.CreateRunTitled(ctx, d, "not-finished", "workspace", false, nil, nil, pipeline.RunOverrides{NotifyMode: "on"}, summarise.Title{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "wrong-success", "event": "message"})
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
r := &runner{ctx: ctx, store: s, events: newRunEventHub()}
|
|
if err := s.BeginStep(ctx, id, 1, "one", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
state := &runState{memory: map[string]memoryValue{}}
|
|
if _, _, _, _, err := r.finishStage(id, "test", "workspace", d.Graph.Stages[0], state, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: protocol.SuccessEnd, Output: map[string]any{"markdown": "result"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if calls.Load() != 0 {
|
|
t.Fatal("terminal stage success published before FinishRun")
|
|
}
|
|
sub := r.events.subscribe("test", "workspace", nil, func(protocol.Message) error { return nil })
|
|
defer sub.close()
|
|
// Marshal failure occurs before FinishRun can commit its proposed success.
|
|
r.finishRunState(id, "test", "workspace", "succeeded", "success", "", 0, 0, "", stage.Values{"invalid": make(chan int)})
|
|
got, err := s.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Status != "running" || len(got.Notifications) != 0 || calls.Load() != 0 {
|
|
t.Fatalf("unpersisted success announced: %+v calls=%d", got, calls.Load())
|
|
}
|
|
sub.mu.Lock()
|
|
defer sub.mu.Unlock()
|
|
if len(sub.pending) != 0 {
|
|
t.Fatalf("unpersisted terminal event emitted: %+v", sub.pending)
|
|
}
|
|
}
|
|
|
|
func TestWaitingHookSharesTransactionalRunIntent(t *testing.T) {
|
|
for _, failFirst := range []bool{false, true} {
|
|
t.Run(map[bool]string{false: "covered", true: "retry"}[failFirst], func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "waiting-hook.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if calls.Add(1) == 1 && failFirst {
|
|
http.Error(w, "transient", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "waiting-ack", "event": "message"})
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
source := strings.Replace(humanPipelineSource, "# stage gate", "# stage gate\n```notify\non=waiting\n```", 1)
|
|
if _, err := s.Submit(ctx, "waiting", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := s.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, err := newRunner(ctx, s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.schedulerWG.Wait() }()
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "waiting-hook", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
r.deliverRunNotifications(time.Now().Add(time.Minute))
|
|
detail, err := s.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantCalls := int32(1)
|
|
if failFirst {
|
|
wantCalls = 2
|
|
}
|
|
if calls.Load() != wantCalls || detail.Status != "waiting" || len(detail.Notifications) != 1 {
|
|
t.Fatalf("waiting hook duplicated/lost: calls=%d detail=%+v", calls.Load(), detail)
|
|
}
|
|
status := detail.Notifications[0].Status
|
|
if status != "sent" && status != "covered" {
|
|
t.Fatalf("missing acknowledgement: %+v", detail.Notifications)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunNotificationModesAndFinalStageDedupe(t *testing.T) {
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
for _, tc := range []struct {
|
|
mode string
|
|
hook bool
|
|
want int32
|
|
}{
|
|
{"on", false, 1}, {"auto", false, 0}, {"off", false, 0}, {"on", true, 1}, {"auto", true, 1}, {"off", true, 0},
|
|
} {
|
|
t.Run(tc.mode+map[bool]string{true: "-hook", false: "-no-hook"}[tc.hook], func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "modes.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
source := pipeline.Template([]string{"one"}, true)
|
|
if tc.hook {
|
|
source = strings.Replace(source, "# stage one", "# stage one\n```notify\non=success\n```", 1)
|
|
}
|
|
if _, err := s.Submit(ctx, "mode-test", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := s.List(ctx)
|
|
r, err := newRunner(ctx, s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success"}, nil
|
|
}
|
|
before := calls.Load()
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mode", NotifyMode: tc.mode, Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err := s.GetRun(ctx, run.ID)
|
|
if err != nil || detail.Status != "succeeded" || detail.NotifyMode != tc.mode || calls.Load()-before != tc.want {
|
|
t.Fatalf("mode contract: %+v calls=%d %v", detail, calls.Load()-before, err)
|
|
}
|
|
if tc.mode == "on" && (len(detail.Notifications) != 1 || (detail.Notifications[0].Status != "sent" && detail.Notifications[0].Status != "covered")) {
|
|
t.Fatalf("missing final audit: %+v", detail.Notifications)
|
|
}
|
|
cancel()
|
|
r.schedulerWG.Wait()
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunNotificationAttentionModesAndRetryBound(t *testing.T) {
|
|
ctx := context.Background()
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "attention.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
http.Error(w, "unavailable", http.StatusServiceUnavailable)
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
r := &runner{ctx: ctx, store: s}
|
|
for i, mode := range []string{"auto", "on", "off"} {
|
|
id := int64(i + 1)
|
|
r.setNotifyMode(id, mode)
|
|
for sequence, status := range []string{"waiting", "failed"} {
|
|
r.notifyRun(id, "test", status, sequence, "stage", "")
|
|
}
|
|
}
|
|
if calls.Load() != 4 {
|
|
t.Fatalf("attention modes: %d calls, want 4", calls.Load())
|
|
}
|
|
r.deliverRunNotifications(time.Now().Add(time.Minute))
|
|
r.deliverRunNotifications(time.Now().Add(2 * time.Minute))
|
|
r.deliverRunNotifications(time.Now().Add(3 * time.Minute))
|
|
if calls.Load() != 12 {
|
|
t.Fatalf("retry bound: %d calls, want 12", calls.Load())
|
|
}
|
|
for _, id := range []int64{1, 2, 3} {
|
|
got, err := s.RunNotifications(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if id == 3 && len(got) != 0 {
|
|
t.Fatalf("off created attempts: %+v", got)
|
|
}
|
|
for _, n := range got {
|
|
if n.Status != "failed" || n.Attempts != 3 {
|
|
t.Fatalf("missing exhausted audit: %+v", n)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunNotificationWorkerResumesPendingAfterRestart(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
path := filepath.Join(t.TempDir(), "worker.db")
|
|
s, err := pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.EnqueueRunNotification(ctx, pipeline.RunNotification{RunID: 7, Sequence: 1, Event: "failed", Channel: "system", Title: "Run failed", Body: "pending"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err = pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
delivered := make(chan struct{}, 2)
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "recovered", "event": "message"})
|
|
delivered <- struct{}{}
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
r, err := newRunner(ctx, s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.schedulerWG.Wait() }()
|
|
select {
|
|
case <-delivered:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("pending outbox was not recovered")
|
|
}
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for {
|
|
got, err := s.RunNotifications(ctx, 7)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) == 1 && got[0].Status == "sent" && got[0].MessageID == "recovered" {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("recovery audit: %+v", got)
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|