398 lines
14 KiB
Go
398 lines
14 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"git.a-wan.net/alan/pipelines/internal/summarise"
|
|
)
|
|
|
|
func TestRunNotificationPublisherFailureRetriesAfterReopen(t *testing.T) {
|
|
t.Setenv("PIPELINES_WEB_PUBLIC_URL", "https://pipelines.example.com")
|
|
ctx := context.Background()
|
|
path := filepath.Join(t.TempDir(), "retry.db")
|
|
s, err := pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if got := r.Header.Get("Click"); got != "https://pipelines.example.com/runs/7" {
|
|
t.Errorf("wrong web retry link: %q", got)
|
|
}
|
|
if calls.Add(1) == 1 {
|
|
http.Error(w, "transient", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
r := &runner{ctx: ctx, store: s}
|
|
// Emulate a durable transition intent saved before the runner callback.
|
|
if err := s.EnqueueRunNotification(ctx, pipeline.RunNotification{RunID: 7, Sequence: 1, Event: "failed", Channel: "system", Title: "Run failed", Body: "Run 7", Click: "pipelines://run/7"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.notifyRun(7, "test", "failed", 1, "stage", "")
|
|
got, err := s.RunNotifications(ctx, 7)
|
|
if err != nil || len(got) != 1 || got[0].Status != "pending" || got[0].Attempts != 1 || got[0].Diagnostic == "" {
|
|
t.Fatalf("failed attempt not audited: %+v %v", got, err)
|
|
}
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err = pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
r = &runner{ctx: ctx, store: s}
|
|
r.deliverRunNotifications(time.Now().Add(time.Minute))
|
|
r.notifyRun(7, "test", "failed", 1, "stage", "")
|
|
got, err = s.RunNotifications(ctx, 7)
|
|
if err != nil || len(got) != 1 || got[0].Status != "sent" || got[0].Attempts != 2 || got[0].MessageID != "ack" || calls.Load() != 2 {
|
|
t.Fatalf("retry/dedupe: %+v calls=%d %v", got, calls.Load(), err)
|
|
}
|
|
}
|
|
|
|
func TestTransactionalSuccessNotificationUsesWebLink(t *testing.T) {
|
|
ctx := context.Background()
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "web-success.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
if _, err := s.Submit(ctx, "test", pipeline.Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := s.List(ctx)
|
|
d, err := s.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
id, _, err := s.CreateRunTitled(ctx, d, "web-success", t.TempDir(), false, nil, nil, pipeline.RunOverrides{NotifyMode: "on"}, summarise.Title{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var calls atomic.Int32
|
|
endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
|
calls.Add(1)
|
|
if got, want := req.Header.Get("Click"), "https://pipelines.example.com/runs/"+fmt.Sprint(id); got != want {
|
|
t.Errorf("click = %q, want %q", got, want)
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "web-success", "event": "message"})
|
|
}))
|
|
defer endpoint.Close()
|
|
t.Setenv("PIPELINES_WEB_PUBLIC_URL", "https://pipelines.example.com")
|
|
t.Setenv("PIPELINES_NTFY_URL", endpoint.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
r := &runner{ctx: ctx, store: s}
|
|
r.notifyRun(id, "test", "succeeded", 0, "", "")
|
|
r.deliverRunNotifications(time.Now())
|
|
got, err := s.RunNotifications(ctx, id)
|
|
if err != nil || len(got) != 1 || got[0].Status != "sent" || calls.Load() != 1 {
|
|
t.Fatalf("notification: %+v calls=%d err=%v", got, calls.Load(), err)
|
|
}
|
|
}
|
|
|
|
func TestFinishRunPersistenceFailureDoesNotAnnounceSuccess(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "not-finished.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
source := strings.Replace(pipeline.Template([]string{"one"}, true), "# stage one", "# stage one\n```notify\non=success\n```", 1)
|
|
if _, err := s.Submit(ctx, "test", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := s.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d, err := s.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
id, _, err := s.CreateRunTitled(ctx, d, "not-finished", "workspace", false, nil, nil, pipeline.RunOverrides{NotifyMode: "on"}, summarise.Title{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "wrong-success", "event": "message"})
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
r := &runner{ctx: ctx, store: s, events: newRunEventHub()}
|
|
if err := s.BeginStep(ctx, id, 1, "one", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
state := &runState{memory: map[string]memoryValue{}}
|
|
if _, _, _, _, err := r.finishStage(id, "test", "workspace", d.Graph.Stages[0], state, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: protocol.SuccessEnd, Output: map[string]any{"markdown": "result"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if calls.Load() != 0 {
|
|
t.Fatal("terminal stage success published before FinishRun")
|
|
}
|
|
sub := r.events.subscribe("test", "workspace", nil, func(protocol.Message) error { return nil })
|
|
defer sub.close()
|
|
// Marshal failure occurs before FinishRun can commit its proposed success.
|
|
r.finishRunState(id, "test", "workspace", "succeeded", "success", "", 0, 0, "", stage.Values{"invalid": make(chan int)})
|
|
got, err := s.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Status != "running" || len(got.Notifications) != 0 || calls.Load() != 0 {
|
|
t.Fatalf("unpersisted success announced: %+v calls=%d", got, calls.Load())
|
|
}
|
|
sub.mu.Lock()
|
|
defer sub.mu.Unlock()
|
|
if len(sub.pending) != 0 {
|
|
t.Fatalf("unpersisted terminal event emitted: %+v", sub.pending)
|
|
}
|
|
}
|
|
|
|
func TestWaitingHookSharesTransactionalRunIntent(t *testing.T) {
|
|
for _, failFirst := range []bool{false, true} {
|
|
t.Run(map[bool]string{false: "covered", true: "retry"}[failFirst], func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "waiting-hook.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if calls.Add(1) == 1 && failFirst {
|
|
http.Error(w, "transient", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "waiting-ack", "event": "message"})
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
source := strings.Replace(humanPipelineSource, "# stage gate", "# stage gate\n```notify\non=waiting\n```", 1)
|
|
if _, err := s.Submit(ctx, "waiting", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := s.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, err := newRunner(ctx, s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.schedulerWG.Wait() }()
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "waiting-hook", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
r.deliverRunNotifications(time.Now().Add(time.Minute))
|
|
detail, err := s.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantCalls := int32(1)
|
|
if failFirst {
|
|
wantCalls = 2
|
|
}
|
|
if calls.Load() != wantCalls || detail.Status != "waiting" || len(detail.Notifications) != 1 {
|
|
t.Fatalf("waiting hook duplicated/lost: calls=%d detail=%+v", calls.Load(), detail)
|
|
}
|
|
status := detail.Notifications[0].Status
|
|
if status != "sent" && status != "covered" {
|
|
t.Fatalf("missing acknowledgement: %+v", detail.Notifications)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunNotificationModesAndFinalStageDedupe(t *testing.T) {
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
for _, tc := range []struct {
|
|
mode string
|
|
hook bool
|
|
want int32
|
|
}{
|
|
{"on", false, 1}, {"auto", false, 0}, {"off", false, 0}, {"on", true, 1}, {"auto", true, 1}, {"off", true, 0},
|
|
} {
|
|
t.Run(tc.mode+map[bool]string{true: "-hook", false: "-no-hook"}[tc.hook], func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "modes.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
source := pipeline.Template([]string{"one"}, true)
|
|
if tc.hook {
|
|
source = strings.Replace(source, "# stage one", "# stage one\n```notify\non=success\n```", 1)
|
|
}
|
|
if _, err := s.Submit(ctx, "mode-test", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := s.List(ctx)
|
|
r, err := newRunner(ctx, s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success"}, nil
|
|
}
|
|
before := calls.Load()
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mode", NotifyMode: tc.mode, Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err := s.GetRun(ctx, run.ID)
|
|
if err != nil || detail.Status != "succeeded" || detail.NotifyMode != tc.mode || calls.Load()-before != tc.want {
|
|
t.Fatalf("mode contract: %+v calls=%d %v", detail, calls.Load()-before, err)
|
|
}
|
|
if tc.mode == "on" && (len(detail.Notifications) != 1 || (detail.Notifications[0].Status != "sent" && detail.Notifications[0].Status != "covered")) {
|
|
t.Fatalf("missing final audit: %+v", detail.Notifications)
|
|
}
|
|
cancel()
|
|
r.schedulerWG.Wait()
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunNotificationAttentionModesAndRetryBound(t *testing.T) {
|
|
ctx := context.Background()
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "attention.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
http.Error(w, "unavailable", http.StatusServiceUnavailable)
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
r := &runner{ctx: ctx, store: s}
|
|
for i, mode := range []string{"auto", "on", "off"} {
|
|
id := int64(i + 1)
|
|
r.setNotifyMode(id, mode)
|
|
for sequence, status := range []string{"waiting", "failed"} {
|
|
r.notifyRun(id, "test", status, sequence, "stage", "")
|
|
}
|
|
}
|
|
if calls.Load() != 4 {
|
|
t.Fatalf("attention modes: %d calls, want 4", calls.Load())
|
|
}
|
|
r.deliverRunNotifications(time.Now().Add(time.Minute))
|
|
r.deliverRunNotifications(time.Now().Add(2 * time.Minute))
|
|
r.deliverRunNotifications(time.Now().Add(3 * time.Minute))
|
|
if calls.Load() != 12 {
|
|
t.Fatalf("retry bound: %d calls, want 12", calls.Load())
|
|
}
|
|
for _, id := range []int64{1, 2, 3} {
|
|
got, err := s.RunNotifications(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if id == 3 && len(got) != 0 {
|
|
t.Fatalf("off created attempts: %+v", got)
|
|
}
|
|
for _, n := range got {
|
|
if n.Status != "failed" || n.Attempts != 3 {
|
|
t.Fatalf("missing exhausted audit: %+v", n)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunNotificationWorkerResumesPendingAfterRestart(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
path := filepath.Join(t.TempDir(), "worker.db")
|
|
s, err := pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.EnqueueRunNotification(ctx, pipeline.RunNotification{RunID: 7, Sequence: 1, Event: "failed", Channel: "system", Title: "Run failed", Body: "pending"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err = pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
delivered := make(chan struct{}, 2)
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "recovered", "event": "message"})
|
|
delivered <- struct{}{}
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
r, err := newRunner(ctx, s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.schedulerWG.Wait() }()
|
|
select {
|
|
case <-delivered:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("pending outbox was not recovered")
|
|
}
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for {
|
|
got, err := s.RunNotifications(ctx, 7)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) == 1 && got[0].Status == "sent" && got[0].MessageID == "recovered" {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("recovery audit: %+v", got)
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|