222 lines
6.9 KiB
Go
222 lines
6.9 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
func TestRunNotificationPublisherFailureRetriesAfterReopen(t *testing.T) {
|
|
ctx := context.Background()
|
|
path := filepath.Join(t.TempDir(), "retry.db")
|
|
s, err := pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if calls.Add(1) == 1 {
|
|
http.Error(w, "transient", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
r := &runner{ctx: ctx, store: s}
|
|
r.notifyRun(7, "test", "failed", 1, "stage", "")
|
|
got, err := s.RunNotifications(ctx, 7)
|
|
if err != nil || len(got) != 1 || got[0].Status != "pending" || got[0].Attempts != 1 || got[0].Diagnostic == "" {
|
|
t.Fatalf("failed attempt not audited: %+v %v", got, err)
|
|
}
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err = pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
r = &runner{ctx: ctx, store: s}
|
|
r.deliverRunNotifications(time.Now().Add(time.Minute))
|
|
r.notifyRun(7, "test", "failed", 1, "stage", "")
|
|
got, err = s.RunNotifications(ctx, 7)
|
|
if err != nil || len(got) != 1 || got[0].Status != "sent" || got[0].Attempts != 2 || got[0].MessageID != "ack" || calls.Load() != 2 {
|
|
t.Fatalf("retry/dedupe: %+v calls=%d %v", got, calls.Load(), err)
|
|
}
|
|
}
|
|
|
|
func TestRunNotificationModesAndFinalStageDedupe(t *testing.T) {
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
for _, tc := range []struct {
|
|
mode string
|
|
hook bool
|
|
want int32
|
|
}{
|
|
{"on", false, 1}, {"auto", false, 0}, {"off", false, 0}, {"on", true, 1}, {"auto", true, 1}, {"off", true, 0},
|
|
} {
|
|
t.Run(tc.mode+map[bool]string{true: "-hook", false: "-no-hook"}[tc.hook], func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "modes.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
source := pipeline.Template([]string{"one"}, true)
|
|
if tc.hook {
|
|
source = strings.Replace(source, "# stage one", "# stage one\n```notify\non=success\n```", 1)
|
|
}
|
|
if _, err := s.Submit(ctx, "mode-test", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := s.List(ctx)
|
|
r, err := newRunner(ctx, s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success"}, nil
|
|
}
|
|
before := calls.Load()
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mode", NotifyMode: tc.mode, Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err := s.GetRun(ctx, run.ID)
|
|
if err != nil || detail.Status != "succeeded" || detail.NotifyMode != tc.mode || calls.Load()-before != tc.want {
|
|
t.Fatalf("mode contract: %+v calls=%d %v", detail, calls.Load()-before, err)
|
|
}
|
|
if tc.mode == "on" && (len(detail.Notifications) != 1 || (detail.Notifications[0].Status != "sent" && detail.Notifications[0].Status != "covered")) {
|
|
t.Fatalf("missing final audit: %+v", detail.Notifications)
|
|
}
|
|
cancel()
|
|
r.schedulerWG.Wait()
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunNotificationAttentionModesAndRetryBound(t *testing.T) {
|
|
ctx := context.Background()
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "attention.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
http.Error(w, "unavailable", http.StatusServiceUnavailable)
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
r := &runner{ctx: ctx, store: s}
|
|
for i, mode := range []string{"auto", "on", "off"} {
|
|
id := int64(i + 1)
|
|
r.setNotifyMode(id, mode)
|
|
for sequence, status := range []string{"waiting", "failed"} {
|
|
r.notifyRun(id, "test", status, sequence, "stage", "")
|
|
}
|
|
}
|
|
if calls.Load() != 4 {
|
|
t.Fatalf("attention modes: %d calls, want 4", calls.Load())
|
|
}
|
|
r.deliverRunNotifications(time.Now().Add(time.Minute))
|
|
r.deliverRunNotifications(time.Now().Add(2 * time.Minute))
|
|
r.deliverRunNotifications(time.Now().Add(3 * time.Minute))
|
|
if calls.Load() != 12 {
|
|
t.Fatalf("retry bound: %d calls, want 12", calls.Load())
|
|
}
|
|
for _, id := range []int64{1, 2, 3} {
|
|
got, err := s.RunNotifications(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if id == 3 && len(got) != 0 {
|
|
t.Fatalf("off created attempts: %+v", got)
|
|
}
|
|
for _, n := range got {
|
|
if n.Status != "failed" || n.Attempts != 3 {
|
|
t.Fatalf("missing exhausted audit: %+v", n)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunNotificationWorkerResumesPendingAfterRestart(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
path := filepath.Join(t.TempDir(), "worker.db")
|
|
s, err := pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.EnqueueRunNotification(ctx, pipeline.RunNotification{RunID: 7, Sequence: 1, Event: "failed", Channel: "system", Title: "Run failed", Body: "pending"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err = pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
delivered := make(chan struct{}, 2)
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"id": "recovered", "event": "message"})
|
|
delivered <- struct{}{}
|
|
}))
|
|
defer server.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
r, err := newRunner(ctx, s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.schedulerWG.Wait() }()
|
|
select {
|
|
case <-delivered:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("pending outbox was not recovered")
|
|
}
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for {
|
|
got, err := s.RunNotifications(ctx, 7)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) == 1 && got[0].Status == "sent" && got[0].MessageID == "recovered" {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("recovery audit: %+v", got)
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|