fix: persist and retry run alerts and honor notification modes

This commit is contained in:
2026-10-20 15:42:21 +01:00
parent e4f48ecbb1
commit 016fcb552a
20 changed files with 697 additions and 41 deletions
+80 -14
View File
@@ -3,6 +3,7 @@ package server
import (
"context"
"fmt"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"io"
@@ -34,10 +35,6 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
if !enabled {
return
}
// The stage's own policy already covers this attention event, so the
// run-level default must not repeat it. Recorded before delivery so a failed
// stage notification is reported by its own receipt, not silently duplicated.
r.markAttentionSent(id, step.Sequence, stageAttentionEvent(event))
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
defer cancel()
summary := fmt.Sprintf("%s · run #%d · stage %s · %s (%s)", name, id, step.Stage, event, step.Outcome)
@@ -128,6 +125,18 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
if !sentArtifact {
send("", nil)
}
// Only a successful publish covers the run alert. A failed stage publisher
// must not swallow the durable run-level fallback and its retries.
for _, receipt := range receipts {
if receipt.Status == "sent" {
covered := stageAttentionEvent(event)
if event == "success" && step.Target == protocol.SuccessEnd {
covered = "succeeded"
}
r.markAttentionSent(id, step.Sequence, covered)
break
}
}
}
// runAttentionEvent maps a run status to the operator attention notification it
@@ -167,8 +176,16 @@ func (r *runner) markAttentionSent(id int64, sequence int, event string) {
if event == "" {
return
}
if r.store != nil {
if err := r.store.CoverRunNotification(r.ctx, id, sequence, event); err != nil {
log.Printf("record stage notification coverage run=%d: %v", id, err)
}
}
r.attentionMu.Lock()
defer r.attentionMu.Unlock()
if r.attentionSent == nil {
r.attentionSent = map[int64]map[string]bool{}
}
if r.attentionSent[id] == nil {
r.attentionSent[id] = map[string]bool{}
}
@@ -206,6 +223,9 @@ func (r *runner) notifyMode(id int64) string {
if r.store != nil {
if stored, err := r.store.RunNotifyMode(r.ctx, id); err == nil && stored != "" {
r.notifyMu.Lock()
if r.notifyModes == nil {
r.notifyModes = map[int64]string{}
}
r.notifyModes[id] = stored
r.notifyMu.Unlock()
return stored
@@ -240,29 +260,32 @@ func runAttentionClick(id int64) string {
// notifyStep and reuses the same PublishNtfy machinery, destination and token
// handling, so no second publisher, topic or service is introduced. The run id
// and pipeline name identify the run and the click action opens it in the app.
// A run that needs nobody produces nothing, and a delivery failure is logged and
// never fails the run: this is a best-effort alert, not persisted run evidence.
// Attempts are persisted before publication. Publisher acknowledgement is not
// proof of device reception; a crash in that acknowledgement window can repeat.
func (r *runner) notifyRun(id int64, name, status string, sequence int, currentStage, diagnostic string) {
if r.notifyMode(id) == "off" {
return
}
event := runAttentionEvent(status)
if status == "succeeded" && r.notifyMode(id) == "on" {
event = "succeeded"
}
if event == "" {
return
}
if r.attentionAlready(id, sequence, event) {
return
}
// Record the delivery attempt before publishing so a repeated transition for
// the same run and event (a re-request, a restart-free re-entry) never
// republishes; delivery itself is never retried.
r.markAttentionSent(id, sequence, event)
title := "Run needs your decision"
summary := fmt.Sprintf("Run #%d · %s · waiting for a human decision", id, name)
if event == "failed" {
title = "Run failed"
summary = fmt.Sprintf("Run #%d · %s · run failed", id, name)
}
if event == "succeeded" {
title = "Run succeeded"
summary = fmt.Sprintf("Run #%d · %s · run succeeded", id, name)
}
if currentStage != "" {
summary += " at stage " + currentStage
}
@@ -270,10 +293,53 @@ func (r *runner) notifyRun(id int64, name, status string, sequence int, currentS
summary += ": " + diagnostic
}
summary = truncateUTF8(summary, 2000)
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
defer cancel()
if _, err := stage.PublishNtfy(ctx, "system", title, summary, "", nil, runAttentionClick(id)); err != nil {
log.Printf("run attention notification run=%d: %v", id, err)
if err := r.store.EnqueueRunNotification(r.ctx, pipeline.RunNotification{RunID: id, Sequence: sequence, Event: event, Channel: "system", Title: title, Body: summary, Click: runAttentionClick(id)}); err != nil {
log.Printf("enqueue run notification run=%d: %v", id, err)
return
}
r.deliverRunNotificationBatch(time.Now(), 1)
}
// resumeRunNotifications also recovers pending/expired leases after restart.
func (r *runner) resumeRunNotifications() {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-r.ctx.Done():
return
case <-ticker.C:
r.deliverRunNotifications(time.Now())
}
}
}
func (r *runner) deliverRunNotifications(now time.Time) {
r.deliverRunNotificationBatch(now, 32)
}
func (r *runner) deliverRunNotificationBatch(now time.Time, limit int) {
// Bound one drain so a queue cannot monopolize the runner.
started := time.Now()
for i := 0; i < limit; i++ {
n, err := r.store.ClaimRunNotification(r.ctx, now.Add(time.Since(started)).Unix())
if err != nil {
log.Printf("claim run notification: %v", err)
return
}
if n == nil {
return
}
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
result, publishErr := stage.PublishNtfy(ctx, n.Channel, n.Title, n.Body, "", nil, n.Click)
cancel()
messageID, _ := result["message_id"].(string)
save, done := context.WithTimeout(context.Background(), 5*time.Second)
err = r.store.FinishRunNotification(save, *n, now.Add(time.Since(started)).Unix(), messageID, publishErr)
done()
if err != nil {
log.Printf("save run notification run=%d: %v", n.RunID, err)
}
}
}
+10 -2
View File
@@ -121,7 +121,7 @@ func TestRunEventWebSocketPushesTransitions(t *testing.T) {
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "sub" {
t.Fatalf("subscribe ack = %+v", ack)
}
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, NotifyMode: "off", Input: map[string]any{"prompt": "test"}})
var runID int64
var created, stepStarted, stepFinished, terminal *protocol.RunEventFrame
@@ -175,6 +175,11 @@ func TestRunEventWebSocketPushesTransitions(t *testing.T) {
if terminal.RunID != runID || terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.FinishedAt == "" {
t.Fatalf("terminal frame = %+v", terminal)
}
for _, frame := range []*protocol.RunEventFrame{created, stepStarted, stepFinished, terminal} {
if frame.NotifyMode != "off" {
t.Fatalf("silent mode lost in event: %+v", frame)
}
}
}
// TestRunEventSubscribeSendsSnapshot proves a client that subscribes to an
@@ -188,7 +193,7 @@ func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
workspace := eventWorkspace(t)
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, NotifyMode: "off", Input: map[string]any{"prompt": "test"}})
if err != nil {
t.Fatal(err)
}
@@ -203,6 +208,9 @@ func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
if first.RunEvent.RunID != run.ID || first.RunEvent.Status != "succeeded" || first.RunEvent.CompletedSteps != 1 || first.RunEvent.FinishedAt == "" {
t.Fatalf("snapshot frame = %+v", first.RunEvent)
}
if first.RunEvent.NotifyMode != "off" {
t.Fatalf("silent mode lost in snapshot: %+v", first.RunEvent)
}
// A snapshot for a run the client never saw also carries the identity its row
// needs, so a fresh subscribe renders it without a second read.
if first.RunEvent.PipelineID != ids[0] || first.RunEvent.Name == "" || first.RunEvent.Version != 1 {
@@ -0,0 +1,221 @@
package server
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func TestRunNotificationPublisherFailureRetriesAfterReopen(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "retry.db")
s, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if calls.Add(1) == 1 {
http.Error(w, "transient", http.StatusServiceUnavailable)
return
}
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r := &runner{ctx: ctx, store: s}
r.notifyRun(7, "test", "failed", 1, "stage", "")
got, err := s.RunNotifications(ctx, 7)
if err != nil || len(got) != 1 || got[0].Status != "pending" || got[0].Attempts != 1 || got[0].Diagnostic == "" {
t.Fatalf("failed attempt not audited: %+v %v", got, err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
r = &runner{ctx: ctx, store: s}
r.deliverRunNotifications(time.Now().Add(time.Minute))
r.notifyRun(7, "test", "failed", 1, "stage", "")
got, err = s.RunNotifications(ctx, 7)
if err != nil || len(got) != 1 || got[0].Status != "sent" || got[0].Attempts != 2 || got[0].MessageID != "ack" || calls.Load() != 2 {
t.Fatalf("retry/dedupe: %+v calls=%d %v", got, calls.Load(), err)
}
}
func TestRunNotificationModesAndFinalStageDedupe(t *testing.T) {
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
for _, tc := range []struct {
mode string
hook bool
want int32
}{
{"on", false, 1}, {"auto", false, 0}, {"off", false, 0}, {"on", true, 1}, {"auto", true, 1}, {"off", true, 0},
} {
t.Run(tc.mode+map[bool]string{true: "-hook", false: "-no-hook"}[tc.hook], func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
s, err := pipeline.Open(filepath.Join(t.TempDir(), "modes.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
source := pipeline.Template([]string{"one"}, true)
if tc.hook {
source = strings.Replace(source, "# stage one", "# stage one\n```notify\non=success\n```", 1)
}
if _, err := s.Submit(ctx, "mode-test", source); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success"}, nil
}
before := calls.Load()
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mode", NotifyMode: tc.mode, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := s.GetRun(ctx, run.ID)
if err != nil || detail.Status != "succeeded" || detail.NotifyMode != tc.mode || calls.Load()-before != tc.want {
t.Fatalf("mode contract: %+v calls=%d %v", detail, calls.Load()-before, err)
}
if tc.mode == "on" && (len(detail.Notifications) != 1 || (detail.Notifications[0].Status != "sent" && detail.Notifications[0].Status != "covered")) {
t.Fatalf("missing final audit: %+v", detail.Notifications)
}
cancel()
r.schedulerWG.Wait()
})
}
}
func TestRunNotificationAttentionModesAndRetryBound(t *testing.T) {
ctx := context.Background()
s, err := pipeline.Open(filepath.Join(t.TempDir(), "attention.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
http.Error(w, "unavailable", http.StatusServiceUnavailable)
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r := &runner{ctx: ctx, store: s}
for i, mode := range []string{"auto", "on", "off"} {
id := int64(i + 1)
r.setNotifyMode(id, mode)
for sequence, status := range []string{"waiting", "failed"} {
r.notifyRun(id, "test", status, sequence, "stage", "")
}
}
if calls.Load() != 4 {
t.Fatalf("attention modes: %d calls, want 4", calls.Load())
}
r.deliverRunNotifications(time.Now().Add(time.Minute))
r.deliverRunNotifications(time.Now().Add(2 * time.Minute))
r.deliverRunNotifications(time.Now().Add(3 * time.Minute))
if calls.Load() != 12 {
t.Fatalf("retry bound: %d calls, want 12", calls.Load())
}
for _, id := range []int64{1, 2, 3} {
got, err := s.RunNotifications(ctx, id)
if err != nil {
t.Fatal(err)
}
if id == 3 && len(got) != 0 {
t.Fatalf("off created attempts: %+v", got)
}
for _, n := range got {
if n.Status != "failed" || n.Attempts != 3 {
t.Fatalf("missing exhausted audit: %+v", n)
}
}
}
}
func TestRunNotificationWorkerResumesPendingAfterRestart(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
path := filepath.Join(t.TempDir(), "worker.db")
s, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
if err := s.EnqueueRunNotification(ctx, pipeline.RunNotification{RunID: 7, Sequence: 1, Event: "failed", Channel: "system", Title: "Run failed", Body: "pending"}); err != nil {
t.Fatal(err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
delivered := make(chan struct{}, 2)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]string{"id": "recovered", "event": "message"})
delivered <- struct{}{}
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.schedulerWG.Wait() }()
select {
case <-delivered:
case <-time.After(5 * time.Second):
t.Fatal("pending outbox was not recovered")
}
deadline := time.Now().Add(5 * time.Second)
for {
got, err := s.RunNotifications(ctx, 7)
if err != nil {
t.Fatal(err)
}
if len(got) == 1 && got[0].Status == "sent" && got[0].MessageID == "recovered" {
break
}
if time.Now().After(deadline) {
t.Fatalf("recovery audit: %+v", got)
}
time.Sleep(10 * time.Millisecond)
}
}
+4
View File
@@ -424,6 +424,9 @@ func (s *runSubscription) close() {
}
func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
if in.NotifyMode != "" {
into.NotifyMode = in.NotifyMode
}
if in.Workspace != "" {
into.Workspace = in.Workspace
}
@@ -500,6 +503,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
continue
}
frame := protocol.RunEventFrame{
NotifyMode: summary.NotifyMode,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
+5 -1
View File
@@ -101,7 +101,10 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
return nil, err
}
root := filepath.Join(s.Directory(), "artifacts")
return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}, nil
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}
r.schedulerWG.Add(1)
go func() { defer r.schedulerWG.Done(); r.resumeRunNotifications() }()
return r, nil
}
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
@@ -1257,6 +1260,7 @@ func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) {
return
}
r.stampRunIdentity(&frame)
frame.NotifyMode = r.notifyMode(frame.RunID)
r.events.emit(frame, terminal)
}