Notify on failed runs and with ntfy disabled

This commit is contained in:
2026-11-12 01:55:28 +00:00
parent 2330f50268
commit 7b1b91f452
10 changed files with 78 additions and 37 deletions
@@ -93,10 +93,12 @@ produce the required three strings, so adding this stage alone cannot cover it.
Independently of any `notify` block, the server publishes one run-level attention
notification to the same already-configured `system` destination when a run needs a
person: when it enters `waiting` (a human stage asked for a decision) and when it
fails. Nothing else notifies — a queued, running, succeeded, canceled or interrupted
run sends nothing. You do not author these; they are the operator default, so a run
you never wired still reaches the operator.
person: when it enters `waiting` (a human stage asked for a decision), or when it
fails or is interrupted. An auto-mode run always notifies for failure/interruption;
a successful auto-mode run notifies only when the operator enables `run_finished`.
Runs marked `on` notify at completion, while `off` disables these notifications.
Queued and running runs send nothing. You do not author these; they are the operator
default, so a run you never wired still reaches the operator.
The payload names the run and its pipeline. When the operator configures
`PIPELINES_WEB_PUBLIC_URL` on the server, run notifications and stage hooks carry
@@ -78,9 +78,8 @@ func TestNotificationSettingsRejectUnknownOrInvalid(t *testing.T) {
}
}
// A run's end (succeeded or failed) enqueues its generic alert only for a run
// marked "on", or for "auto" when run_finished (default off) opts into every
// run end. Waiting notifies auto and on runs regardless; "off" never notifies.
// Failed runs always enqueue their generic alert (unless off). Successful runs
// require "on" or run_finished. Waiting notifies auto and on runs regardless.
func TestRunEndEnqueueFollowsNotifyModeAndRunFinished(t *testing.T) {
for _, status := range []string{"succeeded", "failed", "waiting"} {
for _, tc := range []struct {
@@ -91,6 +90,9 @@ func TestRunEndEnqueueFollowsNotifyModeAndRunFinished(t *testing.T) {
{"auto", false, 0}, {"auto", true, 1}, {"on", false, 1}, {"on", true, 1}, {"off", false, 0}, {"off", true, 0},
} {
want := tc.want
if status == "failed" && tc.mode == "auto" {
want = 1
}
if status == "waiting" && tc.mode == "auto" {
want = 1
}
@@ -138,7 +140,7 @@ func TestLegacyRunFinishedValueIsIgnored(t *testing.T) {
if err := notificationTransition(ctx, s, id, "failed"); err != nil {
t.Fatal(err)
}
if got, err := s.RunNotifications(ctx, id); err != nil || len(got) != 0 {
if got, err := s.RunNotifications(ctx, id); err != nil || len(got) != 1 {
t.Fatalf("notifications = %+v, %v", got, err)
}
}
@@ -144,10 +144,10 @@ func TestRunNotificationTransitionModesAndCoverage(t *testing.T) {
if err != nil {
t.Fatal(err)
}
// run_finished defaults off, so an auto run notifies only
// while waiting; its end is silent unless marked "on".
// Failed runs always notify in auto mode; successful runs require
// "on" or run_finished. Waiting notifies auto and on.
want := 1
if mode == "off" || (mode == "auto" && status != "waiting") {
if mode == "off" || (mode == "auto" && status == "succeeded") {
want = 0
}
if len(ns) != want {
+3 -3
View File
@@ -42,10 +42,10 @@ JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version
case "waiting":
title = "Run needs your decision"
body = "waiting for a human decision"
case "failed", "succeeded":
// A run's end notifies only for runs marked "on", unless the operator
case "failed", "interrupted", "succeeded":
// Successful runs notify only for runs marked "on", unless the operator
// opted into every run end via run_finished, read in this transaction.
if mode != "on" {
if status == "succeeded" && mode != "on" {
settings, err := readNotificationSettings(ctx, tx)
if err != nil {
return err
+2
View File
@@ -290,6 +290,8 @@ func runNotificationKind(event string) string {
return protocol.NotificationRunWaiting
case "failed":
return protocol.NotificationRunFailed
case "interrupted":
return protocol.NotificationRunFailed
case "succeeded":
return protocol.NotificationRunSucceeded
}
+19
View File
@@ -50,6 +50,25 @@ func newAppNotificationFixture(t *testing.T, ctx context.Context) *appNotificati
return &appNotificationFixture{t: t, ctx: ctx, store: store, runner: handler.(*runHandler).runner, url: srv.URL, def: def}
}
func TestToolNotificationRecordsAppNotificationWhenNtfyDisabled(t *testing.T) {
ctx := context.Background()
store := newOperatorStore(t)
if _, err := store.UpdateNotificationSettings(ctx, map[string]any{protocol.NotifyNtfyEnabled: false}); err != nil {
t.Fatal(err)
}
hub := &chatHub{notes: newAppNotifier(store), settings: store}
if ok, diagnostic := hub.publishToolNotification(ctx, "session-1", "Build failed", "Please inspect the run"); !ok || diagnostic != "" {
t.Fatalf("tool notification result = %v, %q", ok, diagnostic)
}
active, _, err := store.ActiveAppNotifications(ctx, "", time.Now())
if err != nil || len(active) != 1 {
t.Fatalf("app notifications = %+v, %v", active, err)
}
if active[0].Kind != protocol.NotificationChatReply || active[0].Title != "Build failed" || active[0].Message != "Please inspect the run" {
t.Fatalf("wrong tool notification: %+v", active[0])
}
}
// failRun creates a "Notify me" run and fails it, which commits a durable run
// notification intent in the same transaction, then drains the outbox.
func (f *appNotificationFixture) failRun(name string) int64 {
+16 -7
View File
@@ -2,6 +2,8 @@ package server
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"log"
@@ -12,6 +14,7 @@ import (
"path/filepath"
"strings"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
@@ -128,14 +131,20 @@ type chatToolNotifier func(ctx context.Context, sessionID, title, message string
// ntfy destination (the one chat attention uses), honoring their ntfy switch.
// Tapping it opens the chat (pipelines://chat/<id>).
func (h *chatHub) publishToolNotification(ctx context.Context, sessionID, title, message string) (bool, string) {
if !chatAttentionConfigured() {
return false, "phone notifications (ntfy) are not configured on this Pipelines server"
}
if !chatNotificationSettings(ctx, h.settings).NtfyEnabled {
return false, "the operator has turned phone notifications off"
}
title = truncateUTF8(title, 256)
message = truncateUTF8(message, 2000)
click, views := chatAttentionLinks(sessionID)
if _, err := chatAttentionPublish(ctx, "system", truncateUTF8(title, 256), truncateUTF8(message, 2000), "", nil, click, views...); err != nil {
digest := sha256.Sum256([]byte(title + "\x00" + message))
if h.notes != nil {
h.notes.record(ctx, pipeline.AppNotificationRecord{
AppNotification: protocol.AppNotification{ID: "chat:" + sessionID + ":notify:" + hex.EncodeToString(digest[:8]), Kind: protocol.NotificationChatReply, Title: title, Message: message, Click: click, Priority: protocol.DefaultNotificationPriority, Tags: []string{}, Group: "chat:" + sessionID},
SessionID: sessionID,
})
}
if !chatAttentionConfigured() || !chatNotificationSettings(ctx, h.settings).NtfyEnabled {
return true, ""
}
if _, err := chatAttentionPublish(ctx, "system", title, message, "", nil, click, views...); err != nil {
log.Printf("chat tool notification session=%s: %v", sessionID, err)
return false, "the notification service refused it: " + err.Error()
}
+9 -5
View File
@@ -229,8 +229,8 @@ func runAttentionEvent(status string) string {
switch strings.TrimSpace(strings.ToLower(status)) {
case protocol.RunStatusWaiting:
return protocol.NotifyWaiting
case "failed":
return "failed"
case "failed", "interrupted":
return strings.TrimSpace(strings.ToLower(status))
}
return ""
}
@@ -375,9 +375,9 @@ func (r *runner) notifyRun(id int64, name, status string, sequence int, currentS
if event == "" {
return
}
// A run's end notifies only runs marked "on", unless the operator opted
// into every run end (run_finished); waiting notifies any non-off run.
if event != protocol.NotifyWaiting && r.notifyMode(id) != "on" {
// Failed or interrupted runs always notify; successful runs notify only when
// marked "on" or when the operator opted into every run end (run_finished).
if event == "succeeded" || (event != protocol.NotifyWaiting && event != "failed" && event != "interrupted") {
settings, err := r.store.NotificationSettings(r.ctx)
if err != nil || !settings.RunFinished {
return
@@ -392,6 +392,10 @@ func (r *runner) notifyRun(id int64, name, status string, sequence int, currentS
title = "Run failed"
summary = fmt.Sprintf("Run #%d · %s · run failed", id, name)
}
if event == "interrupted" {
title = "Run interrupted"
summary = fmt.Sprintf("Run #%d · %s · run interrupted", id, name)
}
if event == "succeeded" {
title = "Run succeeded"
summary = fmt.Sprintf("Run #%d · %s · run succeeded", id, name)
+9 -5
View File
@@ -278,7 +278,7 @@ func TestRunAttentionEventMapping(t *testing.T) {
{"running", ""},
{"succeeded", ""},
{"canceled", ""},
{"interrupted", ""},
{"interrupted", "interrupted"},
{"", ""},
} {
if got := runAttentionEvent(test.status); got != test.want {
@@ -320,7 +320,7 @@ func TestRunAttentionNotificationDelivery(t *testing.T) {
t.Fatal(err)
}
// A run that needs nobody publishes nothing.
for _, status := range []string{"queued", "running", "succeeded", "canceled", "interrupted"} {
for _, status := range []string{"queued", "running", "succeeded", "canceled"} {
r.notifyRun(7, "release", status, 1, "build", "")
}
select {
@@ -332,7 +332,7 @@ func TestRunAttentionNotificationDelivery(t *testing.T) {
r.notifyRun(7, "release", protocol.RunStatusWaiting, 1, "approve", "")
// A run that stays waiting is never republished, on a later call or a re-request.
r.notifyRun(7, "release", protocol.RunStatusWaiting, 1, "approve", "")
// An auto run's failure is silent: only runs marked "on" announce their end.
// An auto run's failure announces its end.
r.notifyRun(8, "release", "failed", 2, "deploy", "provider returned 500")
// A failed run marked "on" publishes once, with its diagnostic.
r.setNotifyMode(7, "on")
@@ -352,12 +352,16 @@ func TestRunAttentionNotificationDelivery(t *testing.T) {
t.Fatalf("notification leaked an authorization header: %q", first.authorization)
}
second := <-deliveries
if second.title != "Run failed" || !strings.Contains(second.message, "Run #7") || !strings.Contains(second.message, "release") || !strings.Contains(second.message, "deploy") || !strings.Contains(second.message, "provider returned 500") {
if second.title != "Run failed" || !strings.Contains(second.message, "Run #8") {
t.Fatalf("failed notification is not actionable: %+v", second)
}
if second.click != "pipelines://run/7" || second.actions != "view, Open in browser, https://pipelines.example.com/runs/7" {
if second.click != "pipelines://run/8" || second.actions != "view, Open in browser, https://pipelines.example.com/runs/8" {
t.Fatalf("failed notification click does not open the run: %q", second.click)
}
third := <-deliveries
if third.title != "Run failed" || !strings.Contains(third.message, "Run #7") {
t.Fatalf("marked failed notification is not actionable: %+v", third)
}
select {
case got := <-deliveries:
t.Fatalf("third notification delivered for two genuine events: %+v", got)
@@ -334,16 +334,15 @@ func TestRunNotificationAttentionModesAndRetryBound(t *testing.T) {
r.notifyRun(id, "test", status, sequence, "stage", "")
}
}
// auto waiting, on waiting and on failed; an auto run's failure is silent
// because run_finished defaults off.
if calls.Load() != 3 {
t.Fatalf("attention modes: %d calls, want 3", calls.Load())
// auto and on waiting, plus auto and on failed; off never notifies.
if calls.Load() != 4 {
t.Fatalf("attention modes: %d calls, want 4", calls.Load())
}
r.deliverRunNotifications(time.Now().Add(time.Minute))
r.deliverRunNotifications(time.Now().Add(2 * time.Minute))
r.deliverRunNotifications(time.Now().Add(3 * time.Minute))
if calls.Load() != 9 {
t.Fatalf("retry bound: %d calls, want 9", calls.Load())
if calls.Load() != 12 {
t.Fatalf("retry bound: %d calls, want 12", calls.Load())
}
for _, id := range []int64{1, 2, 3} {
got, err := s.RunNotifications(ctx, id)