Notify on failed runs and with ntfy disabled
This commit is contained in:
@@ -93,10 +93,12 @@ produce the required three strings, so adding this stage alone cannot cover it.
|
||||
|
||||
Independently of any `notify` block, the server publishes one run-level attention
|
||||
notification to the same already-configured `system` destination when a run needs a
|
||||
person: when it enters `waiting` (a human stage asked for a decision) and when it
|
||||
fails. Nothing else notifies — a queued, running, succeeded, canceled or interrupted
|
||||
run sends nothing. You do not author these; they are the operator default, so a run
|
||||
you never wired still reaches the operator.
|
||||
person: when it enters `waiting` (a human stage asked for a decision), or when it
|
||||
fails or is interrupted. An auto-mode run always notifies for failure/interruption;
|
||||
a successful auto-mode run notifies only when the operator enables `run_finished`.
|
||||
Runs marked `on` notify at completion, while `off` disables these notifications.
|
||||
Queued and running runs send nothing. You do not author these; they are the operator
|
||||
default, so a run you never wired still reaches the operator.
|
||||
|
||||
The payload names the run and its pipeline. When the operator configures
|
||||
`PIPELINES_WEB_PUBLIC_URL` on the server, run notifications and stage hooks carry
|
||||
|
||||
@@ -78,9 +78,8 @@ func TestNotificationSettingsRejectUnknownOrInvalid(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A run's end (succeeded or failed) enqueues its generic alert only for a run
|
||||
// marked "on", or for "auto" when run_finished (default off) opts into every
|
||||
// run end. Waiting notifies auto and on runs regardless; "off" never notifies.
|
||||
// Failed runs always enqueue their generic alert (unless off). Successful runs
|
||||
// require "on" or run_finished. Waiting notifies auto and on runs regardless.
|
||||
func TestRunEndEnqueueFollowsNotifyModeAndRunFinished(t *testing.T) {
|
||||
for _, status := range []string{"succeeded", "failed", "waiting"} {
|
||||
for _, tc := range []struct {
|
||||
@@ -91,6 +90,9 @@ func TestRunEndEnqueueFollowsNotifyModeAndRunFinished(t *testing.T) {
|
||||
{"auto", false, 0}, {"auto", true, 1}, {"on", false, 1}, {"on", true, 1}, {"off", false, 0}, {"off", true, 0},
|
||||
} {
|
||||
want := tc.want
|
||||
if status == "failed" && tc.mode == "auto" {
|
||||
want = 1
|
||||
}
|
||||
if status == "waiting" && tc.mode == "auto" {
|
||||
want = 1
|
||||
}
|
||||
@@ -138,7 +140,7 @@ func TestLegacyRunFinishedValueIsIgnored(t *testing.T) {
|
||||
if err := notificationTransition(ctx, s, id, "failed"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := s.RunNotifications(ctx, id); err != nil || len(got) != 0 {
|
||||
if got, err := s.RunNotifications(ctx, id); err != nil || len(got) != 1 {
|
||||
t.Fatalf("notifications = %+v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,10 +144,10 @@ func TestRunNotificationTransitionModesAndCoverage(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// run_finished defaults off, so an auto run notifies only
|
||||
// while waiting; its end is silent unless marked "on".
|
||||
// Failed runs always notify in auto mode; successful runs require
|
||||
// "on" or run_finished. Waiting notifies auto and on.
|
||||
want := 1
|
||||
if mode == "off" || (mode == "auto" && status != "waiting") {
|
||||
if mode == "off" || (mode == "auto" && status == "succeeded") {
|
||||
want = 0
|
||||
}
|
||||
if len(ns) != want {
|
||||
|
||||
@@ -42,10 +42,10 @@ JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version
|
||||
case "waiting":
|
||||
title = "Run needs your decision"
|
||||
body = "waiting for a human decision"
|
||||
case "failed", "succeeded":
|
||||
// A run's end notifies only for runs marked "on", unless the operator
|
||||
case "failed", "interrupted", "succeeded":
|
||||
// Successful runs notify only for runs marked "on", unless the operator
|
||||
// opted into every run end via run_finished, read in this transaction.
|
||||
if mode != "on" {
|
||||
if status == "succeeded" && mode != "on" {
|
||||
settings, err := readNotificationSettings(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -290,6 +290,8 @@ func runNotificationKind(event string) string {
|
||||
return protocol.NotificationRunWaiting
|
||||
case "failed":
|
||||
return protocol.NotificationRunFailed
|
||||
case "interrupted":
|
||||
return protocol.NotificationRunFailed
|
||||
case "succeeded":
|
||||
return protocol.NotificationRunSucceeded
|
||||
}
|
||||
|
||||
@@ -50,6 +50,25 @@ func newAppNotificationFixture(t *testing.T, ctx context.Context) *appNotificati
|
||||
return &appNotificationFixture{t: t, ctx: ctx, store: store, runner: handler.(*runHandler).runner, url: srv.URL, def: def}
|
||||
}
|
||||
|
||||
func TestToolNotificationRecordsAppNotificationWhenNtfyDisabled(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := newOperatorStore(t)
|
||||
if _, err := store.UpdateNotificationSettings(ctx, map[string]any{protocol.NotifyNtfyEnabled: false}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub := &chatHub{notes: newAppNotifier(store), settings: store}
|
||||
if ok, diagnostic := hub.publishToolNotification(ctx, "session-1", "Build failed", "Please inspect the run"); !ok || diagnostic != "" {
|
||||
t.Fatalf("tool notification result = %v, %q", ok, diagnostic)
|
||||
}
|
||||
active, _, err := store.ActiveAppNotifications(ctx, "", time.Now())
|
||||
if err != nil || len(active) != 1 {
|
||||
t.Fatalf("app notifications = %+v, %v", active, err)
|
||||
}
|
||||
if active[0].Kind != protocol.NotificationChatReply || active[0].Title != "Build failed" || active[0].Message != "Please inspect the run" {
|
||||
t.Fatalf("wrong tool notification: %+v", active[0])
|
||||
}
|
||||
}
|
||||
|
||||
// failRun creates a "Notify me" run and fails it, which commits a durable run
|
||||
// notification intent in the same transaction, then drains the outbox.
|
||||
func (f *appNotificationFixture) failRun(name string) int64 {
|
||||
|
||||
@@ -2,6 +2,8 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
@@ -12,6 +14,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
@@ -128,14 +131,20 @@ type chatToolNotifier func(ctx context.Context, sessionID, title, message string
|
||||
// ntfy destination (the one chat attention uses), honoring their ntfy switch.
|
||||
// Tapping it opens the chat (pipelines://chat/<id>).
|
||||
func (h *chatHub) publishToolNotification(ctx context.Context, sessionID, title, message string) (bool, string) {
|
||||
if !chatAttentionConfigured() {
|
||||
return false, "phone notifications (ntfy) are not configured on this Pipelines server"
|
||||
}
|
||||
if !chatNotificationSettings(ctx, h.settings).NtfyEnabled {
|
||||
return false, "the operator has turned phone notifications off"
|
||||
}
|
||||
title = truncateUTF8(title, 256)
|
||||
message = truncateUTF8(message, 2000)
|
||||
click, views := chatAttentionLinks(sessionID)
|
||||
if _, err := chatAttentionPublish(ctx, "system", truncateUTF8(title, 256), truncateUTF8(message, 2000), "", nil, click, views...); err != nil {
|
||||
digest := sha256.Sum256([]byte(title + "\x00" + message))
|
||||
if h.notes != nil {
|
||||
h.notes.record(ctx, pipeline.AppNotificationRecord{
|
||||
AppNotification: protocol.AppNotification{ID: "chat:" + sessionID + ":notify:" + hex.EncodeToString(digest[:8]), Kind: protocol.NotificationChatReply, Title: title, Message: message, Click: click, Priority: protocol.DefaultNotificationPriority, Tags: []string{}, Group: "chat:" + sessionID},
|
||||
SessionID: sessionID,
|
||||
})
|
||||
}
|
||||
if !chatAttentionConfigured() || !chatNotificationSettings(ctx, h.settings).NtfyEnabled {
|
||||
return true, ""
|
||||
}
|
||||
if _, err := chatAttentionPublish(ctx, "system", title, message, "", nil, click, views...); err != nil {
|
||||
log.Printf("chat tool notification session=%s: %v", sessionID, err)
|
||||
return false, "the notification service refused it: " + err.Error()
|
||||
}
|
||||
|
||||
@@ -229,8 +229,8 @@ func runAttentionEvent(status string) string {
|
||||
switch strings.TrimSpace(strings.ToLower(status)) {
|
||||
case protocol.RunStatusWaiting:
|
||||
return protocol.NotifyWaiting
|
||||
case "failed":
|
||||
return "failed"
|
||||
case "failed", "interrupted":
|
||||
return strings.TrimSpace(strings.ToLower(status))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -375,9 +375,9 @@ func (r *runner) notifyRun(id int64, name, status string, sequence int, currentS
|
||||
if event == "" {
|
||||
return
|
||||
}
|
||||
// A run's end notifies only runs marked "on", unless the operator opted
|
||||
// into every run end (run_finished); waiting notifies any non-off run.
|
||||
if event != protocol.NotifyWaiting && r.notifyMode(id) != "on" {
|
||||
// Failed or interrupted runs always notify; successful runs notify only when
|
||||
// marked "on" or when the operator opted into every run end (run_finished).
|
||||
if event == "succeeded" || (event != protocol.NotifyWaiting && event != "failed" && event != "interrupted") {
|
||||
settings, err := r.store.NotificationSettings(r.ctx)
|
||||
if err != nil || !settings.RunFinished {
|
||||
return
|
||||
@@ -392,6 +392,10 @@ func (r *runner) notifyRun(id int64, name, status string, sequence int, currentS
|
||||
title = "Run failed"
|
||||
summary = fmt.Sprintf("Run #%d · %s · run failed", id, name)
|
||||
}
|
||||
if event == "interrupted" {
|
||||
title = "Run interrupted"
|
||||
summary = fmt.Sprintf("Run #%d · %s · run interrupted", id, name)
|
||||
}
|
||||
if event == "succeeded" {
|
||||
title = "Run succeeded"
|
||||
summary = fmt.Sprintf("Run #%d · %s · run succeeded", id, name)
|
||||
|
||||
@@ -278,7 +278,7 @@ func TestRunAttentionEventMapping(t *testing.T) {
|
||||
{"running", ""},
|
||||
{"succeeded", ""},
|
||||
{"canceled", ""},
|
||||
{"interrupted", ""},
|
||||
{"interrupted", "interrupted"},
|
||||
{"", ""},
|
||||
} {
|
||||
if got := runAttentionEvent(test.status); got != test.want {
|
||||
@@ -320,7 +320,7 @@ func TestRunAttentionNotificationDelivery(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A run that needs nobody publishes nothing.
|
||||
for _, status := range []string{"queued", "running", "succeeded", "canceled", "interrupted"} {
|
||||
for _, status := range []string{"queued", "running", "succeeded", "canceled"} {
|
||||
r.notifyRun(7, "release", status, 1, "build", "")
|
||||
}
|
||||
select {
|
||||
@@ -332,7 +332,7 @@ func TestRunAttentionNotificationDelivery(t *testing.T) {
|
||||
r.notifyRun(7, "release", protocol.RunStatusWaiting, 1, "approve", "")
|
||||
// A run that stays waiting is never republished, on a later call or a re-request.
|
||||
r.notifyRun(7, "release", protocol.RunStatusWaiting, 1, "approve", "")
|
||||
// An auto run's failure is silent: only runs marked "on" announce their end.
|
||||
// An auto run's failure announces its end.
|
||||
r.notifyRun(8, "release", "failed", 2, "deploy", "provider returned 500")
|
||||
// A failed run marked "on" publishes once, with its diagnostic.
|
||||
r.setNotifyMode(7, "on")
|
||||
@@ -352,12 +352,16 @@ func TestRunAttentionNotificationDelivery(t *testing.T) {
|
||||
t.Fatalf("notification leaked an authorization header: %q", first.authorization)
|
||||
}
|
||||
second := <-deliveries
|
||||
if second.title != "Run failed" || !strings.Contains(second.message, "Run #7") || !strings.Contains(second.message, "release") || !strings.Contains(second.message, "deploy") || !strings.Contains(second.message, "provider returned 500") {
|
||||
if second.title != "Run failed" || !strings.Contains(second.message, "Run #8") {
|
||||
t.Fatalf("failed notification is not actionable: %+v", second)
|
||||
}
|
||||
if second.click != "pipelines://run/7" || second.actions != "view, Open in browser, https://pipelines.example.com/runs/7" {
|
||||
if second.click != "pipelines://run/8" || second.actions != "view, Open in browser, https://pipelines.example.com/runs/8" {
|
||||
t.Fatalf("failed notification click does not open the run: %q", second.click)
|
||||
}
|
||||
third := <-deliveries
|
||||
if third.title != "Run failed" || !strings.Contains(third.message, "Run #7") {
|
||||
t.Fatalf("marked failed notification is not actionable: %+v", third)
|
||||
}
|
||||
select {
|
||||
case got := <-deliveries:
|
||||
t.Fatalf("third notification delivered for two genuine events: %+v", got)
|
||||
|
||||
@@ -334,16 +334,15 @@ func TestRunNotificationAttentionModesAndRetryBound(t *testing.T) {
|
||||
r.notifyRun(id, "test", status, sequence, "stage", "")
|
||||
}
|
||||
}
|
||||
// auto waiting, on waiting and on failed; an auto run's failure is silent
|
||||
// because run_finished defaults off.
|
||||
if calls.Load() != 3 {
|
||||
t.Fatalf("attention modes: %d calls, want 3", calls.Load())
|
||||
// auto and on waiting, plus auto and on failed; off never notifies.
|
||||
if calls.Load() != 4 {
|
||||
t.Fatalf("attention modes: %d calls, want 4", calls.Load())
|
||||
}
|
||||
r.deliverRunNotifications(time.Now().Add(time.Minute))
|
||||
r.deliverRunNotifications(time.Now().Add(2 * time.Minute))
|
||||
r.deliverRunNotifications(time.Now().Add(3 * time.Minute))
|
||||
if calls.Load() != 9 {
|
||||
t.Fatalf("retry bound: %d calls, want 9", calls.Load())
|
||||
if calls.Load() != 12 {
|
||||
t.Fatalf("retry bound: %d calls, want 12", calls.Load())
|
||||
}
|
||||
for _, id := range []int64{1, 2, 3} {
|
||||
got, err := s.RunNotifications(ctx, id)
|
||||
|
||||
Reference in New Issue
Block a user