fix: persist and retry run alerts and honor notification modes
This commit is contained in:
+80
-14
@@ -3,6 +3,7 @@ package server
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
"io"
|
||||
@@ -34,10 +35,6 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
|
||||
if !enabled {
|
||||
return
|
||||
}
|
||||
// The stage's own policy already covers this attention event, so the
|
||||
// run-level default must not repeat it. Recorded before delivery so a failed
|
||||
// stage notification is reported by its own receipt, not silently duplicated.
|
||||
r.markAttentionSent(id, step.Sequence, stageAttentionEvent(event))
|
||||
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
summary := fmt.Sprintf("%s · run #%d · stage %s · %s (%s)", name, id, step.Stage, event, step.Outcome)
|
||||
@@ -128,6 +125,18 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
|
||||
if !sentArtifact {
|
||||
send("", nil)
|
||||
}
|
||||
// Only a successful publish covers the run alert. A failed stage publisher
|
||||
// must not swallow the durable run-level fallback and its retries.
|
||||
for _, receipt := range receipts {
|
||||
if receipt.Status == "sent" {
|
||||
covered := stageAttentionEvent(event)
|
||||
if event == "success" && step.Target == protocol.SuccessEnd {
|
||||
covered = "succeeded"
|
||||
}
|
||||
r.markAttentionSent(id, step.Sequence, covered)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runAttentionEvent maps a run status to the operator attention notification it
|
||||
@@ -167,8 +176,16 @@ func (r *runner) markAttentionSent(id int64, sequence int, event string) {
|
||||
if event == "" {
|
||||
return
|
||||
}
|
||||
if r.store != nil {
|
||||
if err := r.store.CoverRunNotification(r.ctx, id, sequence, event); err != nil {
|
||||
log.Printf("record stage notification coverage run=%d: %v", id, err)
|
||||
}
|
||||
}
|
||||
r.attentionMu.Lock()
|
||||
defer r.attentionMu.Unlock()
|
||||
if r.attentionSent == nil {
|
||||
r.attentionSent = map[int64]map[string]bool{}
|
||||
}
|
||||
if r.attentionSent[id] == nil {
|
||||
r.attentionSent[id] = map[string]bool{}
|
||||
}
|
||||
@@ -206,6 +223,9 @@ func (r *runner) notifyMode(id int64) string {
|
||||
if r.store != nil {
|
||||
if stored, err := r.store.RunNotifyMode(r.ctx, id); err == nil && stored != "" {
|
||||
r.notifyMu.Lock()
|
||||
if r.notifyModes == nil {
|
||||
r.notifyModes = map[int64]string{}
|
||||
}
|
||||
r.notifyModes[id] = stored
|
||||
r.notifyMu.Unlock()
|
||||
return stored
|
||||
@@ -240,29 +260,32 @@ func runAttentionClick(id int64) string {
|
||||
// notifyStep and reuses the same PublishNtfy machinery, destination and token
|
||||
// handling, so no second publisher, topic or service is introduced. The run id
|
||||
// and pipeline name identify the run and the click action opens it in the app.
|
||||
// A run that needs nobody produces nothing, and a delivery failure is logged and
|
||||
// never fails the run: this is a best-effort alert, not persisted run evidence.
|
||||
// Attempts are persisted before publication. Publisher acknowledgement is not
|
||||
// proof of device reception; a crash in that acknowledgement window can repeat.
|
||||
func (r *runner) notifyRun(id int64, name, status string, sequence int, currentStage, diagnostic string) {
|
||||
if r.notifyMode(id) == "off" {
|
||||
return
|
||||
}
|
||||
event := runAttentionEvent(status)
|
||||
if status == "succeeded" && r.notifyMode(id) == "on" {
|
||||
event = "succeeded"
|
||||
}
|
||||
if event == "" {
|
||||
return
|
||||
}
|
||||
if r.attentionAlready(id, sequence, event) {
|
||||
return
|
||||
}
|
||||
// Record the delivery attempt before publishing so a repeated transition for
|
||||
// the same run and event (a re-request, a restart-free re-entry) never
|
||||
// republishes; delivery itself is never retried.
|
||||
r.markAttentionSent(id, sequence, event)
|
||||
title := "Run needs your decision"
|
||||
summary := fmt.Sprintf("Run #%d · %s · waiting for a human decision", id, name)
|
||||
if event == "failed" {
|
||||
title = "Run failed"
|
||||
summary = fmt.Sprintf("Run #%d · %s · run failed", id, name)
|
||||
}
|
||||
if event == "succeeded" {
|
||||
title = "Run succeeded"
|
||||
summary = fmt.Sprintf("Run #%d · %s · run succeeded", id, name)
|
||||
}
|
||||
if currentStage != "" {
|
||||
summary += " at stage " + currentStage
|
||||
}
|
||||
@@ -270,10 +293,53 @@ func (r *runner) notifyRun(id int64, name, status string, sequence int, currentS
|
||||
summary += ": " + diagnostic
|
||||
}
|
||||
summary = truncateUTF8(summary, 2000)
|
||||
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := stage.PublishNtfy(ctx, "system", title, summary, "", nil, runAttentionClick(id)); err != nil {
|
||||
log.Printf("run attention notification run=%d: %v", id, err)
|
||||
if err := r.store.EnqueueRunNotification(r.ctx, pipeline.RunNotification{RunID: id, Sequence: sequence, Event: event, Channel: "system", Title: title, Body: summary, Click: runAttentionClick(id)}); err != nil {
|
||||
log.Printf("enqueue run notification run=%d: %v", id, err)
|
||||
return
|
||||
}
|
||||
r.deliverRunNotificationBatch(time.Now(), 1)
|
||||
}
|
||||
|
||||
// resumeRunNotifications also recovers pending/expired leases after restart.
|
||||
func (r *runner) resumeRunNotifications() {
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-r.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
r.deliverRunNotifications(time.Now())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *runner) deliverRunNotifications(now time.Time) {
|
||||
r.deliverRunNotificationBatch(now, 32)
|
||||
}
|
||||
|
||||
func (r *runner) deliverRunNotificationBatch(now time.Time, limit int) {
|
||||
// Bound one drain so a queue cannot monopolize the runner.
|
||||
started := time.Now()
|
||||
for i := 0; i < limit; i++ {
|
||||
n, err := r.store.ClaimRunNotification(r.ctx, now.Add(time.Since(started)).Unix())
|
||||
if err != nil {
|
||||
log.Printf("claim run notification: %v", err)
|
||||
return
|
||||
}
|
||||
if n == nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
|
||||
result, publishErr := stage.PublishNtfy(ctx, n.Channel, n.Title, n.Body, "", nil, n.Click)
|
||||
cancel()
|
||||
messageID, _ := result["message_id"].(string)
|
||||
save, done := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
err = r.store.FinishRunNotification(save, *n, now.Add(time.Since(started)).Unix(), messageID, publishErr)
|
||||
done()
|
||||
if err != nil {
|
||||
log.Printf("save run notification run=%d: %v", n.RunID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user