Files
pipelines/internal/pipeline/run_notifications.go
T
alan b109ab3e06 Notify on run end only for runs marked Notify me
Auto runs no longer alert on failure or success; only notify_mode on does,
unless the run_finished setting (now 'Notify when any run ends', default off)
opts into every run end. Waiting still notifies auto and on runs. The setting
is stored under a new key so a value saved under the old meaning is ignored.
2026-10-31 16:52:04 +00:00

196 lines
8.4 KiB
Go

package pipeline
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"unicode/utf8"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// RunNotification is durable publisher evidence, not proof of device reception.
// A sending lease permits restart recovery; a crash after publish but before
// acknowledgement can repeat a message. Attempts are bounded, not guaranteed.
type RunNotification struct {
RunID int64
Sequence int
Event, Channel, Title, Body, Click, Status, Diagnostic, MessageID string
Attempts int
}
// enqueueRunTransitionNotification must share the state mutation's transaction.
// Read the committed-to-be state, not a runner's proposed outcome or identity.
func enqueueRunTransitionNotification(ctx context.Context, tx *sql.Tx, id int64) error {
var mode, status, name, stage, diagnostic, graphJSON string
var sequence int
if err := tx.QueryRowContext(ctx, `SELECT r.notify_mode,r.status,p.name,r.current_stage,r.diagnostic,
COALESCE((SELECT MAX(sequence) FROM run_steps WHERE run_id=r.id),0),v.graph_json
FROM pipeline_runs r JOIN pipelines p ON p.id=r.pipeline_id
JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&mode, &status, &name, &stage, &diagnostic, &sequence, &graphJSON); err != nil {
return err
}
if mode == "off" {
return nil
}
var title, body string
switch status {
case "waiting":
title = "Run needs your decision"
body = "waiting for a human decision"
case "failed", "succeeded":
// A run's end notifies only for runs marked "on", unless the operator
// opted into every run end via run_finished, read in this transaction.
if mode != "on" {
settings, err := readNotificationSettings(ctx, tx)
if err != nil {
return err
}
if !settings.RunFinished {
return nil
}
}
title = "Run " + status
body = "run " + status
default:
return nil
}
body = fmt.Sprintf("Run #%d · %s · %s", id, name, body)
if stage != "" {
body += " at stage " + stage
}
if diagnostic != "" {
body += ": " + diagnostic
}
body = strings.NewReplacer("\r", " ", "\n", " ", "\x00", " ").Replace(body)
if len(body) > 2000 {
body = body[:2000]
for !utf8.ValidString(body) {
body = body[:len(body)-1]
}
}
// Give a declared stage hook first opportunity to send its artifacts. Its
// initial claim bypasses this delay. If the process dies before that callback,
// the generic durable alert becomes eligible after one minute, not lost.
var graph protocol.Graph
if err := json.Unmarshal([]byte(graphJSON), &graph); err != nil {
return err
}
stageEvent := status
if status == "succeeded" {
stageEvent = "success"
}
if status == "failed" {
stageEvent = "failure"
}
var nextAttempt int64
for _, definition := range graph.Stages {
if definition.Key != stage || definition.Notify == nil {
continue
}
for _, event := range definition.Notify.On {
if event == stageEvent {
nextAttempt = time.Now().Add(time.Minute).Unix()
}
}
}
_, err := tx.ExecContext(ctx, `INSERT INTO run_notifications(run_id,sequence,event,channel,title,body,click,next_attempt)
VALUES (?,?,?,'system',?,?,?,?) ON CONFLICT(run_id,sequence,event,channel) DO NOTHING`, id, sequence, status, title, body, fmt.Sprintf("pipelines://run/%d", id), nextAttempt)
return err
}
// ClaimStageRunNotification arbitrates a transition hook against the outbox worker.
// With no transactional intent (legacy/direct stage callers), it leaves the
// stage's existing publication path alone. Otherwise the same lease protects
// both publishers, so only one may publish this event.
func (s *Store) ClaimStageRunNotification(ctx context.Context, id int64, sequence int, event string, now int64) (*RunNotification, bool, error) {
n := &RunNotification{RunID: id, Sequence: sequence, Event: event, Channel: "system"}
err := s.db.QueryRowContext(ctx, `UPDATE run_notifications SET status='sending',attempts=attempts+1,next_attempt=?
WHERE run_id=? AND sequence=? AND event=? AND channel='system' AND status='pending' AND attempts<3 AND (attempts=0 OR next_attempt<=?)
RETURNING attempts`, now+60, id, sequence, event, now).Scan(&n.Attempts)
if err == nil {
return n, true, nil
}
if !errors.Is(err, sql.ErrNoRows) {
return nil, false, err
}
var exists bool
err = s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM run_notifications WHERE run_id=? AND sequence=? AND event=? AND channel='system')`, id, sequence, event).Scan(&exists)
return nil, !exists, err
}
func (s *Store) initRunNotifications() error {
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS run_notifications (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, event TEXT NOT NULL, channel TEXT NOT NULL,
title TEXT NOT NULL, body TEXT NOT NULL, click TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending', attempts INTEGER NOT NULL DEFAULT 0,
next_attempt INTEGER NOT NULL DEFAULT 0, diagnostic TEXT NOT NULL DEFAULT '', message_id TEXT NOT NULL DEFAULT '',
PRIMARY KEY(run_id,sequence,event,channel));
CREATE INDEX IF NOT EXISTS run_notifications_due ON run_notifications(status,next_attempt);`)
return err
}
func (s *Store) EnqueueRunNotification(ctx context.Context, n RunNotification) error {
_, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO run_notifications
(run_id,sequence,event,channel,title,body,click) VALUES (?,?,?,?,?,?,?)`, n.RunID, n.Sequence, n.Event, n.Channel, n.Title, n.Body, n.Click)
return err
}
// ClaimRunNotification atomically leases one due attempt for 60 seconds. Three
// attempts total bound both transient failures and crash/restart loops.
func (s *Store) ClaimRunNotification(ctx context.Context, now int64) (*RunNotification, error) {
if _, err := s.db.ExecContext(ctx, `UPDATE run_notifications SET status='failed',diagnostic='publisher acknowledgement unknown after final lease expired' WHERE status='sending' AND attempts>=3 AND next_attempt<=?`, now); err != nil {
return nil, err
}
n := new(RunNotification)
err := s.db.QueryRowContext(ctx, `UPDATE run_notifications SET status='sending', attempts=attempts+1, next_attempt=?
WHERE rowid=(SELECT rowid FROM run_notifications WHERE status IN ('pending','sending') AND attempts<3 AND next_attempt<=? ORDER BY next_attempt,rowid LIMIT 1)
RETURNING run_id,sequence,event,channel,title,body,click,status,attempts`, now+60, now).Scan(&n.RunID, &n.Sequence, &n.Event, &n.Channel, &n.Title, &n.Body, &n.Click, &n.Status, &n.Attempts)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
return n, err
}
// CoverRunNotification records a successful stage publication covering the same
// run event, without publishing a second final alert.
func (s *Store) CoverRunNotification(ctx context.Context, id int64, sequence int, event string) error {
_, err := s.db.ExecContext(ctx, `INSERT INTO run_notifications(run_id,sequence,event,channel,title,body,click,status) VALUES (?,?,?,'system','','','','covered')
ON CONFLICT(run_id,sequence,event,channel) DO UPDATE SET status='covered',diagnostic='' WHERE run_notifications.status IN ('pending','sending')`, id, sequence, event)
return err
}
func (s *Store) FinishRunNotification(ctx context.Context, n RunNotification, now int64, messageID string, failure error) error {
status, diagnostic := "sent", ""
if failure != nil {
status, diagnostic = "pending", failure.Error()
if n.Attempts >= 3 {
status = "failed"
}
}
_, err := s.db.ExecContext(ctx, `UPDATE run_notifications SET status=?,diagnostic=?,message_id=?,next_attempt=?,click=?
WHERE run_id=? AND sequence=? AND event=? AND channel=? AND status='sending' AND attempts=?`, status, diagnostic, messageID, now+int64(n.Attempts*5), n.Click, n.RunID, n.Sequence, n.Event, n.Channel, n.Attempts)
return err
}
func (s *Store) RunNotifications(ctx context.Context, id int64) ([]RunNotification, error) {
rows, err := s.db.QueryContext(ctx, `SELECT run_id,sequence,event,channel,status,attempts,diagnostic,message_id FROM run_notifications WHERE run_id=? ORDER BY sequence,event,channel`, id)
if err != nil {
return nil, err
}
defer rows.Close()
result := []RunNotification{}
for rows.Next() {
var n RunNotification
if err := rows.Scan(&n.RunID, &n.Sequence, &n.Event, &n.Channel, &n.Status, &n.Attempts, &n.Diagnostic, &n.MessageID); err != nil {
return nil, err
}
result = append(result, n)
}
return result, rows.Err()
}