191 lines
8.2 KiB
Go
191 lines
8.2 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// RunNotification is durable publisher evidence, not proof of device reception.
|
|
// A sending lease permits restart recovery; a crash after publish but before
|
|
// acknowledgement can repeat a message. Attempts are bounded, not guaranteed.
|
|
type RunNotification struct {
|
|
RunID int64
|
|
Sequence int
|
|
Event, Channel, Title, Body, Click, Status, Diagnostic, MessageID string
|
|
Attempts int
|
|
}
|
|
|
|
// enqueueRunTransitionNotification must share the state mutation's transaction.
|
|
// Read the committed-to-be state, not a runner's proposed outcome or identity.
|
|
func enqueueRunTransitionNotification(ctx context.Context, tx *sql.Tx, id int64) error {
|
|
var mode, status, name, stage, diagnostic, graphJSON string
|
|
var sequence int
|
|
if err := tx.QueryRowContext(ctx, `SELECT r.notify_mode,r.status,p.name,r.current_stage,r.diagnostic,
|
|
COALESCE((SELECT MAX(sequence) FROM run_steps WHERE run_id=r.id),0),v.graph_json
|
|
FROM pipeline_runs r JOIN pipelines p ON p.id=r.pipeline_id
|
|
JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&mode, &status, &name, &stage, &diagnostic, &sequence, &graphJSON); err != nil {
|
|
return err
|
|
}
|
|
if mode == "off" {
|
|
return nil
|
|
}
|
|
var title, body string
|
|
switch status {
|
|
case "waiting":
|
|
title = "Run needs your decision"
|
|
body = "waiting for a human decision"
|
|
case "failed":
|
|
title = "Run failed"
|
|
body = "run failed"
|
|
case "succeeded":
|
|
if mode != "on" {
|
|
return nil
|
|
}
|
|
title = "Run succeeded"
|
|
body = "run succeeded"
|
|
default:
|
|
return nil
|
|
}
|
|
body = fmt.Sprintf("Run #%d · %s · %s", id, name, body)
|
|
if stage != "" {
|
|
body += " at stage " + stage
|
|
}
|
|
if diagnostic != "" {
|
|
body += ": " + diagnostic
|
|
}
|
|
body = strings.NewReplacer("\r", " ", "\n", " ", "\x00", " ").Replace(body)
|
|
if len(body) > 2000 {
|
|
body = body[:2000]
|
|
for !utf8.ValidString(body) {
|
|
body = body[:len(body)-1]
|
|
}
|
|
}
|
|
// Give a declared stage hook first opportunity to send its artifacts. Its
|
|
// initial claim bypasses this delay. If the process dies before that callback,
|
|
// the generic durable alert becomes eligible after one minute, not lost.
|
|
var graph protocol.Graph
|
|
if err := json.Unmarshal([]byte(graphJSON), &graph); err != nil {
|
|
return err
|
|
}
|
|
stageEvent := status
|
|
if status == "succeeded" {
|
|
stageEvent = "success"
|
|
}
|
|
if status == "failed" {
|
|
stageEvent = "failure"
|
|
}
|
|
var nextAttempt int64
|
|
for _, definition := range graph.Stages {
|
|
if definition.Key != stage || definition.Notify == nil {
|
|
continue
|
|
}
|
|
for _, event := range definition.Notify.On {
|
|
if event == stageEvent {
|
|
nextAttempt = time.Now().Add(time.Minute).Unix()
|
|
}
|
|
}
|
|
}
|
|
_, err := tx.ExecContext(ctx, `INSERT INTO run_notifications(run_id,sequence,event,channel,title,body,click,next_attempt)
|
|
VALUES (?,?,?,'system',?,?,?,?) ON CONFLICT(run_id,sequence,event,channel) DO NOTHING`, id, sequence, status, title, body, fmt.Sprintf("pipelines://run/%d", id), nextAttempt)
|
|
return err
|
|
}
|
|
|
|
// ClaimStageRunNotification arbitrates a transition hook against the outbox worker.
|
|
// With no transactional intent (legacy/direct stage callers), it leaves the
|
|
// stage's existing publication path alone. Otherwise the same lease protects
|
|
// both publishers, so only one may publish this event.
|
|
func (s *Store) ClaimStageRunNotification(ctx context.Context, id int64, sequence int, event string, now int64) (*RunNotification, bool, error) {
|
|
n := &RunNotification{RunID: id, Sequence: sequence, Event: event, Channel: "system"}
|
|
err := s.db.QueryRowContext(ctx, `UPDATE run_notifications SET status='sending',attempts=attempts+1,next_attempt=?
|
|
WHERE run_id=? AND sequence=? AND event=? AND channel='system' AND status='pending' AND attempts<3 AND (attempts=0 OR next_attempt<=?)
|
|
RETURNING attempts`, now+60, id, sequence, event, now).Scan(&n.Attempts)
|
|
if err == nil {
|
|
return n, true, nil
|
|
}
|
|
if !errors.Is(err, sql.ErrNoRows) {
|
|
return nil, false, err
|
|
}
|
|
var exists bool
|
|
err = s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM run_notifications WHERE run_id=? AND sequence=? AND event=? AND channel='system')`, id, sequence, event).Scan(&exists)
|
|
return nil, !exists, err
|
|
}
|
|
|
|
func (s *Store) initRunNotifications() error {
|
|
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS run_notifications (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, event TEXT NOT NULL, channel TEXT NOT NULL,
|
|
title TEXT NOT NULL, body TEXT NOT NULL, click TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'pending', attempts INTEGER NOT NULL DEFAULT 0,
|
|
next_attempt INTEGER NOT NULL DEFAULT 0, diagnostic TEXT NOT NULL DEFAULT '', message_id TEXT NOT NULL DEFAULT '',
|
|
PRIMARY KEY(run_id,sequence,event,channel));
|
|
CREATE INDEX IF NOT EXISTS run_notifications_due ON run_notifications(status,next_attempt);`)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) EnqueueRunNotification(ctx context.Context, n RunNotification) error {
|
|
_, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO run_notifications
|
|
(run_id,sequence,event,channel,title,body,click) VALUES (?,?,?,?,?,?,?)`, n.RunID, n.Sequence, n.Event, n.Channel, n.Title, n.Body, n.Click)
|
|
return err
|
|
}
|
|
|
|
// ClaimRunNotification atomically leases one due attempt for 60 seconds. Three
|
|
// attempts total bound both transient failures and crash/restart loops.
|
|
func (s *Store) ClaimRunNotification(ctx context.Context, now int64) (*RunNotification, error) {
|
|
if _, err := s.db.ExecContext(ctx, `UPDATE run_notifications SET status='failed',diagnostic='publisher acknowledgement unknown after final lease expired' WHERE status='sending' AND attempts>=3 AND next_attempt<=?`, now); err != nil {
|
|
return nil, err
|
|
}
|
|
n := new(RunNotification)
|
|
err := s.db.QueryRowContext(ctx, `UPDATE run_notifications SET status='sending', attempts=attempts+1, next_attempt=?
|
|
WHERE rowid=(SELECT rowid FROM run_notifications WHERE status IN ('pending','sending') AND attempts<3 AND next_attempt<=? ORDER BY next_attempt,rowid LIMIT 1)
|
|
RETURNING run_id,sequence,event,channel,title,body,click,status,attempts`, now+60, now).Scan(&n.RunID, &n.Sequence, &n.Event, &n.Channel, &n.Title, &n.Body, &n.Click, &n.Status, &n.Attempts)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return nil, nil
|
|
}
|
|
return n, err
|
|
}
|
|
|
|
// CoverRunNotification records a successful stage publication covering the same
|
|
// run event, without publishing a second final alert.
|
|
func (s *Store) CoverRunNotification(ctx context.Context, id int64, sequence int, event string) error {
|
|
_, err := s.db.ExecContext(ctx, `INSERT INTO run_notifications(run_id,sequence,event,channel,title,body,click,status) VALUES (?,?,?,'system','','','','covered')
|
|
ON CONFLICT(run_id,sequence,event,channel) DO UPDATE SET status='covered',diagnostic='' WHERE run_notifications.status IN ('pending','sending')`, id, sequence, event)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) FinishRunNotification(ctx context.Context, n RunNotification, now int64, messageID string, failure error) error {
|
|
status, diagnostic := "sent", ""
|
|
if failure != nil {
|
|
status, diagnostic = "pending", failure.Error()
|
|
if n.Attempts >= 3 {
|
|
status = "failed"
|
|
}
|
|
}
|
|
_, err := s.db.ExecContext(ctx, `UPDATE run_notifications SET status=?,diagnostic=?,message_id=?,next_attempt=?,click=?
|
|
WHERE run_id=? AND sequence=? AND event=? AND channel=? AND status='sending' AND attempts=?`, status, diagnostic, messageID, now+int64(n.Attempts*5), n.Click, n.RunID, n.Sequence, n.Event, n.Channel, n.Attempts)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) RunNotifications(ctx context.Context, id int64) ([]RunNotification, error) {
|
|
rows, err := s.db.QueryContext(ctx, `SELECT run_id,sequence,event,channel,status,attempts,diagnostic,message_id FROM run_notifications WHERE run_id=? ORDER BY sequence,event,channel`, id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
result := []RunNotification{}
|
|
for rows.Next() {
|
|
var n RunNotification
|
|
if err := rows.Scan(&n.RunID, &n.Sequence, &n.Event, &n.Channel, &n.Status, &n.Attempts, &n.Diagnostic, &n.MessageID); err != nil {
|
|
return nil, err
|
|
}
|
|
result = append(result, n)
|
|
}
|
|
return result, rows.Err()
|
|
}
|