fix: persist and retry run alerts and honor notification modes

This commit is contained in:
2026-10-20 15:42:21 +01:00
parent e4f48ecbb1
commit 016fcb552a
20 changed files with 697 additions and 41 deletions
+1
View File
@@ -621,6 +621,7 @@ func (s *Store) removeRetentionRuns(ctx context.Context, policy RetentionPolicy,
func deleteRetentionRows(ctx context.Context, tx *sql.Tx, ids []int64) error {
for _, table := range []struct{ name, column string }{
{"step_notifications", "run_id"},
{"run_notifications", "run_id"},
{"step_file_changes", "run_id"},
{"step_control", "run_id"},
{"step_artifacts", "run_id"},
+87
View File
@@ -0,0 +1,87 @@
package pipeline
import (
"context"
"database/sql"
"errors"
)
// RunNotification is durable publisher evidence, not proof of device reception.
// A sending lease permits restart recovery; a crash after publish but before
// acknowledgement can repeat a message. Attempts are bounded, not guaranteed.
type RunNotification struct {
RunID int64
Sequence int
Event, Channel, Title, Body, Click, Status, Diagnostic, MessageID string
Attempts int
}
func (s *Store) initRunNotifications() error {
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS run_notifications (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, event TEXT NOT NULL, channel TEXT NOT NULL,
title TEXT NOT NULL, body TEXT NOT NULL, click TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending', attempts INTEGER NOT NULL DEFAULT 0,
next_attempt INTEGER NOT NULL DEFAULT 0, diagnostic TEXT NOT NULL DEFAULT '', message_id TEXT NOT NULL DEFAULT '',
PRIMARY KEY(run_id,sequence,event,channel));
CREATE INDEX IF NOT EXISTS run_notifications_due ON run_notifications(status,next_attempt);`)
return err
}
func (s *Store) EnqueueRunNotification(ctx context.Context, n RunNotification) error {
_, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO run_notifications
(run_id,sequence,event,channel,title,body,click) VALUES (?,?,?,?,?,?,?)`, n.RunID, n.Sequence, n.Event, n.Channel, n.Title, n.Body, n.Click)
return err
}
// ClaimRunNotification atomically leases one due attempt for 60 seconds. Three
// attempts total bound both transient failures and crash/restart loops.
func (s *Store) ClaimRunNotification(ctx context.Context, now int64) (*RunNotification, error) {
if _, err := s.db.ExecContext(ctx, `UPDATE run_notifications SET status='failed',diagnostic='publisher acknowledgement unknown after final lease expired' WHERE status='sending' AND attempts>=3 AND next_attempt<=?`, now); err != nil {
return nil, err
}
n := new(RunNotification)
err := s.db.QueryRowContext(ctx, `UPDATE run_notifications SET status='sending', attempts=attempts+1, next_attempt=?
WHERE rowid=(SELECT rowid FROM run_notifications WHERE status IN ('pending','sending') AND attempts<3 AND next_attempt<=? ORDER BY next_attempt,rowid LIMIT 1)
RETURNING run_id,sequence,event,channel,title,body,click,status,attempts`, now+60, now).Scan(&n.RunID, &n.Sequence, &n.Event, &n.Channel, &n.Title, &n.Body, &n.Click, &n.Status, &n.Attempts)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
return n, err
}
// CoverRunNotification records a successful stage publication covering the same
// run event, without publishing a second final alert.
func (s *Store) CoverRunNotification(ctx context.Context, id int64, sequence int, event string) error {
_, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO run_notifications(run_id,sequence,event,channel,title,body,click,status) VALUES (?,?,?,'system','','','','covered')`, id, sequence, event)
return err
}
func (s *Store) FinishRunNotification(ctx context.Context, n RunNotification, now int64, messageID string, failure error) error {
status, diagnostic := "sent", ""
if failure != nil {
status, diagnostic = "pending", failure.Error()
if n.Attempts >= 3 {
status = "failed"
}
}
_, err := s.db.ExecContext(ctx, `UPDATE run_notifications SET status=?,diagnostic=?,message_id=?,next_attempt=?
WHERE run_id=? AND sequence=? AND event=? AND channel=? AND status='sending' AND attempts=?`, status, diagnostic, messageID, now+int64(n.Attempts*5), n.RunID, n.Sequence, n.Event, n.Channel, n.Attempts)
return err
}
func (s *Store) RunNotifications(ctx context.Context, id int64) ([]RunNotification, error) {
rows, err := s.db.QueryContext(ctx, `SELECT run_id,sequence,event,channel,status,attempts,diagnostic,message_id FROM run_notifications WHERE run_id=? ORDER BY sequence,event,channel`, id)
if err != nil {
return nil, err
}
defer rows.Close()
result := []RunNotification{}
for rows.Next() {
var n RunNotification
if err := rows.Scan(&n.RunID, &n.Sequence, &n.Event, &n.Channel, &n.Status, &n.Attempts, &n.Diagnostic, &n.MessageID); err != nil {
return nil, err
}
result = append(result, n)
}
return result, rows.Err()
}
+131
View File
@@ -0,0 +1,131 @@
package pipeline
import (
"context"
"errors"
"path/filepath"
"testing"
)
func TestRunNotificationPersistenceLeasesAndBound(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "outbox.db")
s, err := Open(path)
if err != nil {
t.Fatal(err)
}
n := RunNotification{RunID: 7, Sequence: 2, Event: "failed", Channel: "system", Title: "Failed", Body: "Run #7", Click: "pipelines://run/7"}
if err := s.EnqueueRunNotification(ctx, n); err != nil {
t.Fatal(err)
}
if err := s.EnqueueRunNotification(ctx, n); err != nil {
t.Fatal(err)
}
first, err := s.ClaimRunNotification(ctx, 100)
if err != nil || first == nil || first.Attempts != 1 {
t.Fatalf("first claim: %+v %v", first, err)
}
if other, err := s.ClaimRunNotification(ctx, 100); err != nil || other != nil {
t.Fatalf("lease double claim: %+v %v", other, err)
}
// Reopen an already-existing schema, preserving an in-flight lease.
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
second, err := s.ClaimRunNotification(ctx, 161)
if err != nil || second == nil || second.Attempts != 2 || second.Body != n.Body {
t.Fatalf("recovery: %+v %v", second, err)
}
if err := s.FinishRunNotification(ctx, *second, 161, "", errors.New("publisher unavailable")); err != nil {
t.Fatal(err)
}
if other, err := s.ClaimRunNotification(ctx, 170); err != nil || other != nil {
t.Fatalf("backoff ignored: %+v %v", other, err)
}
third, err := s.ClaimRunNotification(ctx, 171)
if err != nil || third == nil || third.Attempts != 3 {
t.Fatalf("third claim: %+v %v", third, err)
}
// Final attempt crashes: its expired lease must become failed, not retry forever.
if other, err := s.ClaimRunNotification(ctx, 232); err != nil || other != nil {
t.Fatalf("retry bound ignored: %+v %v", other, err)
}
got, err := s.RunNotifications(ctx, 7)
if err != nil || len(got) != 1 || got[0].Status != "failed" || got[0].Attempts != 3 {
t.Fatalf("audit: %+v %v", got, err)
}
}
func TestRunNotificationStageCoverageSurvivesReopen(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "coverage.db")
s, err := Open(path)
if err != nil {
t.Fatal(err)
}
if err := s.CoverRunNotification(ctx, 7, 1, "succeeded"); err != nil {
t.Fatal(err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
if err := s.EnqueueRunNotification(ctx, RunNotification{RunID: 7, Sequence: 1, Event: "succeeded", Channel: "system"}); err != nil {
t.Fatal(err)
}
if got, err := s.ClaimRunNotification(ctx, 100); err != nil || got != nil {
t.Fatalf("covered stage repeated: %+v %v", got, err)
}
}
func TestRunNotificationMigrationPreservesExistingRun(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "existing.db")
s, err := Open(path)
if err != nil {
t.Fatal(err)
}
if _, err := s.Submit(ctx, "existing", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, err := s.List(ctx)
if err != nil {
t.Fatal(err)
}
d, err := s.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
id, _, err := s.CreateRun(ctx, d, "existing-key", "workspace", false, nil, map[string]any{"prompt": "preserve me"})
if err != nil {
t.Fatal(err)
}
// Simulate the immediately preceding schema, which has no run outbox.
if _, err := s.db.ExecContext(ctx, "DROP TABLE run_notifications"); err != nil {
t.Fatal(err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
got, err := s.GetRun(ctx, id)
if err != nil || got.Input["prompt"] != "preserve me" || got.Status != "queued" || got.NotifyMode != "auto" || len(got.Notifications) != 0 {
t.Fatalf("migration changed run: %+v %v", got, err)
}
if err := s.EnqueueRunNotification(ctx, RunNotification{RunID: id, Sequence: 1, Event: "failed", Channel: "system"}); err != nil {
t.Fatal(err)
}
}
+19 -2
View File
@@ -21,12 +21,12 @@ const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
` + runStamps + `,r.diagnostic,
CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END,
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode
` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version`
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
var output, stageModels string
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels); err != nil {
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode); err != nil {
return err
}
if stageModels != "" && stageModels != "{}" {
@@ -201,6 +201,23 @@ strftime('%Y-%m-%dT%H:%M:%SZ',started_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ'
// The run total is one pass over the steps already loaded above: it adds no
// query and no transcript parse.
r.Usage = protocol.TotalUsage(r.Steps)
alerts, err := tx.QueryContext(ctx, `SELECT sequence,event,channel,status,attempts,diagnostic,message_id FROM run_notifications WHERE run_id=? ORDER BY sequence,event,channel`, id)
if err != nil {
return nil, err
}
for alerts.Next() {
var receipt protocol.RunNotificationReceipt
if err := alerts.Scan(&receipt.Sequence, &receipt.Event, &receipt.Channel, &receipt.Status, &receipt.Attempts, &receipt.Diagnostic, &receipt.MessageID); err != nil {
alerts.Close()
return nil, err
}
r.Notifications = append(r.Notifications, receipt)
}
err = alerts.Err()
alerts.Close()
if err != nil {
return nil, err
}
if err := tx.Commit(); err != nil {
return nil, err
}
+4
View File
@@ -54,6 +54,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU
db.Close()
return nil, err
}
if err := s.initRunNotifications(); err != nil {
db.Close()
return nil, err
}
if err := s.initSchedules(); err != nil {
db.Close()
return nil, err
+1 -1
View File
@@ -153,7 +153,7 @@ type Message struct {
// them; the same fields are echoed on the run record.
Model string `json:"model,omitempty"`
StageModels map[string]string `json:"stage_models,omitempty"`
// NotifyMode is auto (definition), on (ensure run-level attention) or off
// NotifyMode is auto (definition), on (terminal success/failure and attention) or off
// (silence this run). It is optional so older clients keep auto.
NotifyMode string `json:"notify_mode,omitempty"`
// Empty (or "resume") retries the failed visit; "restart" starts at the
+3 -2
View File
@@ -24,8 +24,9 @@ const (
// from the frame instead of re-reading the whole list. They repeat on every
// frame (pipeline identity never changes mid-run).
type RunEventFrame struct {
RunID int64 `json:"run_id"`
Workspace string `json:"workspace,omitempty"`
NotifyMode string `json:"notify_mode,omitempty"`
RunID int64 `json:"run_id"`
Workspace string `json:"workspace,omitempty"`
// PipelineID, Name, Title and Version identify the run's pipeline and its
// operator-facing title, so a new run's row can be built without a fetch.
PipelineID int64 `json:"pipeline_id,omitempty"`
+25 -12
View File
@@ -119,6 +119,7 @@ type SessionExport struct {
}
type RunSummary struct {
NotifyMode string `json:"notify_mode,omitempty"`
ID int64 `json:"id"`
PipelineID int64 `json:"pipeline_id"`
Name string `json:"name"`
@@ -349,18 +350,19 @@ type NotificationReceipt struct {
type RunDetail struct {
RunSummary
Graph Graph `json:"graph"`
RetryOf int64 `json:"retry_of,omitempty"`
RetryMode string `json:"retry_mode,omitempty"`
RetrySource int64 `json:"retry_source,omitempty"`
RetrySourceVersion int `json:"retry_source_version,omitempty"`
RetrySourceHash string `json:"retry_source_hash,omitempty"`
Workspace string `json:"workspace"`
AllowTools bool `json:"allow_tools"`
WritePaths []string `json:"write_paths"`
Input map[string]any `json:"input"`
Output map[string]any `json:"output"`
Steps []RunStep `json:"steps"`
Notifications []RunNotificationReceipt `json:"notifications,omitempty"`
Graph Graph `json:"graph"`
RetryOf int64 `json:"retry_of,omitempty"`
RetryMode string `json:"retry_mode,omitempty"`
RetrySource int64 `json:"retry_source,omitempty"`
RetrySourceVersion int `json:"retry_source_version,omitempty"`
RetrySourceHash string `json:"retry_source_hash,omitempty"`
Workspace string `json:"workspace"`
AllowTools bool `json:"allow_tools"`
WritePaths []string `json:"write_paths"`
Input map[string]any `json:"input"`
Output map[string]any `json:"output"`
Steps []RunStep `json:"steps"`
// Usage is the totalled reported usage of the run's model stages, summed in
// one pass over Steps. It is absent for a run with no model usage recorded
// (including historical runs).
@@ -372,3 +374,14 @@ type RunDetail struct {
TitleSource string `json:"title_source,omitempty"`
TitleModel string `json:"title_model,omitempty"`
}
// RunNotificationReceipt records publisher attempts, not device reception.
type RunNotificationReceipt struct {
Sequence int `json:"sequence"`
Event string `json:"event"`
Channel string `json:"channel"`
Status string `json:"status"`
Attempts int `json:"attempts"`
Diagnostic string `json:"diagnostic,omitempty"`
MessageID string `json:"message_id,omitempty"`
}
+80 -14
View File
@@ -3,6 +3,7 @@ package server
import (
"context"
"fmt"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"io"
@@ -34,10 +35,6 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
if !enabled {
return
}
// The stage's own policy already covers this attention event, so the
// run-level default must not repeat it. Recorded before delivery so a failed
// stage notification is reported by its own receipt, not silently duplicated.
r.markAttentionSent(id, step.Sequence, stageAttentionEvent(event))
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
defer cancel()
summary := fmt.Sprintf("%s · run #%d · stage %s · %s (%s)", name, id, step.Stage, event, step.Outcome)
@@ -128,6 +125,18 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
if !sentArtifact {
send("", nil)
}
// Only a successful publish covers the run alert. A failed stage publisher
// must not swallow the durable run-level fallback and its retries.
for _, receipt := range receipts {
if receipt.Status == "sent" {
covered := stageAttentionEvent(event)
if event == "success" && step.Target == protocol.SuccessEnd {
covered = "succeeded"
}
r.markAttentionSent(id, step.Sequence, covered)
break
}
}
}
// runAttentionEvent maps a run status to the operator attention notification it
@@ -167,8 +176,16 @@ func (r *runner) markAttentionSent(id int64, sequence int, event string) {
if event == "" {
return
}
if r.store != nil {
if err := r.store.CoverRunNotification(r.ctx, id, sequence, event); err != nil {
log.Printf("record stage notification coverage run=%d: %v", id, err)
}
}
r.attentionMu.Lock()
defer r.attentionMu.Unlock()
if r.attentionSent == nil {
r.attentionSent = map[int64]map[string]bool{}
}
if r.attentionSent[id] == nil {
r.attentionSent[id] = map[string]bool{}
}
@@ -206,6 +223,9 @@ func (r *runner) notifyMode(id int64) string {
if r.store != nil {
if stored, err := r.store.RunNotifyMode(r.ctx, id); err == nil && stored != "" {
r.notifyMu.Lock()
if r.notifyModes == nil {
r.notifyModes = map[int64]string{}
}
r.notifyModes[id] = stored
r.notifyMu.Unlock()
return stored
@@ -240,29 +260,32 @@ func runAttentionClick(id int64) string {
// notifyStep and reuses the same PublishNtfy machinery, destination and token
// handling, so no second publisher, topic or service is introduced. The run id
// and pipeline name identify the run and the click action opens it in the app.
// A run that needs nobody produces nothing, and a delivery failure is logged and
// never fails the run: this is a best-effort alert, not persisted run evidence.
// Attempts are persisted before publication. Publisher acknowledgement is not
// proof of device reception; a crash in that acknowledgement window can repeat.
func (r *runner) notifyRun(id int64, name, status string, sequence int, currentStage, diagnostic string) {
if r.notifyMode(id) == "off" {
return
}
event := runAttentionEvent(status)
if status == "succeeded" && r.notifyMode(id) == "on" {
event = "succeeded"
}
if event == "" {
return
}
if r.attentionAlready(id, sequence, event) {
return
}
// Record the delivery attempt before publishing so a repeated transition for
// the same run and event (a re-request, a restart-free re-entry) never
// republishes; delivery itself is never retried.
r.markAttentionSent(id, sequence, event)
title := "Run needs your decision"
summary := fmt.Sprintf("Run #%d · %s · waiting for a human decision", id, name)
if event == "failed" {
title = "Run failed"
summary = fmt.Sprintf("Run #%d · %s · run failed", id, name)
}
if event == "succeeded" {
title = "Run succeeded"
summary = fmt.Sprintf("Run #%d · %s · run succeeded", id, name)
}
if currentStage != "" {
summary += " at stage " + currentStage
}
@@ -270,10 +293,53 @@ func (r *runner) notifyRun(id int64, name, status string, sequence int, currentS
summary += ": " + diagnostic
}
summary = truncateUTF8(summary, 2000)
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
defer cancel()
if _, err := stage.PublishNtfy(ctx, "system", title, summary, "", nil, runAttentionClick(id)); err != nil {
log.Printf("run attention notification run=%d: %v", id, err)
if err := r.store.EnqueueRunNotification(r.ctx, pipeline.RunNotification{RunID: id, Sequence: sequence, Event: event, Channel: "system", Title: title, Body: summary, Click: runAttentionClick(id)}); err != nil {
log.Printf("enqueue run notification run=%d: %v", id, err)
return
}
r.deliverRunNotificationBatch(time.Now(), 1)
}
// resumeRunNotifications also recovers pending/expired leases after restart.
func (r *runner) resumeRunNotifications() {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-r.ctx.Done():
return
case <-ticker.C:
r.deliverRunNotifications(time.Now())
}
}
}
func (r *runner) deliverRunNotifications(now time.Time) {
r.deliverRunNotificationBatch(now, 32)
}
func (r *runner) deliverRunNotificationBatch(now time.Time, limit int) {
// Bound one drain so a queue cannot monopolize the runner.
started := time.Now()
for i := 0; i < limit; i++ {
n, err := r.store.ClaimRunNotification(r.ctx, now.Add(time.Since(started)).Unix())
if err != nil {
log.Printf("claim run notification: %v", err)
return
}
if n == nil {
return
}
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
result, publishErr := stage.PublishNtfy(ctx, n.Channel, n.Title, n.Body, "", nil, n.Click)
cancel()
messageID, _ := result["message_id"].(string)
save, done := context.WithTimeout(context.Background(), 5*time.Second)
err = r.store.FinishRunNotification(save, *n, now.Add(time.Since(started)).Unix(), messageID, publishErr)
done()
if err != nil {
log.Printf("save run notification run=%d: %v", n.RunID, err)
}
}
}
+10 -2
View File
@@ -121,7 +121,7 @@ func TestRunEventWebSocketPushesTransitions(t *testing.T) {
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "sub" {
t.Fatalf("subscribe ack = %+v", ack)
}
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, NotifyMode: "off", Input: map[string]any{"prompt": "test"}})
var runID int64
var created, stepStarted, stepFinished, terminal *protocol.RunEventFrame
@@ -175,6 +175,11 @@ func TestRunEventWebSocketPushesTransitions(t *testing.T) {
if terminal.RunID != runID || terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.FinishedAt == "" {
t.Fatalf("terminal frame = %+v", terminal)
}
for _, frame := range []*protocol.RunEventFrame{created, stepStarted, stepFinished, terminal} {
if frame.NotifyMode != "off" {
t.Fatalf("silent mode lost in event: %+v", frame)
}
}
}
// TestRunEventSubscribeSendsSnapshot proves a client that subscribes to an
@@ -188,7 +193,7 @@ func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
workspace := eventWorkspace(t)
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, NotifyMode: "off", Input: map[string]any{"prompt": "test"}})
if err != nil {
t.Fatal(err)
}
@@ -203,6 +208,9 @@ func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
if first.RunEvent.RunID != run.ID || first.RunEvent.Status != "succeeded" || first.RunEvent.CompletedSteps != 1 || first.RunEvent.FinishedAt == "" {
t.Fatalf("snapshot frame = %+v", first.RunEvent)
}
if first.RunEvent.NotifyMode != "off" {
t.Fatalf("silent mode lost in snapshot: %+v", first.RunEvent)
}
// A snapshot for a run the client never saw also carries the identity its row
// needs, so a fresh subscribe renders it without a second read.
if first.RunEvent.PipelineID != ids[0] || first.RunEvent.Name == "" || first.RunEvent.Version != 1 {
@@ -0,0 +1,221 @@
package server
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func TestRunNotificationPublisherFailureRetriesAfterReopen(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "retry.db")
s, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if calls.Add(1) == 1 {
http.Error(w, "transient", http.StatusServiceUnavailable)
return
}
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r := &runner{ctx: ctx, store: s}
r.notifyRun(7, "test", "failed", 1, "stage", "")
got, err := s.RunNotifications(ctx, 7)
if err != nil || len(got) != 1 || got[0].Status != "pending" || got[0].Attempts != 1 || got[0].Diagnostic == "" {
t.Fatalf("failed attempt not audited: %+v %v", got, err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
r = &runner{ctx: ctx, store: s}
r.deliverRunNotifications(time.Now().Add(time.Minute))
r.notifyRun(7, "test", "failed", 1, "stage", "")
got, err = s.RunNotifications(ctx, 7)
if err != nil || len(got) != 1 || got[0].Status != "sent" || got[0].Attempts != 2 || got[0].MessageID != "ack" || calls.Load() != 2 {
t.Fatalf("retry/dedupe: %+v calls=%d %v", got, calls.Load(), err)
}
}
func TestRunNotificationModesAndFinalStageDedupe(t *testing.T) {
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
for _, tc := range []struct {
mode string
hook bool
want int32
}{
{"on", false, 1}, {"auto", false, 0}, {"off", false, 0}, {"on", true, 1}, {"auto", true, 1}, {"off", true, 0},
} {
t.Run(tc.mode+map[bool]string{true: "-hook", false: "-no-hook"}[tc.hook], func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
s, err := pipeline.Open(filepath.Join(t.TempDir(), "modes.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
source := pipeline.Template([]string{"one"}, true)
if tc.hook {
source = strings.Replace(source, "# stage one", "# stage one\n```notify\non=success\n```", 1)
}
if _, err := s.Submit(ctx, "mode-test", source); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success"}, nil
}
before := calls.Load()
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mode", NotifyMode: tc.mode, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := s.GetRun(ctx, run.ID)
if err != nil || detail.Status != "succeeded" || detail.NotifyMode != tc.mode || calls.Load()-before != tc.want {
t.Fatalf("mode contract: %+v calls=%d %v", detail, calls.Load()-before, err)
}
if tc.mode == "on" && (len(detail.Notifications) != 1 || (detail.Notifications[0].Status != "sent" && detail.Notifications[0].Status != "covered")) {
t.Fatalf("missing final audit: %+v", detail.Notifications)
}
cancel()
r.schedulerWG.Wait()
})
}
}
func TestRunNotificationAttentionModesAndRetryBound(t *testing.T) {
ctx := context.Background()
s, err := pipeline.Open(filepath.Join(t.TempDir(), "attention.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
http.Error(w, "unavailable", http.StatusServiceUnavailable)
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r := &runner{ctx: ctx, store: s}
for i, mode := range []string{"auto", "on", "off"} {
id := int64(i + 1)
r.setNotifyMode(id, mode)
for sequence, status := range []string{"waiting", "failed"} {
r.notifyRun(id, "test", status, sequence, "stage", "")
}
}
if calls.Load() != 4 {
t.Fatalf("attention modes: %d calls, want 4", calls.Load())
}
r.deliverRunNotifications(time.Now().Add(time.Minute))
r.deliverRunNotifications(time.Now().Add(2 * time.Minute))
r.deliverRunNotifications(time.Now().Add(3 * time.Minute))
if calls.Load() != 12 {
t.Fatalf("retry bound: %d calls, want 12", calls.Load())
}
for _, id := range []int64{1, 2, 3} {
got, err := s.RunNotifications(ctx, id)
if err != nil {
t.Fatal(err)
}
if id == 3 && len(got) != 0 {
t.Fatalf("off created attempts: %+v", got)
}
for _, n := range got {
if n.Status != "failed" || n.Attempts != 3 {
t.Fatalf("missing exhausted audit: %+v", n)
}
}
}
}
func TestRunNotificationWorkerResumesPendingAfterRestart(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
path := filepath.Join(t.TempDir(), "worker.db")
s, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
if err := s.EnqueueRunNotification(ctx, pipeline.RunNotification{RunID: 7, Sequence: 1, Event: "failed", Channel: "system", Title: "Run failed", Body: "pending"}); err != nil {
t.Fatal(err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
delivered := make(chan struct{}, 2)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]string{"id": "recovered", "event": "message"})
delivered <- struct{}{}
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.schedulerWG.Wait() }()
select {
case <-delivered:
case <-time.After(5 * time.Second):
t.Fatal("pending outbox was not recovered")
}
deadline := time.Now().Add(5 * time.Second)
for {
got, err := s.RunNotifications(ctx, 7)
if err != nil {
t.Fatal(err)
}
if len(got) == 1 && got[0].Status == "sent" && got[0].MessageID == "recovered" {
break
}
if time.Now().After(deadline) {
t.Fatalf("recovery audit: %+v", got)
}
time.Sleep(10 * time.Millisecond)
}
}
+4
View File
@@ -424,6 +424,9 @@ func (s *runSubscription) close() {
}
func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
if in.NotifyMode != "" {
into.NotifyMode = in.NotifyMode
}
if in.Workspace != "" {
into.Workspace = in.Workspace
}
@@ -500,6 +503,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
continue
}
frame := protocol.RunEventFrame{
NotifyMode: summary.NotifyMode,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
+5 -1
View File
@@ -101,7 +101,10 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
return nil, err
}
root := filepath.Join(s.Directory(), "artifacts")
return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}, nil
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}
r.schedulerWG.Add(1)
go func() { defer r.schedulerWG.Done(); r.resumeRunNotifications() }()
return r, nil
}
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
@@ -1257,6 +1260,7 @@ func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) {
return
}
r.stampRunIdentity(&frame)
frame.NotifyMode = r.notifyMode(frame.RunID)
r.events.emit(frame, terminal)
}