diff --git a/docs/notes/audit-notifications.md b/docs/notes/audit-notifications.md index 4ca293dc..3e9ae601 100644 --- a/docs/notes/audit-notifications.md +++ b/docs/notes/audit-notifications.md @@ -6,6 +6,16 @@ notification hook. `off` suppresses stage, run-level system and browser tray notifications. Needs you remains a status/action surface, not a delivery channel. Summaries and pushed events carry the mode; the browser seeds and folds it. +`FinishRun` and `MarkWaiting` enqueue the notification intent in the same SQLite +transaction as the run/step state change. Enqueue failure rolls back the state; +after commit, the worker can recover the intent without any runner callback. +The store derives the mode, name, stage and sequence from stored rows. A failed +FinishRun does not emit a terminal-success event or run alert. Terminal stage +hooks run only after FinishRun commits. Waiting/terminal hooks and the worker +share an atomic sending lease to avoid racing publications. Declared hooks get +a one-minute initial opportunity to publish their artifacts; if a restart loses +the callback, the generic outbox alert becomes eligible after that delay. + `run_notifications` is an additive, idempotent SQLite migration. Each run/sequence/event/channel has one durable outbox row. Its status is `pending`, `sending`, `sent`, `failed`, or `covered` (a stage already published this event). @@ -22,9 +32,9 @@ publication does not swallow the run-level fallback/retry. Stage artifact publications themselves are not made retryable by this change. Publisher acknowledgement is **not device reception**. A crash after publish -but before the acknowledgement is saved can duplicate a notification. The -transition and enqueue are separate writes: a crash between them can miss an -alert; this is bounded retry of enqueued alerts, not guaranteed delivery. +but before the acknowledgement is saved can duplicate a notification. State and +intent commit together, closing the transition-to-enqueue crash window; this is +still bounded retry, not guaranteed delivery. Browser delivery remains page-session best effort and requires permission plus HTTPS/localhost; successful runs do not become Needs you entries. The older L5 parity description is attention-only for auto; on now adds success. diff --git a/internal/pipeline/run_notification_transition_test.go b/internal/pipeline/run_notification_transition_test.go new file mode 100644 index 00000000..9cea767a --- /dev/null +++ b/internal/pipeline/run_notification_transition_test.go @@ -0,0 +1,198 @@ +package pipeline + +import ( + "context" + "path/filepath" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func notificationTransitionRun(t *testing.T, path, mode string, sources ...string) (*Store, int64) { + t.Helper() + ctx := context.Background() + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + source := humanSource + if len(sources) > 0 { + source = sources[0] + } + if _, err := s.Submit(ctx, "stored pipeline", source); err != nil { + t.Fatal(err) + } + items, err := s.List(ctx) + if err != nil { + t.Fatal(err) + } + d, err := s.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + id, _, err := s.CreateRun(ctx, d, "transition", "workspace", false, nil, map[string]any{"prompt": "task"}) + if err != nil { + t.Fatal(err) + } + if _, err := s.db.ExecContext(ctx, `UPDATE pipeline_runs SET notify_mode=? WHERE id=?`, mode, id); err != nil { + t.Fatal(err) + } + if err := s.BeginStep(ctx, id, 1, "gate", map[string]any{"prompt": "task"}); err != nil { + t.Fatal(err) + } + return s, id +} + +func TestWaitingHookIntentRecoversWithoutPostCommitCallback(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "lost-hook.db") + source := strings.Replace(humanSource, "# stage gate", "# stage gate\n```notify\non=waiting\n```", 1) + s, id := notificationTransitionRun(t, path, "on", source) + if err := notificationTransition(ctx, s, id, "waiting"); err != nil { + t.Fatal(err) + } + if n, err := s.ClaimRunNotification(ctx, time.Now().Unix()); err != nil || n != nil { + t.Fatalf("worker raced hook opportunity: %+v %v", n, err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + n, err := s.ClaimRunNotification(ctx, time.Now().Add(time.Minute+time.Second).Unix()) + if err != nil || n == nil || n.RunID != id || n.Event != "waiting" { + t.Fatalf("lost hook orphaned waiting alert: %+v %v", n, err) + } +} + +func notificationTransition(ctx context.Context, s *Store, id int64, status string) error { + if status == "waiting" { + return s.MarkWaiting(ctx, id, 1, "gate", protocol.NewWaitingEvidence("Choose", []string{"deploy"})) + } + return s.FinishRun(ctx, id, status, "success", "stored diagnostic", map[string]any{"markdown": "result"}) +} + +func TestRunNotificationTransitionIntentAtomicRollbackAndRecovery(t *testing.T) { + for _, status := range []string{"succeeded", "failed", "waiting"} { + t.Run(status, func(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "atomic.db") + s, id := notificationTransitionRun(t, path, "on") + // Force enqueue to fail after both state writes would have happened. + if _, err := s.db.ExecContext(ctx, `CREATE TRIGGER reject_notification BEFORE INSERT ON run_notifications BEGIN SELECT RAISE(ABORT,'forced enqueue failure'); END`); err != nil { + t.Fatal(err) + } + if err := notificationTransition(ctx, s, id, status); err == nil || !strings.Contains(err.Error(), "forced enqueue failure") { + t.Fatalf("enqueue failure missing: %v", err) + } + got, err := s.GetRun(ctx, id) + if err != nil { + t.Fatal(err) + } + if got.Status != "running" || got.FinishedAt != "" || got.Diagnostic != "" || got.Outcome != "" || len(got.Output) != 0 || got.Steps[0].Status != "running" || len(got.Steps[0].Output) != 0 || len(got.Notifications) != 0 { + t.Fatalf("partial transition committed: %+v", got) + } + if _, err := s.db.ExecContext(ctx, `DROP TRIGGER reject_notification`); err != nil { + t.Fatal(err) + } + if err := notificationTransition(ctx, s, id, status); err != nil { + t.Fatal(err) + } + // Simulate a crash immediately after commit: no runner enqueue/delivery. + if err := s.Close(); err != nil { + t.Fatal(err) + } + s, err = Open(path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + got, err = s.GetRun(ctx, id) + if err != nil { + t.Fatal(err) + } + if got.Status != status || len(got.Notifications) != 1 || got.Notifications[0].Status != "pending" { + t.Fatalf("orphaned state after reopen: %+v", got) + } + n, err := s.ClaimRunNotification(ctx, 100) + if err != nil || n == nil { + t.Fatalf("intent not recoverable: %+v %v", n, err) + } + if n.RunID != id || n.Sequence != 1 || n.Event != status || n.Channel != "system" || !strings.Contains(n.Body, "stored pipeline") || !strings.Contains(n.Body, "gate") { + t.Fatalf("intent not derived from stored run: %+v", n) + } + }) + } +} + +func TestRunNotificationTransitionModesAndCoverage(t *testing.T) { + for _, mode := range []string{"auto", "on", "off"} { + for _, status := range []string{"succeeded", "failed", "waiting"} { + t.Run(mode+"/"+status, func(t *testing.T) { + ctx := context.Background() + s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "mode.db"), mode) + defer s.Close() + if err := notificationTransition(ctx, s, id, status); err != nil { + t.Fatal(err) + } + ns, err := s.RunNotifications(ctx, id) + if err != nil { + t.Fatal(err) + } + want := 1 + if mode == "off" || status == "succeeded" && mode != "on" { + want = 0 + } + if len(ns) != want { + t.Fatalf("mode %s status %s: %+v", mode, status, ns) + } + }) + } + } + ctx := context.Background() + s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "covered.db"), "on") + defer s.Close() + if err := s.CoverRunNotification(ctx, id, 1, "succeeded"); err != nil { + t.Fatal(err) + } + if err := notificationTransition(ctx, s, id, "succeeded"); err != nil { + t.Fatal(err) + } + ns, err := s.RunNotifications(ctx, id) + if err != nil || len(ns) != 1 || ns[0].Status != "covered" { + t.Fatalf("stage coverage lost: %+v %v", ns, err) + } + if n, err := s.ClaimRunNotification(ctx, 100); err != nil || n != nil { + t.Fatalf("covered stage duplicated: %+v %v", n, err) + } +} + +func TestWaitingNotificationStageAndWorkerShareLease(t *testing.T) { + ctx := context.Background() + s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "lease.db"), "on") + defer s.Close() + if err := notificationTransition(ctx, s, id, "waiting"); err != nil { + t.Fatal(err) + } + lease, allowed, err := s.ClaimStageRunNotification(ctx, id, 1, "waiting", 100) + if err != nil || !allowed || lease == nil { + t.Fatalf("stage claim: %+v %v %v", lease, allowed, err) + } + if n, err := s.ClaimRunNotification(ctx, 101); err != nil || n != nil { + t.Fatalf("worker duplicated stage: %+v %v", n, err) + } + if err := s.CoverRunNotification(ctx, id, 1, "waiting"); err != nil { + t.Fatal(err) + } + if n, err := s.ClaimRunNotification(ctx, 161); err != nil || n != nil { + t.Fatalf("covered stage retried: %+v %v", n, err) + } + if _, allowed, err := s.ClaimStageRunNotification(ctx, id, 1, "waiting", 161); err != nil || allowed { + t.Fatalf("stage repeated: %v %v", allowed, err) + } +} diff --git a/internal/pipeline/run_notifications.go b/internal/pipeline/run_notifications.go index ed975302..086395ea 100644 --- a/internal/pipeline/run_notifications.go +++ b/internal/pipeline/run_notifications.go @@ -3,7 +3,14 @@ package pipeline import ( "context" "database/sql" + "encoding/json" "errors" + "fmt" + "strings" + "time" + "unicode/utf8" + + "git.a-wan.net/alan/pipelines/internal/protocol" ) // RunNotification is durable publisher evidence, not proof of device reception. @@ -16,6 +23,101 @@ type RunNotification struct { Attempts int } +// enqueueRunTransitionNotification must share the state mutation's transaction. +// Read the committed-to-be state, not a runner's proposed outcome or identity. +func enqueueRunTransitionNotification(ctx context.Context, tx *sql.Tx, id int64) error { + var mode, status, name, stage, diagnostic, graphJSON string + var sequence int + if err := tx.QueryRowContext(ctx, `SELECT r.notify_mode,r.status,p.name,r.current_stage,r.diagnostic, +COALESCE((SELECT MAX(sequence) FROM run_steps WHERE run_id=r.id),0),v.graph_json +FROM pipeline_runs r JOIN pipelines p ON p.id=r.pipeline_id +JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&mode, &status, &name, &stage, &diagnostic, &sequence, &graphJSON); err != nil { + return err + } + if mode == "off" { + return nil + } + var title, body string + switch status { + case "waiting": + title = "Run needs your decision" + body = "waiting for a human decision" + case "failed": + title = "Run failed" + body = "run failed" + case "succeeded": + if mode != "on" { + return nil + } + title = "Run succeeded" + body = "run succeeded" + default: + return nil + } + body = fmt.Sprintf("Run #%d · %s · %s", id, name, body) + if stage != "" { + body += " at stage " + stage + } + if diagnostic != "" { + body += ": " + diagnostic + } + body = strings.NewReplacer("\r", " ", "\n", " ", "\x00", " ").Replace(body) + if len(body) > 2000 { + body = body[:2000] + for !utf8.ValidString(body) { + body = body[:len(body)-1] + } + } + // Give a declared stage hook first opportunity to send its artifacts. Its + // initial claim bypasses this delay. If the process dies before that callback, + // the generic durable alert becomes eligible after one minute, not lost. + var graph protocol.Graph + if err := json.Unmarshal([]byte(graphJSON), &graph); err != nil { + return err + } + stageEvent := status + if status == "succeeded" { + stageEvent = "success" + } + if status == "failed" { + stageEvent = "failure" + } + var nextAttempt int64 + for _, definition := range graph.Stages { + if definition.Key != stage || definition.Notify == nil { + continue + } + for _, event := range definition.Notify.On { + if event == stageEvent { + nextAttempt = time.Now().Add(time.Minute).Unix() + } + } + } + _, err := tx.ExecContext(ctx, `INSERT INTO run_notifications(run_id,sequence,event,channel,title,body,click,next_attempt) +VALUES (?,?,?,'system',?,?,?,?) ON CONFLICT(run_id,sequence,event,channel) DO NOTHING`, id, sequence, status, title, body, fmt.Sprintf("pipelines://run/%d", id), nextAttempt) + return err +} + +// ClaimStageRunNotification arbitrates a transition hook against the outbox worker. +// With no transactional intent (legacy/direct stage callers), it leaves the +// stage's existing publication path alone. Otherwise the same lease protects +// both publishers, so only one may publish this event. +func (s *Store) ClaimStageRunNotification(ctx context.Context, id int64, sequence int, event string, now int64) (*RunNotification, bool, error) { + n := &RunNotification{RunID: id, Sequence: sequence, Event: event, Channel: "system"} + err := s.db.QueryRowContext(ctx, `UPDATE run_notifications SET status='sending',attempts=attempts+1,next_attempt=? +WHERE run_id=? AND sequence=? AND event=? AND channel='system' AND status='pending' AND attempts<3 AND (attempts=0 OR next_attempt<=?) +RETURNING attempts`, now+60, id, sequence, event, now).Scan(&n.Attempts) + if err == nil { + return n, true, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return nil, false, err + } + var exists bool + err = s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM run_notifications WHERE run_id=? AND sequence=? AND event=? AND channel='system')`, id, sequence, event).Scan(&exists) + return nil, !exists, err +} + func (s *Store) initRunNotifications() error { _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS run_notifications ( run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, event TEXT NOT NULL, channel TEXT NOT NULL, @@ -52,7 +154,8 @@ RETURNING run_id,sequence,event,channel,title,body,click,status,attempts`, now+6 // CoverRunNotification records a successful stage publication covering the same // run event, without publishing a second final alert. func (s *Store) CoverRunNotification(ctx context.Context, id int64, sequence int, event string) error { - _, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO run_notifications(run_id,sequence,event,channel,title,body,click,status) VALUES (?,?,?,'system','','','','covered')`, id, sequence, event) + _, err := s.db.ExecContext(ctx, `INSERT INTO run_notifications(run_id,sequence,event,channel,title,body,click,status) VALUES (?,?,?,'system','','','','covered') +ON CONFLICT(run_id,sequence,event,channel) DO UPDATE SET status='covered',diagnostic='' WHERE run_notifications.status IN ('pending','sending')`, id, sequence, event) return err } diff --git a/internal/pipeline/runs_mutate.go b/internal/pipeline/runs_mutate.go index 8bc44137..0dfb69bb 100644 --- a/internal/pipeline/runs_mutate.go +++ b/internal/pipeline/runs_mutate.go @@ -183,6 +183,9 @@ func (s *Store) MarkWaiting(ctx context.Context, id int64, sequence int, key str protocol.RunStatusWaiting, key, id)); err != nil { return err } + if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil { + return err + } return tx.Commit() } @@ -325,12 +328,23 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno if err != nil { return err } - return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE ? END, outcome=CASE WHEN cancel_requested=1 THEN '' ELSE ? END, diagnostic=CASE WHEN cancel_requested=1 THEN 'Canceled by operator; execution stopped' ELSE ? END, output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cleanup_unconfirmed=0 -AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, string(data), id)) +AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, string(data), id)); err != nil { + return err + } + if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil { + return err + } + return tx.Commit() } func (s *Store) MarkCleanupUnconfirmed(ctx context.Context, id int64) error { diff --git a/internal/server/notify.go b/internal/server/notify.go index 0de3c03d..4c015705 100644 --- a/internal/server/notify.go +++ b/internal/server/notify.go @@ -35,6 +35,23 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N if !enabled { return } + var lease *pipeline.RunNotification + runEvent := stageAttentionEvent(event) + if event == "success" && step.Target == protocol.SuccessEnd { + runEvent = "succeeded" + } + if event == protocol.NotifyWaiting || step.Target == protocol.SuccessEnd || step.Target == protocol.FailureEnd { + var allowed bool + var err error + lease, allowed, err = r.store.ClaimStageRunNotification(r.ctx, id, step.Sequence, runEvent, time.Now().Unix()) + if err != nil { + log.Printf("claim stage notification run=%d: %v", id, err) + return + } + if !allowed { + return + } + } ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second) defer cancel() summary := fmt.Sprintf("%s · run #%d · stage %s · %s (%s)", name, id, step.Stage, event, step.Outcome) @@ -134,7 +151,36 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N covered = "succeeded" } r.markAttentionSent(id, step.Sequence, covered) - break + return + } + } + if lease != nil { + diagnostic := "stage notification publication failed" + if len(receipts) > 0 && receipts[0].Diagnostic != "" { + diagnostic = receipts[0].Diagnostic + } + if err := r.store.FinishRunNotification(r.ctx, *lease, time.Now().Unix(), "", fmt.Errorf("%s", diagnostic)); err != nil { + log.Printf("release stage notification run=%d: %v", id, err) + } + } +} + +// Terminal hooks run only after FinishRun commits. They arbitrate against the +// durable intent just like waiting hooks, never publishing a proposed success. +func (r *runner) notifyTerminalStep(id int64, status string) { + detail, err := r.store.GetRun(r.ctx, id) + if err != nil || len(detail.Steps) == 0 { + return + } + step := detail.Steps[len(detail.Steps)-1] + stepStatus, _, _, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic) + if !terminal || stepStatus != status { + return + } + for _, definition := range detail.Graph.Stages { + if definition.Key == step.Stage { + r.notifyStep(id, detail.Name, detail.Workspace, definition.Notify, step) + return } } } diff --git a/internal/server/run_notification_outbox_test.go b/internal/server/run_notification_outbox_test.go index 2e1d0ec3..d4f1fa1b 100644 --- a/internal/server/run_notification_outbox_test.go +++ b/internal/server/run_notification_outbox_test.go @@ -14,6 +14,7 @@ import ( "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" + "git.a-wan.net/alan/pipelines/internal/summarise" ) func TestRunNotificationPublisherFailureRetriesAfterReopen(t *testing.T) { @@ -58,6 +59,128 @@ func TestRunNotificationPublisherFailureRetriesAfterReopen(t *testing.T) { } } +func TestFinishRunPersistenceFailureDoesNotAnnounceSuccess(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s, err := pipeline.Open(filepath.Join(t.TempDir(), "not-finished.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + source := strings.Replace(pipeline.Template([]string{"one"}, true), "# stage one", "# stage one\n```notify\non=success\n```", 1) + if _, err := s.Submit(ctx, "test", source); err != nil { + t.Fatal(err) + } + items, err := s.List(ctx) + if err != nil { + t.Fatal(err) + } + d, err := s.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + id, _, err := s.CreateRunTitled(ctx, d, "not-finished", "workspace", false, nil, nil, pipeline.RunOverrides{NotifyMode: "on"}, summarise.Title{}) + if err != nil { + t.Fatal(err) + } + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + _ = json.NewEncoder(w).Encode(map[string]string{"id": "wrong-success", "event": "message"}) + })) + defer server.Close() + t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake") + t.Setenv("PIPELINES_NTFY_TOKEN", "") + t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "") + r := &runner{ctx: ctx, store: s, events: newRunEventHub()} + if err := s.BeginStep(ctx, id, 1, "one", nil); err != nil { + t.Fatal(err) + } + state := &runState{memory: map[string]memoryValue{}} + if _, _, _, _, err := r.finishStage(id, "test", "workspace", d.Graph.Stages[0], state, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: protocol.SuccessEnd, Output: map[string]any{"markdown": "result"}}); err != nil { + t.Fatal(err) + } + if calls.Load() != 0 { + t.Fatal("terminal stage success published before FinishRun") + } + sub := r.events.subscribe("test", "workspace", nil, func(protocol.Message) error { return nil }) + defer sub.close() + // Marshal failure occurs before FinishRun can commit its proposed success. + r.finishRunState(id, "test", "workspace", "succeeded", "success", "", 0, 0, "", stage.Values{"invalid": make(chan int)}) + got, err := s.GetRun(ctx, id) + if err != nil { + t.Fatal(err) + } + if got.Status != "running" || len(got.Notifications) != 0 || calls.Load() != 0 { + t.Fatalf("unpersisted success announced: %+v calls=%d", got, calls.Load()) + } + sub.mu.Lock() + defer sub.mu.Unlock() + if len(sub.pending) != 0 { + t.Fatalf("unpersisted terminal event emitted: %+v", sub.pending) + } +} + +func TestWaitingHookSharesTransactionalRunIntent(t *testing.T) { + for _, failFirst := range []bool{false, true} { + t.Run(map[bool]string{false: "covered", true: "retry"}[failFirst], func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s, err := pipeline.Open(filepath.Join(t.TempDir(), "waiting-hook.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 && failFirst { + http.Error(w, "transient", http.StatusServiceUnavailable) + return + } + _ = json.NewEncoder(w).Encode(map[string]string{"id": "waiting-ack", "event": "message"}) + })) + defer server.Close() + t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake") + t.Setenv("PIPELINES_NTFY_TOKEN", "") + t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "") + source := strings.Replace(humanPipelineSource, "# stage gate", "# stage gate\n```notify\non=waiting\n```", 1) + if _, err := s.Submit(ctx, "waiting", source); err != nil { + t.Fatal(err) + } + items, err := s.List(ctx) + if err != nil { + t.Fatal(err) + } + r, err := newRunner(ctx, s) + if err != nil { + t.Fatal(err) + } + defer func() { cancel(); r.schedulerWG.Wait() }() + run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "waiting-hook", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + r.deliverRunNotifications(time.Now().Add(time.Minute)) + detail, err := s.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + wantCalls := int32(1) + if failFirst { + wantCalls = 2 + } + if calls.Load() != wantCalls || detail.Status != "waiting" || len(detail.Notifications) != 1 { + t.Fatalf("waiting hook duplicated/lost: calls=%d detail=%+v", calls.Load(), detail) + } + status := detail.Notifications[0].Status + if status != "sent" && status != "covered" { + t.Fatalf("missing acknowledgement: %+v", detail.Notifications) + } + }) + } +} + func TestRunNotificationModesAndFinalStageDedupe(t *testing.T) { var calls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/server/runner.go b/internal/server/runner.go index db64e4f7..676dd699 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -1332,7 +1332,10 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag CompletedSteps: completed, FinishedAt: detail.FinishedAt, }, true) - r.notifyRun(id, name, status, sequence, currentStage, diagnostic) + if status != protocol.RunStatusCanceled { + r.notifyTerminalStep(id, status) + } + r.deliverRunNotificationBatch(time.Now(), 1) // The run is terminal: nothing more can notify for it. r.forgetAttention(id) r.forgetNotifyMode(id) @@ -1422,9 +1425,11 @@ func (r *runner) finishStage(id int64, name, workspace string, selected protocol } state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence} } - r.notifyStep(id, name, workspace, selected.Notify, step) state.input = stage.Values(step.Output) status, outcome, diagnostic, terminal = terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic) + if !terminal { + r.notifyStep(id, name, workspace, selected.Notify, step) + } return status, outcome, diagnostic, terminal, nil } @@ -1588,7 +1593,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting}, }, false) r.notifyStep(id, name, workspace, selected.Notify, waitStep) - r.notifyRun(id, name, protocol.RunStatusWaiting, sequence, key, "") + r.deliverRunNotificationBatch(time.Now(), 1) return } // A stage that declares a `skipped` route is short-circuited when the run