fix: persist and retry run alerts and honor notification modes

This commit is contained in:
2026-10-20 15:42:21 +01:00
parent e4f48ecbb1
commit 016fcb552a
20 changed files with 697 additions and 41 deletions
+15
View File
@@ -574,6 +574,21 @@ Everything else is free-form and preserved under the step's `markdown` output.
the run. Artifacts are served from `/artifacts/<run>/<step>/<name>`; shared files
appear with a `public/` prefix.
### Run notification modes and receipts
`--notify auto|on|off` records `notify_mode` on the run (also returned in
summaries and `run_event` frames). Auto retains stage policies and run-level
waiting/failure defaults. On also alerts terminal success without requiring a
stage hook. Off suppresses stage, run-level system and browser tray alerts.
`inspect --json` / `get_run` includes top-level `notifications` receipts with
`sequence`, `event`, `channel`, `status`, `attempts`, and optional `diagnostic`
and `message_id`. Status is pending, sending, sent, failed, or covered (already
published by the stage). Run-level system alerts persist and retry at most three
times, recovering pending work after restart. A publisher acknowledgement is not
proof of reception; a crash before saving it can duplicate a message. See
[`notes/audit-notifications.md`](notes/audit-notifications.md) for limits.
## Investigating runs with a configured harness
On the runs list or inspector, press `D` or use `:debug`; CLI equivalent:
+36
View File
@@ -0,0 +1,36 @@
# Run notification audit fixes
`notify_mode=auto` retains existing behavior: stage policies plus run-level
waiting/failure defaults. `on` also alerts terminal success, even with no stage
notification hook. `off` suppresses stage, run-level system and browser tray
notifications. Needs you remains a status/action surface, not a delivery channel.
Summaries and pushed events carry the mode; the browser seeds and folds it.
`run_notifications` is an additive, idempotent SQLite migration. Each
run/sequence/event/channel has one durable outbox row. Its status is `pending`,
`sending`, `sent`, `failed`, or `covered` (a stage already published this event).
The worker polls once per second, uses an atomic 60-second sending lease and
allows three total attempts with 5/10-second retry delays. Pending and expired
leases survive restart. A final expired sending lease becomes failed with an
unknown-acknowledgement diagnostic. Existing retention deletes associated rows.
`get_run` exposes run-level `notifications` receipts (sequence, event, channel,
status, attempts, diagnostic, message_id), separate from step receipts.
Successful stage publications cover the corresponding waiting/failure alert;
a successful stage targeting `#success` also covers final success. Failed stage
publication does not swallow the run-level fallback/retry. Stage artifact
publications themselves are not made retryable by this change.
Publisher acknowledgement is **not device reception**. A crash after publish
but before the acknowledgement is saved can duplicate a notification. The
transition and enqueue are separate writes: a crash between them can miss an
alert; this is bounded retry of enqueued alerts, not guaranteed delivery.
Browser delivery remains page-session best effort and requires permission plus
HTTPS/localhost; successful runs do not become Needs you entries. The older L5
parity description is attention-only for auto; on now adds success.
Tests use temporary databases and fake HTTP publishers only, covering modes,
no-hook success, stage dedupe, failed publisher retry/exhaustion, restart/lease
recovery, audit persistence, and browser off/on. No live notifications, deployed
database/configuration, service restart, binary installation or OS reception
verification was part of this work. No new UI captures were taken.
+1
View File
@@ -621,6 +621,7 @@ func (s *Store) removeRetentionRuns(ctx context.Context, policy RetentionPolicy,
func deleteRetentionRows(ctx context.Context, tx *sql.Tx, ids []int64) error {
for _, table := range []struct{ name, column string }{
{"step_notifications", "run_id"},
{"run_notifications", "run_id"},
{"step_file_changes", "run_id"},
{"step_control", "run_id"},
{"step_artifacts", "run_id"},
+87
View File
@@ -0,0 +1,87 @@
package pipeline
import (
"context"
"database/sql"
"errors"
)
// RunNotification is durable publisher evidence, not proof of device reception.
// A sending lease permits restart recovery; a crash after publish but before
// acknowledgement can repeat a message. Attempts are bounded, not guaranteed.
type RunNotification struct {
RunID int64
Sequence int
Event, Channel, Title, Body, Click, Status, Diagnostic, MessageID string
Attempts int
}
func (s *Store) initRunNotifications() error {
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS run_notifications (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, event TEXT NOT NULL, channel TEXT NOT NULL,
title TEXT NOT NULL, body TEXT NOT NULL, click TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending', attempts INTEGER NOT NULL DEFAULT 0,
next_attempt INTEGER NOT NULL DEFAULT 0, diagnostic TEXT NOT NULL DEFAULT '', message_id TEXT NOT NULL DEFAULT '',
PRIMARY KEY(run_id,sequence,event,channel));
CREATE INDEX IF NOT EXISTS run_notifications_due ON run_notifications(status,next_attempt);`)
return err
}
func (s *Store) EnqueueRunNotification(ctx context.Context, n RunNotification) error {
_, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO run_notifications
(run_id,sequence,event,channel,title,body,click) VALUES (?,?,?,?,?,?,?)`, n.RunID, n.Sequence, n.Event, n.Channel, n.Title, n.Body, n.Click)
return err
}
// ClaimRunNotification atomically leases one due attempt for 60 seconds. Three
// attempts total bound both transient failures and crash/restart loops.
func (s *Store) ClaimRunNotification(ctx context.Context, now int64) (*RunNotification, error) {
if _, err := s.db.ExecContext(ctx, `UPDATE run_notifications SET status='failed',diagnostic='publisher acknowledgement unknown after final lease expired' WHERE status='sending' AND attempts>=3 AND next_attempt<=?`, now); err != nil {
return nil, err
}
n := new(RunNotification)
err := s.db.QueryRowContext(ctx, `UPDATE run_notifications SET status='sending', attempts=attempts+1, next_attempt=?
WHERE rowid=(SELECT rowid FROM run_notifications WHERE status IN ('pending','sending') AND attempts<3 AND next_attempt<=? ORDER BY next_attempt,rowid LIMIT 1)
RETURNING run_id,sequence,event,channel,title,body,click,status,attempts`, now+60, now).Scan(&n.RunID, &n.Sequence, &n.Event, &n.Channel, &n.Title, &n.Body, &n.Click, &n.Status, &n.Attempts)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
return n, err
}
// CoverRunNotification records a successful stage publication covering the same
// run event, without publishing a second final alert.
func (s *Store) CoverRunNotification(ctx context.Context, id int64, sequence int, event string) error {
_, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO run_notifications(run_id,sequence,event,channel,title,body,click,status) VALUES (?,?,?,'system','','','','covered')`, id, sequence, event)
return err
}
func (s *Store) FinishRunNotification(ctx context.Context, n RunNotification, now int64, messageID string, failure error) error {
status, diagnostic := "sent", ""
if failure != nil {
status, diagnostic = "pending", failure.Error()
if n.Attempts >= 3 {
status = "failed"
}
}
_, err := s.db.ExecContext(ctx, `UPDATE run_notifications SET status=?,diagnostic=?,message_id=?,next_attempt=?
WHERE run_id=? AND sequence=? AND event=? AND channel=? AND status='sending' AND attempts=?`, status, diagnostic, messageID, now+int64(n.Attempts*5), n.RunID, n.Sequence, n.Event, n.Channel, n.Attempts)
return err
}
func (s *Store) RunNotifications(ctx context.Context, id int64) ([]RunNotification, error) {
rows, err := s.db.QueryContext(ctx, `SELECT run_id,sequence,event,channel,status,attempts,diagnostic,message_id FROM run_notifications WHERE run_id=? ORDER BY sequence,event,channel`, id)
if err != nil {
return nil, err
}
defer rows.Close()
result := []RunNotification{}
for rows.Next() {
var n RunNotification
if err := rows.Scan(&n.RunID, &n.Sequence, &n.Event, &n.Channel, &n.Status, &n.Attempts, &n.Diagnostic, &n.MessageID); err != nil {
return nil, err
}
result = append(result, n)
}
return result, rows.Err()
}
+131
View File
@@ -0,0 +1,131 @@
package pipeline
import (
"context"
"errors"
"path/filepath"
"testing"
)
func TestRunNotificationPersistenceLeasesAndBound(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "outbox.db")
s, err := Open(path)
if err != nil {
t.Fatal(err)
}
n := RunNotification{RunID: 7, Sequence: 2, Event: "failed", Channel: "system", Title: "Failed", Body: "Run #7", Click: "pipelines://run/7"}
if err := s.EnqueueRunNotification(ctx, n); err != nil {
t.Fatal(err)
}
if err := s.EnqueueRunNotification(ctx, n); err != nil {
t.Fatal(err)
}
first, err := s.ClaimRunNotification(ctx, 100)
if err != nil || first == nil || first.Attempts != 1 {
t.Fatalf("first claim: %+v %v", first, err)
}
if other, err := s.ClaimRunNotification(ctx, 100); err != nil || other != nil {
t.Fatalf("lease double claim: %+v %v", other, err)
}
// Reopen an already-existing schema, preserving an in-flight lease.
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
second, err := s.ClaimRunNotification(ctx, 161)
if err != nil || second == nil || second.Attempts != 2 || second.Body != n.Body {
t.Fatalf("recovery: %+v %v", second, err)
}
if err := s.FinishRunNotification(ctx, *second, 161, "", errors.New("publisher unavailable")); err != nil {
t.Fatal(err)
}
if other, err := s.ClaimRunNotification(ctx, 170); err != nil || other != nil {
t.Fatalf("backoff ignored: %+v %v", other, err)
}
third, err := s.ClaimRunNotification(ctx, 171)
if err != nil || third == nil || third.Attempts != 3 {
t.Fatalf("third claim: %+v %v", third, err)
}
// Final attempt crashes: its expired lease must become failed, not retry forever.
if other, err := s.ClaimRunNotification(ctx, 232); err != nil || other != nil {
t.Fatalf("retry bound ignored: %+v %v", other, err)
}
got, err := s.RunNotifications(ctx, 7)
if err != nil || len(got) != 1 || got[0].Status != "failed" || got[0].Attempts != 3 {
t.Fatalf("audit: %+v %v", got, err)
}
}
func TestRunNotificationStageCoverageSurvivesReopen(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "coverage.db")
s, err := Open(path)
if err != nil {
t.Fatal(err)
}
if err := s.CoverRunNotification(ctx, 7, 1, "succeeded"); err != nil {
t.Fatal(err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
if err := s.EnqueueRunNotification(ctx, RunNotification{RunID: 7, Sequence: 1, Event: "succeeded", Channel: "system"}); err != nil {
t.Fatal(err)
}
if got, err := s.ClaimRunNotification(ctx, 100); err != nil || got != nil {
t.Fatalf("covered stage repeated: %+v %v", got, err)
}
}
func TestRunNotificationMigrationPreservesExistingRun(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "existing.db")
s, err := Open(path)
if err != nil {
t.Fatal(err)
}
if _, err := s.Submit(ctx, "existing", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, err := s.List(ctx)
if err != nil {
t.Fatal(err)
}
d, err := s.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
id, _, err := s.CreateRun(ctx, d, "existing-key", "workspace", false, nil, map[string]any{"prompt": "preserve me"})
if err != nil {
t.Fatal(err)
}
// Simulate the immediately preceding schema, which has no run outbox.
if _, err := s.db.ExecContext(ctx, "DROP TABLE run_notifications"); err != nil {
t.Fatal(err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
got, err := s.GetRun(ctx, id)
if err != nil || got.Input["prompt"] != "preserve me" || got.Status != "queued" || got.NotifyMode != "auto" || len(got.Notifications) != 0 {
t.Fatalf("migration changed run: %+v %v", got, err)
}
if err := s.EnqueueRunNotification(ctx, RunNotification{RunID: id, Sequence: 1, Event: "failed", Channel: "system"}); err != nil {
t.Fatal(err)
}
}
+19 -2
View File
@@ -21,12 +21,12 @@ const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
` + runStamps + `,r.diagnostic,
CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END,
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode
` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version`
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
var output, stageModels string
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels); err != nil {
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode); err != nil {
return err
}
if stageModels != "" && stageModels != "{}" {
@@ -201,6 +201,23 @@ strftime('%Y-%m-%dT%H:%M:%SZ',started_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ'
// The run total is one pass over the steps already loaded above: it adds no
// query and no transcript parse.
r.Usage = protocol.TotalUsage(r.Steps)
alerts, err := tx.QueryContext(ctx, `SELECT sequence,event,channel,status,attempts,diagnostic,message_id FROM run_notifications WHERE run_id=? ORDER BY sequence,event,channel`, id)
if err != nil {
return nil, err
}
for alerts.Next() {
var receipt protocol.RunNotificationReceipt
if err := alerts.Scan(&receipt.Sequence, &receipt.Event, &receipt.Channel, &receipt.Status, &receipt.Attempts, &receipt.Diagnostic, &receipt.MessageID); err != nil {
alerts.Close()
return nil, err
}
r.Notifications = append(r.Notifications, receipt)
}
err = alerts.Err()
alerts.Close()
if err != nil {
return nil, err
}
if err := tx.Commit(); err != nil {
return nil, err
}
+4
View File
@@ -54,6 +54,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU
db.Close()
return nil, err
}
if err := s.initRunNotifications(); err != nil {
db.Close()
return nil, err
}
if err := s.initSchedules(); err != nil {
db.Close()
return nil, err
+1 -1
View File
@@ -153,7 +153,7 @@ type Message struct {
// them; the same fields are echoed on the run record.
Model string `json:"model,omitempty"`
StageModels map[string]string `json:"stage_models,omitempty"`
// NotifyMode is auto (definition), on (ensure run-level attention) or off
// NotifyMode is auto (definition), on (terminal success/failure and attention) or off
// (silence this run). It is optional so older clients keep auto.
NotifyMode string `json:"notify_mode,omitempty"`
// Empty (or "resume") retries the failed visit; "restart" starts at the
+3 -2
View File
@@ -24,8 +24,9 @@ const (
// from the frame instead of re-reading the whole list. They repeat on every
// frame (pipeline identity never changes mid-run).
type RunEventFrame struct {
RunID int64 `json:"run_id"`
Workspace string `json:"workspace,omitempty"`
NotifyMode string `json:"notify_mode,omitempty"`
RunID int64 `json:"run_id"`
Workspace string `json:"workspace,omitempty"`
// PipelineID, Name, Title and Version identify the run's pipeline and its
// operator-facing title, so a new run's row can be built without a fetch.
PipelineID int64 `json:"pipeline_id,omitempty"`
+25 -12
View File
@@ -119,6 +119,7 @@ type SessionExport struct {
}
type RunSummary struct {
NotifyMode string `json:"notify_mode,omitempty"`
ID int64 `json:"id"`
PipelineID int64 `json:"pipeline_id"`
Name string `json:"name"`
@@ -349,18 +350,19 @@ type NotificationReceipt struct {
type RunDetail struct {
RunSummary
Graph Graph `json:"graph"`
RetryOf int64 `json:"retry_of,omitempty"`
RetryMode string `json:"retry_mode,omitempty"`
RetrySource int64 `json:"retry_source,omitempty"`
RetrySourceVersion int `json:"retry_source_version,omitempty"`
RetrySourceHash string `json:"retry_source_hash,omitempty"`
Workspace string `json:"workspace"`
AllowTools bool `json:"allow_tools"`
WritePaths []string `json:"write_paths"`
Input map[string]any `json:"input"`
Output map[string]any `json:"output"`
Steps []RunStep `json:"steps"`
Notifications []RunNotificationReceipt `json:"notifications,omitempty"`
Graph Graph `json:"graph"`
RetryOf int64 `json:"retry_of,omitempty"`
RetryMode string `json:"retry_mode,omitempty"`
RetrySource int64 `json:"retry_source,omitempty"`
RetrySourceVersion int `json:"retry_source_version,omitempty"`
RetrySourceHash string `json:"retry_source_hash,omitempty"`
Workspace string `json:"workspace"`
AllowTools bool `json:"allow_tools"`
WritePaths []string `json:"write_paths"`
Input map[string]any `json:"input"`
Output map[string]any `json:"output"`
Steps []RunStep `json:"steps"`
// Usage is the totalled reported usage of the run's model stages, summed in
// one pass over Steps. It is absent for a run with no model usage recorded
// (including historical runs).
@@ -372,3 +374,14 @@ type RunDetail struct {
TitleSource string `json:"title_source,omitempty"`
TitleModel string `json:"title_model,omitempty"`
}
// RunNotificationReceipt records publisher attempts, not device reception.
type RunNotificationReceipt struct {
Sequence int `json:"sequence"`
Event string `json:"event"`
Channel string `json:"channel"`
Status string `json:"status"`
Attempts int `json:"attempts"`
Diagnostic string `json:"diagnostic,omitempty"`
MessageID string `json:"message_id,omitempty"`
}
+80 -14
View File
@@ -3,6 +3,7 @@ package server
import (
"context"
"fmt"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"io"
@@ -34,10 +35,6 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
if !enabled {
return
}
// The stage's own policy already covers this attention event, so the
// run-level default must not repeat it. Recorded before delivery so a failed
// stage notification is reported by its own receipt, not silently duplicated.
r.markAttentionSent(id, step.Sequence, stageAttentionEvent(event))
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
defer cancel()
summary := fmt.Sprintf("%s · run #%d · stage %s · %s (%s)", name, id, step.Stage, event, step.Outcome)
@@ -128,6 +125,18 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
if !sentArtifact {
send("", nil)
}
// Only a successful publish covers the run alert. A failed stage publisher
// must not swallow the durable run-level fallback and its retries.
for _, receipt := range receipts {
if receipt.Status == "sent" {
covered := stageAttentionEvent(event)
if event == "success" && step.Target == protocol.SuccessEnd {
covered = "succeeded"
}
r.markAttentionSent(id, step.Sequence, covered)
break
}
}
}
// runAttentionEvent maps a run status to the operator attention notification it
@@ -167,8 +176,16 @@ func (r *runner) markAttentionSent(id int64, sequence int, event string) {
if event == "" {
return
}
if r.store != nil {
if err := r.store.CoverRunNotification(r.ctx, id, sequence, event); err != nil {
log.Printf("record stage notification coverage run=%d: %v", id, err)
}
}
r.attentionMu.Lock()
defer r.attentionMu.Unlock()
if r.attentionSent == nil {
r.attentionSent = map[int64]map[string]bool{}
}
if r.attentionSent[id] == nil {
r.attentionSent[id] = map[string]bool{}
}
@@ -206,6 +223,9 @@ func (r *runner) notifyMode(id int64) string {
if r.store != nil {
if stored, err := r.store.RunNotifyMode(r.ctx, id); err == nil && stored != "" {
r.notifyMu.Lock()
if r.notifyModes == nil {
r.notifyModes = map[int64]string{}
}
r.notifyModes[id] = stored
r.notifyMu.Unlock()
return stored
@@ -240,29 +260,32 @@ func runAttentionClick(id int64) string {
// notifyStep and reuses the same PublishNtfy machinery, destination and token
// handling, so no second publisher, topic or service is introduced. The run id
// and pipeline name identify the run and the click action opens it in the app.
// A run that needs nobody produces nothing, and a delivery failure is logged and
// never fails the run: this is a best-effort alert, not persisted run evidence.
// Attempts are persisted before publication. Publisher acknowledgement is not
// proof of device reception; a crash in that acknowledgement window can repeat.
func (r *runner) notifyRun(id int64, name, status string, sequence int, currentStage, diagnostic string) {
if r.notifyMode(id) == "off" {
return
}
event := runAttentionEvent(status)
if status == "succeeded" && r.notifyMode(id) == "on" {
event = "succeeded"
}
if event == "" {
return
}
if r.attentionAlready(id, sequence, event) {
return
}
// Record the delivery attempt before publishing so a repeated transition for
// the same run and event (a re-request, a restart-free re-entry) never
// republishes; delivery itself is never retried.
r.markAttentionSent(id, sequence, event)
title := "Run needs your decision"
summary := fmt.Sprintf("Run #%d · %s · waiting for a human decision", id, name)
if event == "failed" {
title = "Run failed"
summary = fmt.Sprintf("Run #%d · %s · run failed", id, name)
}
if event == "succeeded" {
title = "Run succeeded"
summary = fmt.Sprintf("Run #%d · %s · run succeeded", id, name)
}
if currentStage != "" {
summary += " at stage " + currentStage
}
@@ -270,10 +293,53 @@ func (r *runner) notifyRun(id int64, name, status string, sequence int, currentS
summary += ": " + diagnostic
}
summary = truncateUTF8(summary, 2000)
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
defer cancel()
if _, err := stage.PublishNtfy(ctx, "system", title, summary, "", nil, runAttentionClick(id)); err != nil {
log.Printf("run attention notification run=%d: %v", id, err)
if err := r.store.EnqueueRunNotification(r.ctx, pipeline.RunNotification{RunID: id, Sequence: sequence, Event: event, Channel: "system", Title: title, Body: summary, Click: runAttentionClick(id)}); err != nil {
log.Printf("enqueue run notification run=%d: %v", id, err)
return
}
r.deliverRunNotificationBatch(time.Now(), 1)
}
// resumeRunNotifications also recovers pending/expired leases after restart.
func (r *runner) resumeRunNotifications() {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-r.ctx.Done():
return
case <-ticker.C:
r.deliverRunNotifications(time.Now())
}
}
}
func (r *runner) deliverRunNotifications(now time.Time) {
r.deliverRunNotificationBatch(now, 32)
}
func (r *runner) deliverRunNotificationBatch(now time.Time, limit int) {
// Bound one drain so a queue cannot monopolize the runner.
started := time.Now()
for i := 0; i < limit; i++ {
n, err := r.store.ClaimRunNotification(r.ctx, now.Add(time.Since(started)).Unix())
if err != nil {
log.Printf("claim run notification: %v", err)
return
}
if n == nil {
return
}
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
result, publishErr := stage.PublishNtfy(ctx, n.Channel, n.Title, n.Body, "", nil, n.Click)
cancel()
messageID, _ := result["message_id"].(string)
save, done := context.WithTimeout(context.Background(), 5*time.Second)
err = r.store.FinishRunNotification(save, *n, now.Add(time.Since(started)).Unix(), messageID, publishErr)
done()
if err != nil {
log.Printf("save run notification run=%d: %v", n.RunID, err)
}
}
}
+10 -2
View File
@@ -121,7 +121,7 @@ func TestRunEventWebSocketPushesTransitions(t *testing.T) {
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "sub" {
t.Fatalf("subscribe ack = %+v", ack)
}
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, NotifyMode: "off", Input: map[string]any{"prompt": "test"}})
var runID int64
var created, stepStarted, stepFinished, terminal *protocol.RunEventFrame
@@ -175,6 +175,11 @@ func TestRunEventWebSocketPushesTransitions(t *testing.T) {
if terminal.RunID != runID || terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.FinishedAt == "" {
t.Fatalf("terminal frame = %+v", terminal)
}
for _, frame := range []*protocol.RunEventFrame{created, stepStarted, stepFinished, terminal} {
if frame.NotifyMode != "off" {
t.Fatalf("silent mode lost in event: %+v", frame)
}
}
}
// TestRunEventSubscribeSendsSnapshot proves a client that subscribes to an
@@ -188,7 +193,7 @@ func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
workspace := eventWorkspace(t)
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, NotifyMode: "off", Input: map[string]any{"prompt": "test"}})
if err != nil {
t.Fatal(err)
}
@@ -203,6 +208,9 @@ func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
if first.RunEvent.RunID != run.ID || first.RunEvent.Status != "succeeded" || first.RunEvent.CompletedSteps != 1 || first.RunEvent.FinishedAt == "" {
t.Fatalf("snapshot frame = %+v", first.RunEvent)
}
if first.RunEvent.NotifyMode != "off" {
t.Fatalf("silent mode lost in snapshot: %+v", first.RunEvent)
}
// A snapshot for a run the client never saw also carries the identity its row
// needs, so a fresh subscribe renders it without a second read.
if first.RunEvent.PipelineID != ids[0] || first.RunEvent.Name == "" || first.RunEvent.Version != 1 {
@@ -0,0 +1,221 @@
package server
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func TestRunNotificationPublisherFailureRetriesAfterReopen(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "retry.db")
s, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if calls.Add(1) == 1 {
http.Error(w, "transient", http.StatusServiceUnavailable)
return
}
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r := &runner{ctx: ctx, store: s}
r.notifyRun(7, "test", "failed", 1, "stage", "")
got, err := s.RunNotifications(ctx, 7)
if err != nil || len(got) != 1 || got[0].Status != "pending" || got[0].Attempts != 1 || got[0].Diagnostic == "" {
t.Fatalf("failed attempt not audited: %+v %v", got, err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
r = &runner{ctx: ctx, store: s}
r.deliverRunNotifications(time.Now().Add(time.Minute))
r.notifyRun(7, "test", "failed", 1, "stage", "")
got, err = s.RunNotifications(ctx, 7)
if err != nil || len(got) != 1 || got[0].Status != "sent" || got[0].Attempts != 2 || got[0].MessageID != "ack" || calls.Load() != 2 {
t.Fatalf("retry/dedupe: %+v calls=%d %v", got, calls.Load(), err)
}
}
func TestRunNotificationModesAndFinalStageDedupe(t *testing.T) {
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
for _, tc := range []struct {
mode string
hook bool
want int32
}{
{"on", false, 1}, {"auto", false, 0}, {"off", false, 0}, {"on", true, 1}, {"auto", true, 1}, {"off", true, 0},
} {
t.Run(tc.mode+map[bool]string{true: "-hook", false: "-no-hook"}[tc.hook], func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
s, err := pipeline.Open(filepath.Join(t.TempDir(), "modes.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
source := pipeline.Template([]string{"one"}, true)
if tc.hook {
source = strings.Replace(source, "# stage one", "# stage one\n```notify\non=success\n```", 1)
}
if _, err := s.Submit(ctx, "mode-test", source); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success"}, nil
}
before := calls.Load()
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mode", NotifyMode: tc.mode, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := s.GetRun(ctx, run.ID)
if err != nil || detail.Status != "succeeded" || detail.NotifyMode != tc.mode || calls.Load()-before != tc.want {
t.Fatalf("mode contract: %+v calls=%d %v", detail, calls.Load()-before, err)
}
if tc.mode == "on" && (len(detail.Notifications) != 1 || (detail.Notifications[0].Status != "sent" && detail.Notifications[0].Status != "covered")) {
t.Fatalf("missing final audit: %+v", detail.Notifications)
}
cancel()
r.schedulerWG.Wait()
})
}
}
func TestRunNotificationAttentionModesAndRetryBound(t *testing.T) {
ctx := context.Background()
s, err := pipeline.Open(filepath.Join(t.TempDir(), "attention.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
http.Error(w, "unavailable", http.StatusServiceUnavailable)
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r := &runner{ctx: ctx, store: s}
for i, mode := range []string{"auto", "on", "off"} {
id := int64(i + 1)
r.setNotifyMode(id, mode)
for sequence, status := range []string{"waiting", "failed"} {
r.notifyRun(id, "test", status, sequence, "stage", "")
}
}
if calls.Load() != 4 {
t.Fatalf("attention modes: %d calls, want 4", calls.Load())
}
r.deliverRunNotifications(time.Now().Add(time.Minute))
r.deliverRunNotifications(time.Now().Add(2 * time.Minute))
r.deliverRunNotifications(time.Now().Add(3 * time.Minute))
if calls.Load() != 12 {
t.Fatalf("retry bound: %d calls, want 12", calls.Load())
}
for _, id := range []int64{1, 2, 3} {
got, err := s.RunNotifications(ctx, id)
if err != nil {
t.Fatal(err)
}
if id == 3 && len(got) != 0 {
t.Fatalf("off created attempts: %+v", got)
}
for _, n := range got {
if n.Status != "failed" || n.Attempts != 3 {
t.Fatalf("missing exhausted audit: %+v", n)
}
}
}
}
func TestRunNotificationWorkerResumesPendingAfterRestart(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
path := filepath.Join(t.TempDir(), "worker.db")
s, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
if err := s.EnqueueRunNotification(ctx, pipeline.RunNotification{RunID: 7, Sequence: 1, Event: "failed", Channel: "system", Title: "Run failed", Body: "pending"}); err != nil {
t.Fatal(err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
delivered := make(chan struct{}, 2)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]string{"id": "recovered", "event": "message"})
delivered <- struct{}{}
}))
defer server.Close()
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.schedulerWG.Wait() }()
select {
case <-delivered:
case <-time.After(5 * time.Second):
t.Fatal("pending outbox was not recovered")
}
deadline := time.Now().Add(5 * time.Second)
for {
got, err := s.RunNotifications(ctx, 7)
if err != nil {
t.Fatal(err)
}
if len(got) == 1 && got[0].Status == "sent" && got[0].MessageID == "recovered" {
break
}
if time.Now().After(deadline) {
t.Fatalf("recovery audit: %+v", got)
}
time.Sleep(10 * time.Millisecond)
}
}
+4
View File
@@ -424,6 +424,9 @@ func (s *runSubscription) close() {
}
func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
if in.NotifyMode != "" {
into.NotifyMode = in.NotifyMode
}
if in.Workspace != "" {
into.Workspace = in.Workspace
}
@@ -500,6 +503,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
continue
}
frame := protocol.RunEventFrame{
NotifyMode: summary.NotifyMode,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
+5 -1
View File
@@ -101,7 +101,10 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
return nil, err
}
root := filepath.Join(s.Directory(), "artifacts")
return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}, nil
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}
r.schedulerWG.Add(1)
go func() { defer r.schedulerWG.Done(); r.resumeRunNotifications() }()
return r, nil
}
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
@@ -1257,6 +1260,7 @@ func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) {
return
}
r.stampRunIdentity(&frame)
frame.NotifyMode = r.notifyMode(frame.RunID)
r.events.emit(frame, terminal)
}
+1 -1
View File
@@ -166,7 +166,7 @@ to it, but the server never lists it and a reload drops it.
| B3 | Equal phone dock cells | — | done — the narrow dock is a four-cell grid (one cell per destination plus Search) with a subgrid for the tablist, so every control has the same width and height. At 390×844 the measured widths are 92/92/92/92 and the row is 48px high. The phone shell uses `100svh` so a browser toolbar cannot hide the tab row while leaving only the Connected/Settings status row visible. |
| B4 | Full-height pipeline Source | — | done — the Source block no longer caps itself at `60vh`; it is as tall as its content and the detail pane owns the scroll. Verified at 390×844: a 4386px definition sits in the full pane (656px viewport) and scrolls. |
| B14 | Run a pipeline from the web | `run_pipeline` | done — the pipeline detail has a Run action that opens a dialog with the prompt, an absolute server working-directory field with recent-directory suggestions, and a per-run notification choice (pipeline default / notify / silent). The request carries `pipeline_id`, the pinned version, a fresh request key, `workspace`, `input.prompt` and `notify_mode` only when overriding. A live probe started a real run from the dialog with `notify_mode=off` and the server stored that mode; the run was canceled. |
| B15 | Per-run notification override | `notify_mode` | done — `pipelines run --notify auto\|on\|off` (and the same option on retry) records the override with the run. Off suppresses both stage policies and run-level waiting/failed alerts; auto follows the definition; on keeps run-level attention. The mode is stored in `pipeline_runs`, reloaded after a restart, and inherited by a retry unless overridden. |
| B15 | Per-run notification override | `notify_mode` | done — `pipelines run --notify auto\|on\|off` (and the same option on retry) records the override with the run. Off suppresses stage, run-level and browser tray notifications (Needs you still reflects run state). Auto retains existing definition plus waiting/failed defaults; on additionally alerts terminal success, including runs without notify hooks. Mode travels in summaries/events and is inherited by retries unless overridden. Run-level system publications use a durable bounded-retry outbox and publisher receipts; stage final success coverage avoids a second system alert. Browser mode behavior is unit-tested, not OS-observed; no new UI captures in this audit. |
| B16 | One live status line for tools | — | done — the current tool call now shares the `N tools called` line; the separate bottom per-tool line is gone and the lower indicator appears only while no tool is running. The live reasoning label's duration ticks every second without waiting for a frame. Chat frames for a background session no longer repaint a Runs/Pipelines screen: verified a synthetic stream while opening run 12's detail left the run screen interactive with no per-frame replacements. |
| B17 | Clear list chrome and chat recency | — | done — list pages no longer repeat the destination title/subtitle from the rail/dock, and the manual Refresh controls are gone; the visible list refreshes once when the page regains focus, preserving its scroll position and leaving open menus, dialogs and text fields alone. Chat rows replace the meaningless `Idle`/`Open` status with the server's latest-update recency (`just now`, `15 min ago`, `2 days ago`), keep `Streaming`/`New`/`Closed`, and give each directory a full-width elevated band with centred text that expands/collapses exactly like Closed (square corners, not a rounded chip), abbreviating the server home prefix to `~` (`~/repos/pipelines`) in both the band and the row meta. `New chat` is now a filled primary control beside the search field. Runs and Pipelines use the same title-free list header. Verified at 390×844 and 1440×900 in a real browser, with no console errors; the web unit suite is 633 tests. |
| B18 | Chat attachments in the message and paste | `chat_prompt.attachments` | done — composer files are read into base64 and sent with the prompt (1 MiB total, under the 2 MiB websocket frame), the server writes them to private temporary files and passes `file://` references to OpenCode's prompt `files`, and both the live user turn and reloaded history render a named file chip inside the message. Pasting an image/file into the textarea attaches it with the same picker path; a pasted image without a filename gets a generated name. The optimistic user bubble shows the file immediately, and OpenCode history maps the top-level user `files` envelope (never its base64 data) into the same part. |
+20 -6
View File
@@ -1,6 +1,5 @@
// Pure, DOM-free notification model for failed/waiting runs (L5). It mirrors
// android/.../notify/RunAlerts.kt: the same two qualifying kinds, the same
// headlines, the same run line and the same one-notification-per-run+kind rule.
// Pure, DOM-free notification model. Auto retains failed/waiting alerts;
// per-run on also alerts terminal success and off suppresses every tray alert.
//
// Everything here is pure: no DOM, no clock, no storage access. The page owns
// the Notification constructor, the permission request gesture and localStorage;
@@ -13,14 +12,16 @@ import { AttentionKind, attentionKey, attentionKindOf } from "./attention.js";
// Kinds, headlines and the run line
// ---------------------------------------------------------------------------
/** The two run events that raise a notification (RunAlerts.kt RunAlertKind). */
/** Notification kinds; success does not create a Needs you entry. */
export const RunAlertKind = Object.freeze({
Decision: AttentionKind.Decision,
Failed: AttentionKind.Failed,
Succeeded: Object.freeze({ name: "Succeeded" }),
});
/** `alertKindOf`: an attention kind is already exactly the notification kind. */
export function alertKindOf(kind) {
if (kind?.name === "Succeeded") return RunAlertKind.Succeeded;
if (kind === AttentionKind.Decision || kind?.name === AttentionKind.Decision.name) {
return AttentionKind.Decision;
}
@@ -37,6 +38,8 @@ export function alertTitle(kind) {
return "A run needs your decision";
case "Failed":
return "A run failed";
case "Succeeded":
return "A run succeeded";
default:
return "";
}
@@ -90,15 +93,18 @@ export function alertBody(run) {
*/
export function createAlertLedger() {
const lastStatus = new Map();
const modes = new Map();
const notified = new Set();
let seeding = false;
function seed(runs) {
const next = new Map();
modes.clear();
for (const run of runs ?? []) {
const runId = runIdOf(run?.id ?? run?.run_id);
if (runId == null) continue;
next.set(runId, normalizeStatus(run?.status));
modes.set(runId, normalizeStatus(run?.notify_mode) || "auto");
}
lastStatus.clear();
for (const [runId, status] of next) lastStatus.set(runId, status);
@@ -113,16 +119,23 @@ export function createAlertLedger() {
const previous = lastStatus.get(runId);
lastStatus.set(runId, status);
const kind = attentionKindOf(status);
if (frame?.notify_mode != null) modes.set(runId, normalizeStatus(frame.notify_mode));
const mode = modes.get(runId) || "auto";
const kind = status === "succeeded" && mode === "on" ? RunAlertKind.Succeeded : attentionKindOf(status);
if (!kind) {
forget(runId, AttentionKind.Decision);
forget(runId, AttentionKind.Failed);
forget(runId, RunAlertKind.Succeeded);
return null;
}
// A run can only be in one qualifying state; forget the other kind's key so
// a later transition back into it is a new event.
forget(runId, kind === AttentionKind.Decision ? AttentionKind.Failed : AttentionKind.Decision);
for (const other of Object.values(RunAlertKind)) {
if (other !== kind) forget(runId, other);
}
if (seeding) return null;
if (mode === "off") return null;
if (previous === undefined || previous === status) return null;
const key = alertKey(runId, kind);
@@ -143,6 +156,7 @@ export function createAlertLedger() {
function reset() {
lastStatus.clear();
modes.clear();
seeding = false;
}
+4
View File
@@ -172,6 +172,7 @@ function sameResourceWait(a, b) {
/** The run-state fields a frame advances, shared by the summary row and the detail. */
function runStateFields(current) {
return {
notify_mode: current.notify_mode ?? "auto",
resource_wait: current.resource_wait ?? null,
status: current.status ?? "",
outcome: current.outcome ?? "",
@@ -196,6 +197,7 @@ function applyRunStateFields(current, frame) {
? "queued"
: advanceRunStatus(before.status, frame.status);
const merged = {
notify_mode: blank(frame.notify_mode) ? before.notify_mode : frame.notify_mode,
resource_wait: isTerminalRunStatus(status)
? null
: frame.resource_wait_set === true
@@ -210,6 +212,7 @@ function applyRunStateFields(current, frame) {
executing: executingFor(status, before.executing),
};
const changed =
merged.notify_mode !== before.notify_mode ||
!sameResourceWait(merged.resource_wait, before.resource_wait) ||
merged.status !== before.status ||
merged.outcome !== before.outcome ||
@@ -236,6 +239,7 @@ export function newRunSummaryFromFrame(frame) {
id: Number(frame.run_id),
pipeline_id: pipelineId,
name,
notify_mode: frame.notify_mode ?? "auto",
title: blank(frame.title) ? null : frame.title,
version: Number(frame.version) || 0,
status: frame.status ?? "",
+20
View File
@@ -31,6 +31,26 @@ const frame = (runId, status, extra = {}) => ({
...extra,
});
test("off suppresses every browser alert from summaries and pushed modes", () => {
for (const status of ["waiting", "failed", "succeeded"]) {
const ledger = createAlertLedger();
ledger.seed([{ id: 1, status: "running", notify_mode: "off" }]);
assert.equal(ledger.observe(frame(1, status)), null);
ledger.seed([{ id: 1, status: "running", notify_mode: "on" }]);
assert.equal(ledger.observe(frame(1, status, { notify_mode: "off" })), null);
}
});
test("on adds final success once while auto retains attention-only semantics", () => {
const ledger = createAlertLedger();
ledger.seed([{ id: 1, status: "running", notify_mode: "on" }, { id: 2, status: "running", notify_mode: "auto" }]);
const alert = ledger.observe(frame(1, "succeeded"));
assert.equal(alert.title, "A run succeeded");
assert.equal(alert.key, "1:Succeeded");
assert.equal(ledger.observe(frame(1, "succeeded")), null);
assert.equal(ledger.observe(frame(2, "succeeded")), null);
});
test("headlines are exactly the app's two, and unknown kinds are blank", () => {
assert.equal(alertTitle(AttentionKind.Failed), "A run failed");
assert.equal(alertTitle(AttentionKind.Decision), "A run needs your decision");
+10
View File
@@ -87,6 +87,16 @@ function frame({
// --- lifecycle vocabulary ----------------------------------------------------
test("run summaries and detail preserve notification mode from live frames", () => {
const incoming = { ...frame({ pipelineId: 4, name: "test", status: "queued" }), notify_mode: "off" };
assert.equal(newRunSummaryFromFrame(incoming).notify_mode, "off");
const runs = [summary(1)];
const folded = foldRunEvent(runs, { ...incoming, status: "running" });
assert.equal(folded.runs[0].notify_mode, "off");
const result = foldRunEventDetail(detail({status:"running"}), { ...incoming, status: "failed" });
assert.equal(result.notify_mode, "off");
});
test("the lifecycle ladders mirror RunEvents.kt", () => {
assert.deepEqual(TERMINAL_RUN_STATUSES, ["succeeded", "failed", "canceled", "interrupted"]);
assert.equal(runLifecycleRank("queued"), 0);