From a6221caa6e4b901f5df0314f99998f4ab1870adc Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Thu, 12 Nov 2026 11:16:13 +0000 Subject: [PATCH] Notify only on unexpected run failures Classify each failed run from its terminal visit: a stage that deliberately routes to #failure with a reason is expected and does not notify or ask for attention; errors, budgets, timeouts, interruptions and exhausted visit limits (from the recorded control evidence) are unexpected and always notify. --- .../attention/domain/attention_models.dart | 8 +++- app/lib/features/runs/domain/run_events.dart | 5 ++ .../attention/attention_models_test.dart | 26 ++++++++++ .../author/skill/references/notifications.md | 7 ++- .../run_notification_transition_test.go | 48 +++++++++++++++++++ internal/pipeline/run_notifications.go | 15 ++++-- internal/pipeline/runs.go | 4 +- internal/pipeline/runs_mutate.go | 44 +++++++++++++++-- internal/pipeline/runs_schema.go | 9 +++- internal/protocol/runevents.go | 2 + internal/protocol/runs.go | 2 + internal/server/notify.go | 11 +++++ internal/server/notify_test.go | 20 ++++---- internal/server/runevents.go | 2 + internal/server/runner.go | 2 + 15 files changed, 184 insertions(+), 21 deletions(-) diff --git a/app/lib/features/attention/domain/attention_models.dart b/app/lib/features/attention/domain/attention_models.dart index 6381b86d..948a199c 100644 --- a/app/lib/features/attention/domain/attention_models.dart +++ b/app/lib/features/attention/domain/attention_models.dart @@ -65,6 +65,9 @@ AttentionKind? attentionKindOf(dynamic status) { /// onto a waiting run must not hide a genuine human decision. AttentionKind? attentionKindForRun(JsonMap run) { final kind = attentionKindOf(run['status']); + if (kind == AttentionKind.failed && _word(run['failure_kind']) == 'expected') { + return null; + } if (kind != AttentionKind.decision) return kind; if (_positive(run['export_pending_sequence'])) return null; return kind; @@ -77,7 +80,10 @@ String attentionKey(int runId, AttentionKind kind) => '$runId:${kind.wireName}'; /// shows the server's own diagnostic when it recorded one. String attentionDetail(AttentionKind kind, JsonMap run) { if (kind == AttentionKind.decision) return 'Waiting for your decision'; - final diagnostic = '${run['diagnostic'] ?? ''}'.trim(); + final failureReason = '${run['failure_reason'] ?? ''}'.trim(); + final diagnostic = failureReason.isNotEmpty + ? failureReason + : '${run['diagnostic'] ?? ''}'.trim(); return diagnostic.isNotEmpty ? diagnostic : 'Run failed'; } diff --git a/app/lib/features/runs/domain/run_events.dart b/app/lib/features/runs/domain/run_events.dart index 2e066781..f5ff2664 100644 --- a/app/lib/features/runs/domain/run_events.dart +++ b/app/lib/features/runs/domain/run_events.dart @@ -119,6 +119,9 @@ JsonMap _mergeRunFields(JsonMap current, JsonMap frame) { for (final key in ['outcome', 'current_stage', 'created_at', 'finished_at']) { if (!_blank(frame[key])) merged[key] = frame[key]; } + for (final key in ['failure_kind', 'failure_reason']) { + if (frame[key] != null) merged[key] = frame[key]; + } // Snapshot and terminal frames carry the measured size; others omit it. if (frame['size_bytes'] is num) merged['size_bytes'] = frame['size_bytes']; final oldExport = _positiveInt(current['export_pending_sequence']); @@ -168,6 +171,8 @@ JsonMap? newRunSummaryFromFrame(JsonMap frame) { 'finished_at': frame['finished_at'] ?? '', 'outcome': frame['outcome'] ?? '', 'current_stage': frame['current_stage'] ?? '', + 'failure_kind': frame['failure_kind'] ?? '', + 'failure_reason': frame['failure_reason'] ?? '', 'resource_wait': frame['resource_wait'], 'cancel_requested': frame['cancel_requested'] == true, 'cleanup_unconfirmed': frame['cleanup_unconfirmed'] == true, diff --git a/app/test/features/attention/attention_models_test.dart b/app/test/features/attention/attention_models_test.dart index bc3ba0c8..e2045bbc 100644 --- a/app/test/features/attention/attention_models_test.dart +++ b/app/test/features/attention/attention_models_test.dart @@ -98,6 +98,12 @@ void main() { ), AttentionKind.failed, ); + expect( + attentionKindForRun( + run({'status': 'failed', 'failure_kind': 'expected'}), + ), + isNull, + ); }); test( @@ -151,6 +157,26 @@ void main() { attentionDetail(AttentionKind.failed, run({'diagnostic': null})), 'Run failed', ); + expect( + attentionDetail( + AttentionKind.failed, + run({ + 'diagnostic': 'generic diagnostic', + 'failure_reason': 'Provider authentication failed', + }), + ), + 'Provider authentication failed', + ); + expect( + attentionDetail( + AttentionKind.failed, + run({ + 'diagnostic': 'generic diagnostic', + 'failure_reason': ' ', + }), + ), + 'generic diagnostic', + ); }, ); diff --git a/internal/author/skill/references/notifications.md b/internal/author/skill/references/notifications.md index 1250a01e..0cc88bad 100644 --- a/internal/author/skill/references/notifications.md +++ b/internal/author/skill/references/notifications.md @@ -94,8 +94,11 @@ produce the required three strings, so adding this stage alone cannot cover it. Independently of any `notify` block, the server publishes one run-level attention notification to the same already-configured `system` destination when a run needs a person: when it enters `waiting` (a human stage asked for a decision), or when it -fails or is interrupted. An auto-mode run always notifies for failure/interruption; -a successful auto-mode run notifies only when the operator enables `run_finished`. +has an unexpected failure or interruption. A deliberate route to `#failure` with +its reason is recorded as an expected failure: it remains in Runs but does not +notify or appear in Needs you. Explicit `run_finished`/`on` completion notifications +still include expected failures. An auto-mode run always notifies unexpected +failure/interruption; a successful auto-mode run notifies only when the operator enables `run_finished`. Runs marked `on` notify at completion, while `off` disables these notifications. Queued and running runs send nothing. You do not author these; they are the operator default, so a run you never wired still reaches the operator. diff --git a/internal/pipeline/run_notification_transition_test.go b/internal/pipeline/run_notification_transition_test.go index b6d978ec..77eb0bc4 100644 --- a/internal/pipeline/run_notification_transition_test.go +++ b/internal/pipeline/run_notification_transition_test.go @@ -176,6 +176,54 @@ func TestRunNotificationTransitionModesAndCoverage(t *testing.T) { } } +func TestFailedRunNotificationClassifiesExpectedAndUnexpectedCauses(t *testing.T) { + tests := []struct { + name, status, outcome, diagnostic string + expectedKind string + notify bool + completeStep bool + control *protocol.ControlEvidence + }{ + {name: "blocked with reason", status: "failed", outcome: "blocked", diagnostic: "Blocked: approval is required", expectedKind: "expected", notify: false, completeStep: true}, + {name: "harness error", status: "failed", outcome: "error", diagnostic: "agent exception: process exited", expectedKind: "unexpected", notify: true}, + {name: "budget exceeded", status: "failed", outcome: "error", diagnostic: "token budget exceeded", expectedKind: "unexpected", notify: true}, + {name: "timeout", status: "failed", outcome: "error", diagnostic: "stage timed out", expectedKind: "unexpected", notify: true}, + {name: "interrupted", status: "interrupted", outcome: "", diagnostic: "Server stopped; run was not resumed", expectedKind: "unexpected", notify: true}, + {name: "visit limit", status: "failed", outcome: "error", diagnostic: "stopped stage after 10 visits", expectedKind: "unexpected", notify: true}, + {name: "blocked reason mentions stopping", status: "failed", outcome: "blocked", diagnostic: "Blocked: stopped until approval", expectedKind: "expected", notify: false, completeStep: true}, + {name: "visit limit from control evidence", status: "failed", outcome: "rejected", diagnostic: "Pipelines stopped review after 3 visits on outcome rejected", expectedKind: "unexpected", notify: true, completeStep: true, control: &protocol.ControlEvidence{Visit: 3, MaxVisits: 3, LimitReached: true}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "classification.db"), "auto") + defer s.Close() + if test.completeStep { + if err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "gate", Status: "completed", Outcome: test.outcome, Target: protocol.FailureEnd, Diagnostic: test.diagnostic, Control: test.control}); err != nil { + t.Fatal(err) + } + } + if err := s.FinishRun(ctx, id, test.status, test.outcome, test.diagnostic, nil); err != nil { + t.Fatal(err) + } + got, err := s.GetRun(ctx, id) + if err != nil { + t.Fatal(err) + } + if got.FailureKind != test.expectedKind || got.FailureReason != test.diagnostic { + t.Fatalf("classification: kind=%q reason=%q, want %q/%q", got.FailureKind, got.FailureReason, test.expectedKind, test.diagnostic) + } + ns, err := s.RunNotifications(ctx, id) + if err != nil { + t.Fatal(err) + } + if (len(ns) != 0) != test.notify { + t.Fatalf("notifications=%+v, want notify=%v", ns, test.notify) + } + }) + } +} + func TestWaitingNotificationStageAndWorkerShareLease(t *testing.T) { ctx := context.Background() s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "lease.db"), "on") diff --git a/internal/pipeline/run_notifications.go b/internal/pipeline/run_notifications.go index e9973a2f..0b3201aa 100644 --- a/internal/pipeline/run_notifications.go +++ b/internal/pipeline/run_notifications.go @@ -26,12 +26,12 @@ type RunNotification struct { // enqueueRunTransitionNotification must share the state mutation's transaction. // Read the committed-to-be state, not a runner's proposed outcome or identity. func enqueueRunTransitionNotification(ctx context.Context, tx *sql.Tx, id int64) error { - var mode, status, name, stage, diagnostic, graphJSON string + var mode, status, name, stage, diagnostic, failureKind, graphJSON string var sequence int - if err := tx.QueryRowContext(ctx, `SELECT r.notify_mode,r.status,p.name,r.current_stage,r.diagnostic, + if err := tx.QueryRowContext(ctx, `SELECT r.notify_mode,r.status,p.name,r.current_stage,r.diagnostic,r.failure_kind, COALESCE((SELECT MAX(sequence) FROM run_steps WHERE run_id=r.id),0),v.graph_json FROM pipeline_runs r JOIN pipelines p ON p.id=r.pipeline_id -JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&mode, &status, &name, &stage, &diagnostic, &sequence, &graphJSON); err != nil { + JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&mode, &status, &name, &stage, &diagnostic, &failureKind, &sequence, &graphJSON); err != nil { return err } if mode == "off" { @@ -43,6 +43,15 @@ JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version title = "Run needs your decision" body = "waiting for a human decision" case "failed", "interrupted", "succeeded": + if status == "failed" && failureKind == "expected" && mode != "on" { + settings, err := readNotificationSettings(ctx, tx) + if err != nil { + return err + } + if !settings.RunFinished { + return nil + } + } // Successful runs notify only for runs marked "on", unless the operator // opted into every run end via run_finished, read in this transaction. if status == "succeeded" && mode != "on" { diff --git a/internal/pipeline/runs.go b/internal/pipeline/runs.go index 87a08e20..17ccb1e2 100644 --- a/internal/pipeline/runs.go +++ b/internal/pipeline/runs.go @@ -20,7 +20,7 @@ const runFrom = `FROM pipeline_runs r JOIN pipelines p ON p.id=r.pipeline_id` const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',r.finished_at),'')` const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage, -` + runStamps + `,r.diagnostic, +` + runStamps + `,r.diagnostic,r.failure_kind,r.failure_reason, CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END, (SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode,r.cancel_requested,r.cleanup_unconfirmed,r.output_format, CASE WHEN r.status='waiting' THEN COALESCE((SELECT sequence FROM session_export_gates WHERE run_id=r.id),0) ELSE 0 END, @@ -32,7 +32,7 @@ MAX(r.size_bytes,0) func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error { var output, stageModels string - if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat, &r.ExportPendingSequence, &r.Scheduled, &r.ScheduleID, &r.ScheduleName, &r.SizeBytes); err != nil { + if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &r.FailureKind, &r.FailureReason, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat, &r.ExportPendingSequence, &r.Scheduled, &r.ScheduleID, &r.ScheduleName, &r.SizeBytes); err != nil { return err } if stageModels != "" && stageModels != "{}" { diff --git a/internal/pipeline/runs_mutate.go b/internal/pipeline/runs_mutate.go index 80b8fb62..cefe4e2d 100644 --- a/internal/pipeline/runs_mutate.go +++ b/internal/pipeline/runs_mutate.go @@ -385,12 +385,15 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno return err } defer tx.Rollback() + failureKind, failureReason := classifyRunFailure(ctx, tx, id, status, diagnostic) if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE ? END, outcome=CASE WHEN cancel_requested=1 THEN '' ELSE ? END, -diagnostic=CASE WHEN cancel_requested=1 THEN 'Canceled by operator; execution stopped' ELSE ? END, -output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cleanup_unconfirmed=0 -AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, string(data), id)); err != nil { + diagnostic=CASE WHEN cancel_requested=1 THEN 'Canceled by operator; execution stopped' ELSE ? END, + failure_kind=CASE WHEN cancel_requested=1 THEN '' ELSE ? END, + failure_reason=CASE WHEN cancel_requested=1 THEN '' ELSE ? END, + output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cleanup_unconfirmed=0 + AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, failureKind, failureReason, string(data), id)); err != nil { return err } if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil { @@ -404,6 +407,41 @@ AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_a return tx.Commit() } +// classifyRunFailure uses the durable terminal visit rather than an error string +// alone. A completed visit that deliberately routes to #failure is expected; +// runtime failures, limits and interruptions require operator attention. +func classifyRunFailure(ctx context.Context, tx *sql.Tx, id int64, status, diagnostic string) (string, string) { + if status == "interrupted" { + return "unexpected", diagnostic + } + if status != "failed" { + return "", "" + } + var sequence int64 + var stepStatus, outcome, target, stepDiagnostic string + err := tx.QueryRowContext(ctx, `SELECT sequence,status,outcome,target,diagnostic FROM run_steps WHERE run_id=? ORDER BY sequence DESC LIMIT 1`, id).Scan(&sequence, &stepStatus, &outcome, &target, &stepDiagnostic) + reason := strings.TrimSpace(diagnostic) + if reason == "" { + reason = strings.TrimSpace(stepDiagnostic) + } + if err == nil && target == protocol.FailureEnd && stepStatus == "completed" && outcome != "error" && reason != "" && !stepLimitReached(ctx, tx, id, sequence) { + return "expected", reason + } + return "unexpected", reason +} + +// stepLimitReached reports whether the runner, not the stage, routed the visit +// to #failure because the stage exhausted its visit limit (the recorded control +// evidence), so a stage reason that merely mentions stopping stays expected. +func stepLimitReached(ctx context.Context, tx *sql.Tx, id, sequence int64) bool { + var data string + if tx.QueryRowContext(ctx, `SELECT evidence_json FROM step_control WHERE run_id=? AND sequence=?`, id, sequence).Scan(&data) != nil { + return false + } + var control protocol.ControlEvidence + return json.Unmarshal([]byte(data), &control) == nil && control.LimitReached +} + // SetRunEnvironment records the directory environment a run resolved (names // and files only). A resumed run records its latest resolution. func (s *Store) SetRunEnvironment(ctx context.Context, id int64, environment *protocol.RunEnvironment) error { diff --git a/internal/pipeline/runs_schema.go b/internal/pipeline/runs_schema.go index de0ff3b7..900ad830 100644 --- a/internal/pipeline/runs_schema.go +++ b/internal/pipeline/runs_schema.go @@ -79,6 +79,11 @@ CREATE TABLE IF NOT EXISTS retry_checkpoints ( if err := s.addColumnIfMissing("pipeline_runs", "environment_json", "TEXT NOT NULL DEFAULT ''"); err != nil { return err } + for _, column := range []string{"failure_kind", "failure_reason"} { + if err := s.addColumnIfMissing("pipeline_runs", column, "TEXT NOT NULL DEFAULT ''"); err != nil { + return err + } + } if err := s.dropResourceSchema(); err != nil { return err } @@ -239,11 +244,11 @@ func (s *Store) RecoverRunsKeeping(ctx context.Context, keep map[int64]bool, rea if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET diagnostic=? WHERE run_id=? AND status='interrupted' AND sequence=(SELECT MAX(sequence) FROM run_steps WHERE run_id=?)", "Server restarted; "+reason, id, id); err != nil { return err } - if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cancel_requested=0 AND cleanup_unconfirmed=0", "Server restarted; "+reason+"; run was not resumed", id); err != nil { + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic=?,failure_kind='unexpected',failure_reason=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cancel_requested=0 AND cleanup_unconfirmed=0", "Server restarted; "+reason+"; run was not resumed", "Server restarted; "+reason+"; run was not resumed", id); err != nil { return err } } - if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0"+runFilter, append([]any{protocol.RunStatusWaiting}, keptArgs...)...); err != nil { + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',failure_kind=CASE WHEN cancel_requested=1 THEN '' ELSE 'unexpected' END,failure_reason=CASE WHEN cancel_requested=1 THEN '' ELSE 'Server stopped; owned sessions stopped; run was not resumed' END,finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0"+runFilter, append([]any{protocol.RunStatusWaiting}, keptArgs...)...); err != nil { return err } // An export checkpoint is a paused handoff, not an interrupted model call. diff --git a/internal/protocol/runevents.go b/internal/protocol/runevents.go index 55324a5f..d5915216 100644 --- a/internal/protocol/runevents.go +++ b/internal/protocol/runevents.go @@ -54,6 +54,8 @@ type RunEventFrame struct { CompletedSteps int `json:"completed_steps,omitempty"` CreatedAt string `json:"created_at,omitempty"` FinishedAt string `json:"finished_at,omitempty"` + FailureKind string `json:"failure_kind,omitempty"` + FailureReason string `json:"failure_reason,omitempty"` // Artifacts is the number of files recorded for the referenced step, so a // client can surface artifact capture without reading the step. Artifacts int `json:"artifacts,omitempty"` diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index a1726d0b..96a65c5c 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -187,6 +187,8 @@ type RunSummary struct { CreatedAt string `json:"created_at"` FinishedAt string `json:"finished_at"` Diagnostic string `json:"diagnostic"` + FailureKind string `json:"failure_kind,omitempty"` + FailureReason string `json:"failure_reason,omitempty"` ResultPreview string `json:"result_preview,omitempty"` CompletedSteps int `json:"completed_steps"` // Model and StageModels are the launch-time overrides this run records so it diff --git a/internal/server/notify.go b/internal/server/notify.go index 8e7daf64..ae99ec82 100644 --- a/internal/server/notify.go +++ b/internal/server/notify.go @@ -207,6 +207,9 @@ func (r *runner) notifyTerminalStep(id int64, status string) { if err != nil || len(detail.Steps) == 0 { return } + if status == "failed" && detail.FailureKind == "expected" { + return + } step := detail.Steps[len(detail.Steps)-1] stepStatus, _, _, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic) if !terminal || stepStatus != status { @@ -375,6 +378,14 @@ func (r *runner) notifyRun(id int64, name, status string, sequence int, currentS if event == "" { return } + if status == "failed" { + if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail.FailureKind == "expected" { + settings, settingsErr := r.store.NotificationSettings(r.ctx) + if settingsErr != nil || (r.notifyMode(id) != "on" && !settings.RunFinished) { + return + } + } + } // Failed or interrupted runs always notify; successful runs notify only when // marked "on" or when the operator opted into every run end (run_finished). if event == "succeeded" || (event != protocol.NotifyWaiting && event != "failed" && event != "interrupted") { diff --git a/internal/server/notify_test.go b/internal/server/notify_test.go index 8790e32a..dd6debff 100644 --- a/internal/server/notify_test.go +++ b/internal/server/notify_test.go @@ -158,13 +158,14 @@ func TestTerminalNotificationRecipe(t *testing.T) { for index, test := range []struct { name, stage, outcome, status string executionError, deliveryError bool + notify bool }{ - {"success", "report", "success", "succeeded", false, false}, - {"research blocked", "research", "blocked", "failed", false, false}, - {"research execution error", "research", "error", "failed", true, false}, - {"report blocked", "report", "blocked", "failed", false, false}, - {"report execution error", "report", "error", "failed", true, false}, - {"delivery error preserves success", "report", "success", "succeeded", false, true}, + {"success", "report", "success", "succeeded", false, false, true}, + {"research blocked", "research", "blocked", "failed", false, false, false}, + {"research execution error", "research", "error", "failed", true, false, true}, + {"report blocked", "report", "blocked", "failed", false, false, false}, + {"report execution error", "report", "error", "failed", true, false, true}, + {"delivery error preserves success", "report", "success", "succeeded", false, true, true}, } { t.Run(test.name, func(t *testing.T) { deliveryFailure.Store(test.deliveryError) @@ -187,13 +188,16 @@ func TestTerminalNotificationRecipe(t *testing.T) { if err != nil { t.Fatal(err) } - if detail.Status != test.status || calls.Load()-before != 1 { + if detail.Status != test.status || (calls.Load()-before == 1) != test.notify { t.Fatal("wrong verdict or duplicate/missing terminal alert", detail.Status, calls.Load()-before) } last := detail.Steps[len(detail.Steps)-1] - if last.Stage != test.stage || len(last.Notifications) != 1 { + if last.Stage != test.stage || (len(last.Notifications) == 1) != test.notify { t.Fatal("notification not retained on terminal stage", last) } + if !test.notify { + return + } for _, step := range detail.Steps[:len(detail.Steps)-1] { if len(step.Notifications) != 0 { t.Fatal("intermediate success alerted", step) diff --git a/internal/server/runevents.go b/internal/server/runevents.go index 38b91a76..92fbd5f0 100644 --- a/internal/server/runevents.go +++ b/internal/server/runevents.go @@ -568,6 +568,8 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner CompletedSteps: summary.CompletedSteps, CreatedAt: summary.CreatedAt, FinishedAt: summary.FinishedAt, + FailureKind: summary.FailureKind, + FailureReason: summary.FailureReason, CancelRequested: summary.CancelRequested, CleanupUnconfirmed: summary.CleanupUnconfirmed, ExportPendingSequence: summary.ExportPendingSequence, diff --git a/internal/server/runner.go b/internal/server/runner.go index aeb4df4c..693e90ba 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -1496,6 +1496,8 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag CurrentStage: currentStage, CompletedSteps: completed, FinishedAt: detail.FinishedAt, + FailureKind: detail.FailureKind, + FailureReason: detail.FailureReason, SizeBytes: size, }, true) r.kickSizes()