Merge branch 'pipelines/run-20261112-1047-notify-only-on-unexpected-run-failures-64a9'

This commit is contained in:
2026-11-12 11:24:12 +00:00
15 changed files with 184 additions and 21 deletions
@@ -65,6 +65,9 @@ AttentionKind? attentionKindOf(dynamic status) {
/// onto a waiting run must not hide a genuine human decision.
AttentionKind? attentionKindForRun(JsonMap run) {
final kind = attentionKindOf(run['status']);
if (kind == AttentionKind.failed && _word(run['failure_kind']) == 'expected') {
return null;
}
if (kind != AttentionKind.decision) return kind;
if (_positive(run['export_pending_sequence'])) return null;
return kind;
@@ -77,7 +80,10 @@ String attentionKey(int runId, AttentionKind kind) => '$runId:${kind.wireName}';
/// shows the server's own diagnostic when it recorded one.
String attentionDetail(AttentionKind kind, JsonMap run) {
if (kind == AttentionKind.decision) return 'Waiting for your decision';
final diagnostic = '${run['diagnostic'] ?? ''}'.trim();
final failureReason = '${run['failure_reason'] ?? ''}'.trim();
final diagnostic = failureReason.isNotEmpty
? failureReason
: '${run['diagnostic'] ?? ''}'.trim();
return diagnostic.isNotEmpty ? diagnostic : 'Run failed';
}
@@ -119,6 +119,9 @@ JsonMap _mergeRunFields(JsonMap current, JsonMap frame) {
for (final key in ['outcome', 'current_stage', 'created_at', 'finished_at']) {
if (!_blank(frame[key])) merged[key] = frame[key];
}
for (final key in ['failure_kind', 'failure_reason']) {
if (frame[key] != null) merged[key] = frame[key];
}
// Snapshot and terminal frames carry the measured size; others omit it.
if (frame['size_bytes'] is num) merged['size_bytes'] = frame['size_bytes'];
final oldExport = _positiveInt(current['export_pending_sequence']);
@@ -168,6 +171,8 @@ JsonMap? newRunSummaryFromFrame(JsonMap frame) {
'finished_at': frame['finished_at'] ?? '',
'outcome': frame['outcome'] ?? '',
'current_stage': frame['current_stage'] ?? '',
'failure_kind': frame['failure_kind'] ?? '',
'failure_reason': frame['failure_reason'] ?? '',
'resource_wait': frame['resource_wait'],
'cancel_requested': frame['cancel_requested'] == true,
'cleanup_unconfirmed': frame['cleanup_unconfirmed'] == true,
@@ -98,6 +98,12 @@ void main() {
),
AttentionKind.failed,
);
expect(
attentionKindForRun(
run({'status': 'failed', 'failure_kind': 'expected'}),
),
isNull,
);
});
test(
@@ -151,6 +157,26 @@ void main() {
attentionDetail(AttentionKind.failed, run({'diagnostic': null})),
'Run failed',
);
expect(
attentionDetail(
AttentionKind.failed,
run({
'diagnostic': 'generic diagnostic',
'failure_reason': 'Provider authentication failed',
}),
),
'Provider authentication failed',
);
expect(
attentionDetail(
AttentionKind.failed,
run({
'diagnostic': 'generic diagnostic',
'failure_reason': ' ',
}),
),
'generic diagnostic',
);
},
);
@@ -94,8 +94,11 @@ produce the required three strings, so adding this stage alone cannot cover it.
Independently of any `notify` block, the server publishes one run-level attention
notification to the same already-configured `system` destination when a run needs a
person: when it enters `waiting` (a human stage asked for a decision), or when it
fails or is interrupted. An auto-mode run always notifies for failure/interruption;
a successful auto-mode run notifies only when the operator enables `run_finished`.
has an unexpected failure or interruption. A deliberate route to `#failure` with
its reason is recorded as an expected failure: it remains in Runs but does not
notify or appear in Needs you. Explicit `run_finished`/`on` completion notifications
still include expected failures. An auto-mode run always notifies unexpected
failure/interruption; a successful auto-mode run notifies only when the operator enables `run_finished`.
Runs marked `on` notify at completion, while `off` disables these notifications.
Queued and running runs send nothing. You do not author these; they are the operator
default, so a run you never wired still reaches the operator.
@@ -176,6 +176,54 @@ func TestRunNotificationTransitionModesAndCoverage(t *testing.T) {
}
}
func TestFailedRunNotificationClassifiesExpectedAndUnexpectedCauses(t *testing.T) {
tests := []struct {
name, status, outcome, diagnostic string
expectedKind string
notify bool
completeStep bool
control *protocol.ControlEvidence
}{
{name: "blocked with reason", status: "failed", outcome: "blocked", diagnostic: "Blocked: approval is required", expectedKind: "expected", notify: false, completeStep: true},
{name: "harness error", status: "failed", outcome: "error", diagnostic: "agent exception: process exited", expectedKind: "unexpected", notify: true},
{name: "budget exceeded", status: "failed", outcome: "error", diagnostic: "token budget exceeded", expectedKind: "unexpected", notify: true},
{name: "timeout", status: "failed", outcome: "error", diagnostic: "stage timed out", expectedKind: "unexpected", notify: true},
{name: "interrupted", status: "interrupted", outcome: "", diagnostic: "Server stopped; run was not resumed", expectedKind: "unexpected", notify: true},
{name: "visit limit", status: "failed", outcome: "error", diagnostic: "stopped stage after 10 visits", expectedKind: "unexpected", notify: true},
{name: "blocked reason mentions stopping", status: "failed", outcome: "blocked", diagnostic: "Blocked: stopped until approval", expectedKind: "expected", notify: false, completeStep: true},
{name: "visit limit from control evidence", status: "failed", outcome: "rejected", diagnostic: "Pipelines stopped review after 3 visits on outcome rejected", expectedKind: "unexpected", notify: true, completeStep: true, control: &protocol.ControlEvidence{Visit: 3, MaxVisits: 3, LimitReached: true}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctx := context.Background()
s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "classification.db"), "auto")
defer s.Close()
if test.completeStep {
if err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "gate", Status: "completed", Outcome: test.outcome, Target: protocol.FailureEnd, Diagnostic: test.diagnostic, Control: test.control}); err != nil {
t.Fatal(err)
}
}
if err := s.FinishRun(ctx, id, test.status, test.outcome, test.diagnostic, nil); err != nil {
t.Fatal(err)
}
got, err := s.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
if got.FailureKind != test.expectedKind || got.FailureReason != test.diagnostic {
t.Fatalf("classification: kind=%q reason=%q, want %q/%q", got.FailureKind, got.FailureReason, test.expectedKind, test.diagnostic)
}
ns, err := s.RunNotifications(ctx, id)
if err != nil {
t.Fatal(err)
}
if (len(ns) != 0) != test.notify {
t.Fatalf("notifications=%+v, want notify=%v", ns, test.notify)
}
})
}
}
func TestWaitingNotificationStageAndWorkerShareLease(t *testing.T) {
ctx := context.Background()
s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "lease.db"), "on")
+12 -3
View File
@@ -26,12 +26,12 @@ type RunNotification struct {
// enqueueRunTransitionNotification must share the state mutation's transaction.
// Read the committed-to-be state, not a runner's proposed outcome or identity.
func enqueueRunTransitionNotification(ctx context.Context, tx *sql.Tx, id int64) error {
var mode, status, name, stage, diagnostic, graphJSON string
var mode, status, name, stage, diagnostic, failureKind, graphJSON string
var sequence int
if err := tx.QueryRowContext(ctx, `SELECT r.notify_mode,r.status,p.name,r.current_stage,r.diagnostic,
if err := tx.QueryRowContext(ctx, `SELECT r.notify_mode,r.status,p.name,r.current_stage,r.diagnostic,r.failure_kind,
COALESCE((SELECT MAX(sequence) FROM run_steps WHERE run_id=r.id),0),v.graph_json
FROM pipeline_runs r JOIN pipelines p ON p.id=r.pipeline_id
JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&mode, &status, &name, &stage, &diagnostic, &sequence, &graphJSON); err != nil {
JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&mode, &status, &name, &stage, &diagnostic, &failureKind, &sequence, &graphJSON); err != nil {
return err
}
if mode == "off" {
@@ -43,6 +43,15 @@ JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version
title = "Run needs your decision"
body = "waiting for a human decision"
case "failed", "interrupted", "succeeded":
if status == "failed" && failureKind == "expected" && mode != "on" {
settings, err := readNotificationSettings(ctx, tx)
if err != nil {
return err
}
if !settings.RunFinished {
return nil
}
}
// Successful runs notify only for runs marked "on", unless the operator
// opted into every run end via run_finished, read in this transaction.
if status == "succeeded" && mode != "on" {
+2 -2
View File
@@ -20,7 +20,7 @@ const runFrom = `FROM pipeline_runs r JOIN pipelines p ON p.id=r.pipeline_id`
const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',r.finished_at),'')`
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
` + runStamps + `,r.diagnostic,
` + runStamps + `,r.diagnostic,r.failure_kind,r.failure_reason,
CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END,
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode,r.cancel_requested,r.cleanup_unconfirmed,r.output_format,
CASE WHEN r.status='waiting' THEN COALESCE((SELECT sequence FROM session_export_gates WHERE run_id=r.id),0) ELSE 0 END,
@@ -32,7 +32,7 @@ MAX(r.size_bytes,0)
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
var output, stageModels string
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat, &r.ExportPendingSequence, &r.Scheduled, &r.ScheduleID, &r.ScheduleName, &r.SizeBytes); err != nil {
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &r.FailureKind, &r.FailureReason, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat, &r.ExportPendingSequence, &r.Scheduled, &r.ScheduleID, &r.ScheduleName, &r.SizeBytes); err != nil {
return err
}
if stageModels != "" && stageModels != "{}" {
+41 -3
View File
@@ -385,12 +385,15 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno
return err
}
defer tx.Rollback()
failureKind, failureReason := classifyRunFailure(ctx, tx, id, status, diagnostic)
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET
status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE ? END,
outcome=CASE WHEN cancel_requested=1 THEN '' ELSE ? END,
diagnostic=CASE WHEN cancel_requested=1 THEN 'Canceled by operator; execution stopped' ELSE ? END,
output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cleanup_unconfirmed=0
AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, string(data), id)); err != nil {
diagnostic=CASE WHEN cancel_requested=1 THEN 'Canceled by operator; execution stopped' ELSE ? END,
failure_kind=CASE WHEN cancel_requested=1 THEN '' ELSE ? END,
failure_reason=CASE WHEN cancel_requested=1 THEN '' ELSE ? END,
output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cleanup_unconfirmed=0
AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, failureKind, failureReason, string(data), id)); err != nil {
return err
}
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
@@ -404,6 +407,41 @@ AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_a
return tx.Commit()
}
// classifyRunFailure uses the durable terminal visit rather than an error string
// alone. A completed visit that deliberately routes to #failure is expected;
// runtime failures, limits and interruptions require operator attention.
func classifyRunFailure(ctx context.Context, tx *sql.Tx, id int64, status, diagnostic string) (string, string) {
if status == "interrupted" {
return "unexpected", diagnostic
}
if status != "failed" {
return "", ""
}
var sequence int64
var stepStatus, outcome, target, stepDiagnostic string
err := tx.QueryRowContext(ctx, `SELECT sequence,status,outcome,target,diagnostic FROM run_steps WHERE run_id=? ORDER BY sequence DESC LIMIT 1`, id).Scan(&sequence, &stepStatus, &outcome, &target, &stepDiagnostic)
reason := strings.TrimSpace(diagnostic)
if reason == "" {
reason = strings.TrimSpace(stepDiagnostic)
}
if err == nil && target == protocol.FailureEnd && stepStatus == "completed" && outcome != "error" && reason != "" && !stepLimitReached(ctx, tx, id, sequence) {
return "expected", reason
}
return "unexpected", reason
}
// stepLimitReached reports whether the runner, not the stage, routed the visit
// to #failure because the stage exhausted its visit limit (the recorded control
// evidence), so a stage reason that merely mentions stopping stays expected.
func stepLimitReached(ctx context.Context, tx *sql.Tx, id, sequence int64) bool {
var data string
if tx.QueryRowContext(ctx, `SELECT evidence_json FROM step_control WHERE run_id=? AND sequence=?`, id, sequence).Scan(&data) != nil {
return false
}
var control protocol.ControlEvidence
return json.Unmarshal([]byte(data), &control) == nil && control.LimitReached
}
// SetRunEnvironment records the directory environment a run resolved (names
// and files only). A resumed run records its latest resolution.
func (s *Store) SetRunEnvironment(ctx context.Context, id int64, environment *protocol.RunEnvironment) error {
+7 -2
View File
@@ -79,6 +79,11 @@ CREATE TABLE IF NOT EXISTS retry_checkpoints (
if err := s.addColumnIfMissing("pipeline_runs", "environment_json", "TEXT NOT NULL DEFAULT ''"); err != nil {
return err
}
for _, column := range []string{"failure_kind", "failure_reason"} {
if err := s.addColumnIfMissing("pipeline_runs", column, "TEXT NOT NULL DEFAULT ''"); err != nil {
return err
}
}
if err := s.dropResourceSchema(); err != nil {
return err
}
@@ -239,11 +244,11 @@ func (s *Store) RecoverRunsKeeping(ctx context.Context, keep map[int64]bool, rea
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET diagnostic=? WHERE run_id=? AND status='interrupted' AND sequence=(SELECT MAX(sequence) FROM run_steps WHERE run_id=?)", "Server restarted; "+reason, id, id); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cancel_requested=0 AND cleanup_unconfirmed=0", "Server restarted; "+reason+"; run was not resumed", id); err != nil {
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic=?,failure_kind='unexpected',failure_reason=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cancel_requested=0 AND cleanup_unconfirmed=0", "Server restarted; "+reason+"; run was not resumed", "Server restarted; "+reason+"; run was not resumed", id); err != nil {
return err
}
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0"+runFilter, append([]any{protocol.RunStatusWaiting}, keptArgs...)...); err != nil {
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',failure_kind=CASE WHEN cancel_requested=1 THEN '' ELSE 'unexpected' END,failure_reason=CASE WHEN cancel_requested=1 THEN '' ELSE 'Server stopped; owned sessions stopped; run was not resumed' END,finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0"+runFilter, append([]any{protocol.RunStatusWaiting}, keptArgs...)...); err != nil {
return err
}
// An export checkpoint is a paused handoff, not an interrupted model call.
+2
View File
@@ -54,6 +54,8 @@ type RunEventFrame struct {
CompletedSteps int `json:"completed_steps,omitempty"`
CreatedAt string `json:"created_at,omitempty"`
FinishedAt string `json:"finished_at,omitempty"`
FailureKind string `json:"failure_kind,omitempty"`
FailureReason string `json:"failure_reason,omitempty"`
// Artifacts is the number of files recorded for the referenced step, so a
// client can surface artifact capture without reading the step.
Artifacts int `json:"artifacts,omitempty"`
+2
View File
@@ -187,6 +187,8 @@ type RunSummary struct {
CreatedAt string `json:"created_at"`
FinishedAt string `json:"finished_at"`
Diagnostic string `json:"diagnostic"`
FailureKind string `json:"failure_kind,omitempty"`
FailureReason string `json:"failure_reason,omitempty"`
ResultPreview string `json:"result_preview,omitempty"`
CompletedSteps int `json:"completed_steps"`
// Model and StageModels are the launch-time overrides this run records so it
+11
View File
@@ -207,6 +207,9 @@ func (r *runner) notifyTerminalStep(id int64, status string) {
if err != nil || len(detail.Steps) == 0 {
return
}
if status == "failed" && detail.FailureKind == "expected" {
return
}
step := detail.Steps[len(detail.Steps)-1]
stepStatus, _, _, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
if !terminal || stepStatus != status {
@@ -375,6 +378,14 @@ func (r *runner) notifyRun(id int64, name, status string, sequence int, currentS
if event == "" {
return
}
if status == "failed" {
if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail.FailureKind == "expected" {
settings, settingsErr := r.store.NotificationSettings(r.ctx)
if settingsErr != nil || (r.notifyMode(id) != "on" && !settings.RunFinished) {
return
}
}
}
// Failed or interrupted runs always notify; successful runs notify only when
// marked "on" or when the operator opted into every run end (run_finished).
if event == "succeeded" || (event != protocol.NotifyWaiting && event != "failed" && event != "interrupted") {
+12 -8
View File
@@ -158,13 +158,14 @@ func TestTerminalNotificationRecipe(t *testing.T) {
for index, test := range []struct {
name, stage, outcome, status string
executionError, deliveryError bool
notify bool
}{
{"success", "report", "success", "succeeded", false, false},
{"research blocked", "research", "blocked", "failed", false, false},
{"research execution error", "research", "error", "failed", true, false},
{"report blocked", "report", "blocked", "failed", false, false},
{"report execution error", "report", "error", "failed", true, false},
{"delivery error preserves success", "report", "success", "succeeded", false, true},
{"success", "report", "success", "succeeded", false, false, true},
{"research blocked", "research", "blocked", "failed", false, false, false},
{"research execution error", "research", "error", "failed", true, false, true},
{"report blocked", "report", "blocked", "failed", false, false, false},
{"report execution error", "report", "error", "failed", true, false, true},
{"delivery error preserves success", "report", "success", "succeeded", false, true, true},
} {
t.Run(test.name, func(t *testing.T) {
deliveryFailure.Store(test.deliveryError)
@@ -187,13 +188,16 @@ func TestTerminalNotificationRecipe(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if detail.Status != test.status || calls.Load()-before != 1 {
if detail.Status != test.status || (calls.Load()-before == 1) != test.notify {
t.Fatal("wrong verdict or duplicate/missing terminal alert", detail.Status, calls.Load()-before)
}
last := detail.Steps[len(detail.Steps)-1]
if last.Stage != test.stage || len(last.Notifications) != 1 {
if last.Stage != test.stage || (len(last.Notifications) == 1) != test.notify {
t.Fatal("notification not retained on terminal stage", last)
}
if !test.notify {
return
}
for _, step := range detail.Steps[:len(detail.Steps)-1] {
if len(step.Notifications) != 0 {
t.Fatal("intermediate success alerted", step)
+2
View File
@@ -568,6 +568,8 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
FailureKind: summary.FailureKind,
FailureReason: summary.FailureReason,
CancelRequested: summary.CancelRequested,
CleanupUnconfirmed: summary.CleanupUnconfirmed,
ExportPendingSequence: summary.ExportPendingSequence,
+2
View File
@@ -1496,6 +1496,8 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag
CurrentStage: currentStage,
CompletedSteps: completed,
FinishedAt: detail.FinishedAt,
FailureKind: detail.FailureKind,
FailureReason: detail.FailureReason,
SizeBytes: size,
}, true)
r.kickSizes()