Merge branch 'pipelines/run-20261112-1047-notify-only-on-unexpected-run-failures-64a9'
This commit is contained in:
@@ -65,6 +65,9 @@ AttentionKind? attentionKindOf(dynamic status) {
|
||||
/// onto a waiting run must not hide a genuine human decision.
|
||||
AttentionKind? attentionKindForRun(JsonMap run) {
|
||||
final kind = attentionKindOf(run['status']);
|
||||
if (kind == AttentionKind.failed && _word(run['failure_kind']) == 'expected') {
|
||||
return null;
|
||||
}
|
||||
if (kind != AttentionKind.decision) return kind;
|
||||
if (_positive(run['export_pending_sequence'])) return null;
|
||||
return kind;
|
||||
@@ -77,7 +80,10 @@ String attentionKey(int runId, AttentionKind kind) => '$runId:${kind.wireName}';
|
||||
/// shows the server's own diagnostic when it recorded one.
|
||||
String attentionDetail(AttentionKind kind, JsonMap run) {
|
||||
if (kind == AttentionKind.decision) return 'Waiting for your decision';
|
||||
final diagnostic = '${run['diagnostic'] ?? ''}'.trim();
|
||||
final failureReason = '${run['failure_reason'] ?? ''}'.trim();
|
||||
final diagnostic = failureReason.isNotEmpty
|
||||
? failureReason
|
||||
: '${run['diagnostic'] ?? ''}'.trim();
|
||||
return diagnostic.isNotEmpty ? diagnostic : 'Run failed';
|
||||
}
|
||||
|
||||
|
||||
@@ -119,6 +119,9 @@ JsonMap _mergeRunFields(JsonMap current, JsonMap frame) {
|
||||
for (final key in ['outcome', 'current_stage', 'created_at', 'finished_at']) {
|
||||
if (!_blank(frame[key])) merged[key] = frame[key];
|
||||
}
|
||||
for (final key in ['failure_kind', 'failure_reason']) {
|
||||
if (frame[key] != null) merged[key] = frame[key];
|
||||
}
|
||||
// Snapshot and terminal frames carry the measured size; others omit it.
|
||||
if (frame['size_bytes'] is num) merged['size_bytes'] = frame['size_bytes'];
|
||||
final oldExport = _positiveInt(current['export_pending_sequence']);
|
||||
@@ -168,6 +171,8 @@ JsonMap? newRunSummaryFromFrame(JsonMap frame) {
|
||||
'finished_at': frame['finished_at'] ?? '',
|
||||
'outcome': frame['outcome'] ?? '',
|
||||
'current_stage': frame['current_stage'] ?? '',
|
||||
'failure_kind': frame['failure_kind'] ?? '',
|
||||
'failure_reason': frame['failure_reason'] ?? '',
|
||||
'resource_wait': frame['resource_wait'],
|
||||
'cancel_requested': frame['cancel_requested'] == true,
|
||||
'cleanup_unconfirmed': frame['cleanup_unconfirmed'] == true,
|
||||
|
||||
@@ -98,6 +98,12 @@ void main() {
|
||||
),
|
||||
AttentionKind.failed,
|
||||
);
|
||||
expect(
|
||||
attentionKindForRun(
|
||||
run({'status': 'failed', 'failure_kind': 'expected'}),
|
||||
),
|
||||
isNull,
|
||||
);
|
||||
});
|
||||
|
||||
test(
|
||||
@@ -151,6 +157,26 @@ void main() {
|
||||
attentionDetail(AttentionKind.failed, run({'diagnostic': null})),
|
||||
'Run failed',
|
||||
);
|
||||
expect(
|
||||
attentionDetail(
|
||||
AttentionKind.failed,
|
||||
run({
|
||||
'diagnostic': 'generic diagnostic',
|
||||
'failure_reason': 'Provider authentication failed',
|
||||
}),
|
||||
),
|
||||
'Provider authentication failed',
|
||||
);
|
||||
expect(
|
||||
attentionDetail(
|
||||
AttentionKind.failed,
|
||||
run({
|
||||
'diagnostic': 'generic diagnostic',
|
||||
'failure_reason': ' ',
|
||||
}),
|
||||
),
|
||||
'generic diagnostic',
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -94,8 +94,11 @@ produce the required three strings, so adding this stage alone cannot cover it.
|
||||
Independently of any `notify` block, the server publishes one run-level attention
|
||||
notification to the same already-configured `system` destination when a run needs a
|
||||
person: when it enters `waiting` (a human stage asked for a decision), or when it
|
||||
fails or is interrupted. An auto-mode run always notifies for failure/interruption;
|
||||
a successful auto-mode run notifies only when the operator enables `run_finished`.
|
||||
has an unexpected failure or interruption. A deliberate route to `#failure` with
|
||||
its reason is recorded as an expected failure: it remains in Runs but does not
|
||||
notify or appear in Needs you. Explicit `run_finished`/`on` completion notifications
|
||||
still include expected failures. An auto-mode run always notifies unexpected
|
||||
failure/interruption; a successful auto-mode run notifies only when the operator enables `run_finished`.
|
||||
Runs marked `on` notify at completion, while `off` disables these notifications.
|
||||
Queued and running runs send nothing. You do not author these; they are the operator
|
||||
default, so a run you never wired still reaches the operator.
|
||||
|
||||
@@ -176,6 +176,54 @@ func TestRunNotificationTransitionModesAndCoverage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedRunNotificationClassifiesExpectedAndUnexpectedCauses(t *testing.T) {
|
||||
tests := []struct {
|
||||
name, status, outcome, diagnostic string
|
||||
expectedKind string
|
||||
notify bool
|
||||
completeStep bool
|
||||
control *protocol.ControlEvidence
|
||||
}{
|
||||
{name: "blocked with reason", status: "failed", outcome: "blocked", diagnostic: "Blocked: approval is required", expectedKind: "expected", notify: false, completeStep: true},
|
||||
{name: "harness error", status: "failed", outcome: "error", diagnostic: "agent exception: process exited", expectedKind: "unexpected", notify: true},
|
||||
{name: "budget exceeded", status: "failed", outcome: "error", diagnostic: "token budget exceeded", expectedKind: "unexpected", notify: true},
|
||||
{name: "timeout", status: "failed", outcome: "error", diagnostic: "stage timed out", expectedKind: "unexpected", notify: true},
|
||||
{name: "interrupted", status: "interrupted", outcome: "", diagnostic: "Server stopped; run was not resumed", expectedKind: "unexpected", notify: true},
|
||||
{name: "visit limit", status: "failed", outcome: "error", diagnostic: "stopped stage after 10 visits", expectedKind: "unexpected", notify: true},
|
||||
{name: "blocked reason mentions stopping", status: "failed", outcome: "blocked", diagnostic: "Blocked: stopped until approval", expectedKind: "expected", notify: false, completeStep: true},
|
||||
{name: "visit limit from control evidence", status: "failed", outcome: "rejected", diagnostic: "Pipelines stopped review after 3 visits on outcome rejected", expectedKind: "unexpected", notify: true, completeStep: true, control: &protocol.ControlEvidence{Visit: 3, MaxVisits: 3, LimitReached: true}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "classification.db"), "auto")
|
||||
defer s.Close()
|
||||
if test.completeStep {
|
||||
if err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "gate", Status: "completed", Outcome: test.outcome, Target: protocol.FailureEnd, Diagnostic: test.diagnostic, Control: test.control}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := s.FinishRun(ctx, id, test.status, test.outcome, test.diagnostic, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := s.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.FailureKind != test.expectedKind || got.FailureReason != test.diagnostic {
|
||||
t.Fatalf("classification: kind=%q reason=%q, want %q/%q", got.FailureKind, got.FailureReason, test.expectedKind, test.diagnostic)
|
||||
}
|
||||
ns, err := s.RunNotifications(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if (len(ns) != 0) != test.notify {
|
||||
t.Fatalf("notifications=%+v, want notify=%v", ns, test.notify)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitingNotificationStageAndWorkerShareLease(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "lease.db"), "on")
|
||||
|
||||
@@ -26,12 +26,12 @@ type RunNotification struct {
|
||||
// enqueueRunTransitionNotification must share the state mutation's transaction.
|
||||
// Read the committed-to-be state, not a runner's proposed outcome or identity.
|
||||
func enqueueRunTransitionNotification(ctx context.Context, tx *sql.Tx, id int64) error {
|
||||
var mode, status, name, stage, diagnostic, graphJSON string
|
||||
var mode, status, name, stage, diagnostic, failureKind, graphJSON string
|
||||
var sequence int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT r.notify_mode,r.status,p.name,r.current_stage,r.diagnostic,
|
||||
if err := tx.QueryRowContext(ctx, `SELECT r.notify_mode,r.status,p.name,r.current_stage,r.diagnostic,r.failure_kind,
|
||||
COALESCE((SELECT MAX(sequence) FROM run_steps WHERE run_id=r.id),0),v.graph_json
|
||||
FROM pipeline_runs r JOIN pipelines p ON p.id=r.pipeline_id
|
||||
JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&mode, &status, &name, &stage, &diagnostic, &sequence, &graphJSON); err != nil {
|
||||
JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&mode, &status, &name, &stage, &diagnostic, &failureKind, &sequence, &graphJSON); err != nil {
|
||||
return err
|
||||
}
|
||||
if mode == "off" {
|
||||
@@ -43,6 +43,15 @@ JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version
|
||||
title = "Run needs your decision"
|
||||
body = "waiting for a human decision"
|
||||
case "failed", "interrupted", "succeeded":
|
||||
if status == "failed" && failureKind == "expected" && mode != "on" {
|
||||
settings, err := readNotificationSettings(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !settings.RunFinished {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// Successful runs notify only for runs marked "on", unless the operator
|
||||
// opted into every run end via run_finished, read in this transaction.
|
||||
if status == "succeeded" && mode != "on" {
|
||||
|
||||
@@ -20,7 +20,7 @@ const runFrom = `FROM pipeline_runs r JOIN pipelines p ON p.id=r.pipeline_id`
|
||||
const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',r.finished_at),'')`
|
||||
|
||||
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
|
||||
` + runStamps + `,r.diagnostic,
|
||||
` + runStamps + `,r.diagnostic,r.failure_kind,r.failure_reason,
|
||||
CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END,
|
||||
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode,r.cancel_requested,r.cleanup_unconfirmed,r.output_format,
|
||||
CASE WHEN r.status='waiting' THEN COALESCE((SELECT sequence FROM session_export_gates WHERE run_id=r.id),0) ELSE 0 END,
|
||||
@@ -32,7 +32,7 @@ MAX(r.size_bytes,0)
|
||||
|
||||
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
|
||||
var output, stageModels string
|
||||
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat, &r.ExportPendingSequence, &r.Scheduled, &r.ScheduleID, &r.ScheduleName, &r.SizeBytes); err != nil {
|
||||
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &r.FailureKind, &r.FailureReason, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat, &r.ExportPendingSequence, &r.Scheduled, &r.ScheduleID, &r.ScheduleName, &r.SizeBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
if stageModels != "" && stageModels != "{}" {
|
||||
|
||||
@@ -385,12 +385,15 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
failureKind, failureReason := classifyRunFailure(ctx, tx, id, status, diagnostic)
|
||||
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET
|
||||
status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE ? END,
|
||||
outcome=CASE WHEN cancel_requested=1 THEN '' ELSE ? END,
|
||||
diagnostic=CASE WHEN cancel_requested=1 THEN 'Canceled by operator; execution stopped' ELSE ? END,
|
||||
output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cleanup_unconfirmed=0
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, string(data), id)); err != nil {
|
||||
diagnostic=CASE WHEN cancel_requested=1 THEN 'Canceled by operator; execution stopped' ELSE ? END,
|
||||
failure_kind=CASE WHEN cancel_requested=1 THEN '' ELSE ? END,
|
||||
failure_reason=CASE WHEN cancel_requested=1 THEN '' ELSE ? END,
|
||||
output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cleanup_unconfirmed=0
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, failureKind, failureReason, string(data), id)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
|
||||
@@ -404,6 +407,41 @@ AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_a
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// classifyRunFailure uses the durable terminal visit rather than an error string
|
||||
// alone. A completed visit that deliberately routes to #failure is expected;
|
||||
// runtime failures, limits and interruptions require operator attention.
|
||||
func classifyRunFailure(ctx context.Context, tx *sql.Tx, id int64, status, diagnostic string) (string, string) {
|
||||
if status == "interrupted" {
|
||||
return "unexpected", diagnostic
|
||||
}
|
||||
if status != "failed" {
|
||||
return "", ""
|
||||
}
|
||||
var sequence int64
|
||||
var stepStatus, outcome, target, stepDiagnostic string
|
||||
err := tx.QueryRowContext(ctx, `SELECT sequence,status,outcome,target,diagnostic FROM run_steps WHERE run_id=? ORDER BY sequence DESC LIMIT 1`, id).Scan(&sequence, &stepStatus, &outcome, &target, &stepDiagnostic)
|
||||
reason := strings.TrimSpace(diagnostic)
|
||||
if reason == "" {
|
||||
reason = strings.TrimSpace(stepDiagnostic)
|
||||
}
|
||||
if err == nil && target == protocol.FailureEnd && stepStatus == "completed" && outcome != "error" && reason != "" && !stepLimitReached(ctx, tx, id, sequence) {
|
||||
return "expected", reason
|
||||
}
|
||||
return "unexpected", reason
|
||||
}
|
||||
|
||||
// stepLimitReached reports whether the runner, not the stage, routed the visit
|
||||
// to #failure because the stage exhausted its visit limit (the recorded control
|
||||
// evidence), so a stage reason that merely mentions stopping stays expected.
|
||||
func stepLimitReached(ctx context.Context, tx *sql.Tx, id, sequence int64) bool {
|
||||
var data string
|
||||
if tx.QueryRowContext(ctx, `SELECT evidence_json FROM step_control WHERE run_id=? AND sequence=?`, id, sequence).Scan(&data) != nil {
|
||||
return false
|
||||
}
|
||||
var control protocol.ControlEvidence
|
||||
return json.Unmarshal([]byte(data), &control) == nil && control.LimitReached
|
||||
}
|
||||
|
||||
// SetRunEnvironment records the directory environment a run resolved (names
|
||||
// and files only). A resumed run records its latest resolution.
|
||||
func (s *Store) SetRunEnvironment(ctx context.Context, id int64, environment *protocol.RunEnvironment) error {
|
||||
|
||||
@@ -79,6 +79,11 @@ CREATE TABLE IF NOT EXISTS retry_checkpoints (
|
||||
if err := s.addColumnIfMissing("pipeline_runs", "environment_json", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, column := range []string{"failure_kind", "failure_reason"} {
|
||||
if err := s.addColumnIfMissing("pipeline_runs", column, "TEXT NOT NULL DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := s.dropResourceSchema(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -239,11 +244,11 @@ func (s *Store) RecoverRunsKeeping(ctx context.Context, keep map[int64]bool, rea
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET diagnostic=? WHERE run_id=? AND status='interrupted' AND sequence=(SELECT MAX(sequence) FROM run_steps WHERE run_id=?)", "Server restarted; "+reason, id, id); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cancel_requested=0 AND cleanup_unconfirmed=0", "Server restarted; "+reason+"; run was not resumed", id); err != nil {
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic=?,failure_kind='unexpected',failure_reason=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cancel_requested=0 AND cleanup_unconfirmed=0", "Server restarted; "+reason+"; run was not resumed", "Server restarted; "+reason+"; run was not resumed", id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0"+runFilter, append([]any{protocol.RunStatusWaiting}, keptArgs...)...); err != nil {
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',failure_kind=CASE WHEN cancel_requested=1 THEN '' ELSE 'unexpected' END,failure_reason=CASE WHEN cancel_requested=1 THEN '' ELSE 'Server stopped; owned sessions stopped; run was not resumed' END,finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0"+runFilter, append([]any{protocol.RunStatusWaiting}, keptArgs...)...); err != nil {
|
||||
return err
|
||||
}
|
||||
// An export checkpoint is a paused handoff, not an interrupted model call.
|
||||
|
||||
@@ -54,6 +54,8 @@ type RunEventFrame struct {
|
||||
CompletedSteps int `json:"completed_steps,omitempty"`
|
||||
CreatedAt string `json:"created_at,omitempty"`
|
||||
FinishedAt string `json:"finished_at,omitempty"`
|
||||
FailureKind string `json:"failure_kind,omitempty"`
|
||||
FailureReason string `json:"failure_reason,omitempty"`
|
||||
// Artifacts is the number of files recorded for the referenced step, so a
|
||||
// client can surface artifact capture without reading the step.
|
||||
Artifacts int `json:"artifacts,omitempty"`
|
||||
|
||||
@@ -187,6 +187,8 @@ type RunSummary struct {
|
||||
CreatedAt string `json:"created_at"`
|
||||
FinishedAt string `json:"finished_at"`
|
||||
Diagnostic string `json:"diagnostic"`
|
||||
FailureKind string `json:"failure_kind,omitempty"`
|
||||
FailureReason string `json:"failure_reason,omitempty"`
|
||||
ResultPreview string `json:"result_preview,omitempty"`
|
||||
CompletedSteps int `json:"completed_steps"`
|
||||
// Model and StageModels are the launch-time overrides this run records so it
|
||||
|
||||
@@ -207,6 +207,9 @@ func (r *runner) notifyTerminalStep(id int64, status string) {
|
||||
if err != nil || len(detail.Steps) == 0 {
|
||||
return
|
||||
}
|
||||
if status == "failed" && detail.FailureKind == "expected" {
|
||||
return
|
||||
}
|
||||
step := detail.Steps[len(detail.Steps)-1]
|
||||
stepStatus, _, _, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
|
||||
if !terminal || stepStatus != status {
|
||||
@@ -375,6 +378,14 @@ func (r *runner) notifyRun(id int64, name, status string, sequence int, currentS
|
||||
if event == "" {
|
||||
return
|
||||
}
|
||||
if status == "failed" {
|
||||
if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail.FailureKind == "expected" {
|
||||
settings, settingsErr := r.store.NotificationSettings(r.ctx)
|
||||
if settingsErr != nil || (r.notifyMode(id) != "on" && !settings.RunFinished) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// Failed or interrupted runs always notify; successful runs notify only when
|
||||
// marked "on" or when the operator opted into every run end (run_finished).
|
||||
if event == "succeeded" || (event != protocol.NotifyWaiting && event != "failed" && event != "interrupted") {
|
||||
|
||||
@@ -158,13 +158,14 @@ func TestTerminalNotificationRecipe(t *testing.T) {
|
||||
for index, test := range []struct {
|
||||
name, stage, outcome, status string
|
||||
executionError, deliveryError bool
|
||||
notify bool
|
||||
}{
|
||||
{"success", "report", "success", "succeeded", false, false},
|
||||
{"research blocked", "research", "blocked", "failed", false, false},
|
||||
{"research execution error", "research", "error", "failed", true, false},
|
||||
{"report blocked", "report", "blocked", "failed", false, false},
|
||||
{"report execution error", "report", "error", "failed", true, false},
|
||||
{"delivery error preserves success", "report", "success", "succeeded", false, true},
|
||||
{"success", "report", "success", "succeeded", false, false, true},
|
||||
{"research blocked", "research", "blocked", "failed", false, false, false},
|
||||
{"research execution error", "research", "error", "failed", true, false, true},
|
||||
{"report blocked", "report", "blocked", "failed", false, false, false},
|
||||
{"report execution error", "report", "error", "failed", true, false, true},
|
||||
{"delivery error preserves success", "report", "success", "succeeded", false, true, true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
deliveryFailure.Store(test.deliveryError)
|
||||
@@ -187,13 +188,16 @@ func TestTerminalNotificationRecipe(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.Status != test.status || calls.Load()-before != 1 {
|
||||
if detail.Status != test.status || (calls.Load()-before == 1) != test.notify {
|
||||
t.Fatal("wrong verdict or duplicate/missing terminal alert", detail.Status, calls.Load()-before)
|
||||
}
|
||||
last := detail.Steps[len(detail.Steps)-1]
|
||||
if last.Stage != test.stage || len(last.Notifications) != 1 {
|
||||
if last.Stage != test.stage || (len(last.Notifications) == 1) != test.notify {
|
||||
t.Fatal("notification not retained on terminal stage", last)
|
||||
}
|
||||
if !test.notify {
|
||||
return
|
||||
}
|
||||
for _, step := range detail.Steps[:len(detail.Steps)-1] {
|
||||
if len(step.Notifications) != 0 {
|
||||
t.Fatal("intermediate success alerted", step)
|
||||
|
||||
@@ -568,6 +568,8 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
|
||||
CompletedSteps: summary.CompletedSteps,
|
||||
CreatedAt: summary.CreatedAt,
|
||||
FinishedAt: summary.FinishedAt,
|
||||
FailureKind: summary.FailureKind,
|
||||
FailureReason: summary.FailureReason,
|
||||
CancelRequested: summary.CancelRequested,
|
||||
CleanupUnconfirmed: summary.CleanupUnconfirmed,
|
||||
ExportPendingSequence: summary.ExportPendingSequence,
|
||||
|
||||
@@ -1496,6 +1496,8 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag
|
||||
CurrentStage: currentStage,
|
||||
CompletedSteps: completed,
|
||||
FinishedAt: detail.FinishedAt,
|
||||
FailureKind: detail.FailureKind,
|
||||
FailureReason: detail.FailureReason,
|
||||
SizeBytes: size,
|
||||
}, true)
|
||||
r.kickSizes()
|
||||
|
||||
Reference in New Issue
Block a user