fix: commit run notification intents with state transitions
This commit is contained in:
@@ -6,6 +6,16 @@ notification hook. `off` suppresses stage, run-level system and browser tray
|
||||
notifications. Needs you remains a status/action surface, not a delivery channel.
|
||||
Summaries and pushed events carry the mode; the browser seeds and folds it.
|
||||
|
||||
`FinishRun` and `MarkWaiting` enqueue the notification intent in the same SQLite
|
||||
transaction as the run/step state change. Enqueue failure rolls back the state;
|
||||
after commit, the worker can recover the intent without any runner callback.
|
||||
The store derives the mode, name, stage and sequence from stored rows. A failed
|
||||
FinishRun does not emit a terminal-success event or run alert. Terminal stage
|
||||
hooks run only after FinishRun commits. Waiting/terminal hooks and the worker
|
||||
share an atomic sending lease to avoid racing publications. Declared hooks get
|
||||
a one-minute initial opportunity to publish their artifacts; if a restart loses
|
||||
the callback, the generic outbox alert becomes eligible after that delay.
|
||||
|
||||
`run_notifications` is an additive, idempotent SQLite migration. Each
|
||||
run/sequence/event/channel has one durable outbox row. Its status is `pending`,
|
||||
`sending`, `sent`, `failed`, or `covered` (a stage already published this event).
|
||||
@@ -22,9 +32,9 @@ publication does not swallow the run-level fallback/retry. Stage artifact
|
||||
publications themselves are not made retryable by this change.
|
||||
|
||||
Publisher acknowledgement is **not device reception**. A crash after publish
|
||||
but before the acknowledgement is saved can duplicate a notification. The
|
||||
transition and enqueue are separate writes: a crash between them can miss an
|
||||
alert; this is bounded retry of enqueued alerts, not guaranteed delivery.
|
||||
but before the acknowledgement is saved can duplicate a notification. State and
|
||||
intent commit together, closing the transition-to-enqueue crash window; this is
|
||||
still bounded retry, not guaranteed delivery.
|
||||
Browser delivery remains page-session best effort and requires permission plus
|
||||
HTTPS/localhost; successful runs do not become Needs you entries. The older L5
|
||||
parity description is attention-only for auto; on now adds success.
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func notificationTransitionRun(t *testing.T, path, mode string, sources ...string) (*Store, int64) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source := humanSource
|
||||
if len(sources) > 0 {
|
||||
source = sources[0]
|
||||
}
|
||||
if _, err := s.Submit(ctx, "stored pipeline", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := s.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := s.Get(ctx, items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, _, err := s.CreateRun(ctx, d, "transition", "workspace", false, nil, map[string]any{"prompt": "task"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, `UPDATE pipeline_runs SET notify_mode=? WHERE id=?`, mode, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.BeginStep(ctx, id, 1, "gate", map[string]any{"prompt": "task"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s, id
|
||||
}
|
||||
|
||||
func TestWaitingHookIntentRecoversWithoutPostCommitCallback(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "lost-hook.db")
|
||||
source := strings.Replace(humanSource, "# stage gate", "# stage gate\n```notify\non=waiting\n```", 1)
|
||||
s, id := notificationTransitionRun(t, path, "on", source)
|
||||
if err := notificationTransition(ctx, s, id, "waiting"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, err := s.ClaimRunNotification(ctx, time.Now().Unix()); err != nil || n != nil {
|
||||
t.Fatalf("worker raced hook opportunity: %+v %v", n, err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
n, err := s.ClaimRunNotification(ctx, time.Now().Add(time.Minute+time.Second).Unix())
|
||||
if err != nil || n == nil || n.RunID != id || n.Event != "waiting" {
|
||||
t.Fatalf("lost hook orphaned waiting alert: %+v %v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
func notificationTransition(ctx context.Context, s *Store, id int64, status string) error {
|
||||
if status == "waiting" {
|
||||
return s.MarkWaiting(ctx, id, 1, "gate", protocol.NewWaitingEvidence("Choose", []string{"deploy"}))
|
||||
}
|
||||
return s.FinishRun(ctx, id, status, "success", "stored diagnostic", map[string]any{"markdown": "result"})
|
||||
}
|
||||
|
||||
func TestRunNotificationTransitionIntentAtomicRollbackAndRecovery(t *testing.T) {
|
||||
for _, status := range []string{"succeeded", "failed", "waiting"} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "atomic.db")
|
||||
s, id := notificationTransitionRun(t, path, "on")
|
||||
// Force enqueue to fail after both state writes would have happened.
|
||||
if _, err := s.db.ExecContext(ctx, `CREATE TRIGGER reject_notification BEFORE INSERT ON run_notifications BEGIN SELECT RAISE(ABORT,'forced enqueue failure'); END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := notificationTransition(ctx, s, id, status); err == nil || !strings.Contains(err.Error(), "forced enqueue failure") {
|
||||
t.Fatalf("enqueue failure missing: %v", err)
|
||||
}
|
||||
got, err := s.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Status != "running" || got.FinishedAt != "" || got.Diagnostic != "" || got.Outcome != "" || len(got.Output) != 0 || got.Steps[0].Status != "running" || len(got.Steps[0].Output) != 0 || len(got.Notifications) != 0 {
|
||||
t.Fatalf("partial transition committed: %+v", got)
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, `DROP TRIGGER reject_notification`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := notificationTransition(ctx, s, id, status); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate a crash immediately after commit: no runner enqueue/delivery.
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err = Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
got, err = s.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Status != status || len(got.Notifications) != 1 || got.Notifications[0].Status != "pending" {
|
||||
t.Fatalf("orphaned state after reopen: %+v", got)
|
||||
}
|
||||
n, err := s.ClaimRunNotification(ctx, 100)
|
||||
if err != nil || n == nil {
|
||||
t.Fatalf("intent not recoverable: %+v %v", n, err)
|
||||
}
|
||||
if n.RunID != id || n.Sequence != 1 || n.Event != status || n.Channel != "system" || !strings.Contains(n.Body, "stored pipeline") || !strings.Contains(n.Body, "gate") {
|
||||
t.Fatalf("intent not derived from stored run: %+v", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNotificationTransitionModesAndCoverage(t *testing.T) {
|
||||
for _, mode := range []string{"auto", "on", "off"} {
|
||||
for _, status := range []string{"succeeded", "failed", "waiting"} {
|
||||
t.Run(mode+"/"+status, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "mode.db"), mode)
|
||||
defer s.Close()
|
||||
if err := notificationTransition(ctx, s, id, status); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns, err := s.RunNotifications(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := 1
|
||||
if mode == "off" || status == "succeeded" && mode != "on" {
|
||||
want = 0
|
||||
}
|
||||
if len(ns) != want {
|
||||
t.Fatalf("mode %s status %s: %+v", mode, status, ns)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
ctx := context.Background()
|
||||
s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "covered.db"), "on")
|
||||
defer s.Close()
|
||||
if err := s.CoverRunNotification(ctx, id, 1, "succeeded"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := notificationTransition(ctx, s, id, "succeeded"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns, err := s.RunNotifications(ctx, id)
|
||||
if err != nil || len(ns) != 1 || ns[0].Status != "covered" {
|
||||
t.Fatalf("stage coverage lost: %+v %v", ns, err)
|
||||
}
|
||||
if n, err := s.ClaimRunNotification(ctx, 100); err != nil || n != nil {
|
||||
t.Fatalf("covered stage duplicated: %+v %v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitingNotificationStageAndWorkerShareLease(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "lease.db"), "on")
|
||||
defer s.Close()
|
||||
if err := notificationTransition(ctx, s, id, "waiting"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lease, allowed, err := s.ClaimStageRunNotification(ctx, id, 1, "waiting", 100)
|
||||
if err != nil || !allowed || lease == nil {
|
||||
t.Fatalf("stage claim: %+v %v %v", lease, allowed, err)
|
||||
}
|
||||
if n, err := s.ClaimRunNotification(ctx, 101); err != nil || n != nil {
|
||||
t.Fatalf("worker duplicated stage: %+v %v", n, err)
|
||||
}
|
||||
if err := s.CoverRunNotification(ctx, id, 1, "waiting"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, err := s.ClaimRunNotification(ctx, 161); err != nil || n != nil {
|
||||
t.Fatalf("covered stage retried: %+v %v", n, err)
|
||||
}
|
||||
if _, allowed, err := s.ClaimStageRunNotification(ctx, id, 1, "waiting", 161); err != nil || allowed {
|
||||
t.Fatalf("stage repeated: %v %v", allowed, err)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,14 @@ package pipeline
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// RunNotification is durable publisher evidence, not proof of device reception.
|
||||
@@ -16,6 +23,101 @@ type RunNotification struct {
|
||||
Attempts int
|
||||
}
|
||||
|
||||
// enqueueRunTransitionNotification must share the state mutation's transaction.
|
||||
// Read the committed-to-be state, not a runner's proposed outcome or identity.
|
||||
func enqueueRunTransitionNotification(ctx context.Context, tx *sql.Tx, id int64) error {
|
||||
var mode, status, name, stage, diagnostic, graphJSON string
|
||||
var sequence int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT r.notify_mode,r.status,p.name,r.current_stage,r.diagnostic,
|
||||
COALESCE((SELECT MAX(sequence) FROM run_steps WHERE run_id=r.id),0),v.graph_json
|
||||
FROM pipeline_runs r JOIN pipelines p ON p.id=r.pipeline_id
|
||||
JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&mode, &status, &name, &stage, &diagnostic, &sequence, &graphJSON); err != nil {
|
||||
return err
|
||||
}
|
||||
if mode == "off" {
|
||||
return nil
|
||||
}
|
||||
var title, body string
|
||||
switch status {
|
||||
case "waiting":
|
||||
title = "Run needs your decision"
|
||||
body = "waiting for a human decision"
|
||||
case "failed":
|
||||
title = "Run failed"
|
||||
body = "run failed"
|
||||
case "succeeded":
|
||||
if mode != "on" {
|
||||
return nil
|
||||
}
|
||||
title = "Run succeeded"
|
||||
body = "run succeeded"
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
body = fmt.Sprintf("Run #%d · %s · %s", id, name, body)
|
||||
if stage != "" {
|
||||
body += " at stage " + stage
|
||||
}
|
||||
if diagnostic != "" {
|
||||
body += ": " + diagnostic
|
||||
}
|
||||
body = strings.NewReplacer("\r", " ", "\n", " ", "\x00", " ").Replace(body)
|
||||
if len(body) > 2000 {
|
||||
body = body[:2000]
|
||||
for !utf8.ValidString(body) {
|
||||
body = body[:len(body)-1]
|
||||
}
|
||||
}
|
||||
// Give a declared stage hook first opportunity to send its artifacts. Its
|
||||
// initial claim bypasses this delay. If the process dies before that callback,
|
||||
// the generic durable alert becomes eligible after one minute, not lost.
|
||||
var graph protocol.Graph
|
||||
if err := json.Unmarshal([]byte(graphJSON), &graph); err != nil {
|
||||
return err
|
||||
}
|
||||
stageEvent := status
|
||||
if status == "succeeded" {
|
||||
stageEvent = "success"
|
||||
}
|
||||
if status == "failed" {
|
||||
stageEvent = "failure"
|
||||
}
|
||||
var nextAttempt int64
|
||||
for _, definition := range graph.Stages {
|
||||
if definition.Key != stage || definition.Notify == nil {
|
||||
continue
|
||||
}
|
||||
for _, event := range definition.Notify.On {
|
||||
if event == stageEvent {
|
||||
nextAttempt = time.Now().Add(time.Minute).Unix()
|
||||
}
|
||||
}
|
||||
}
|
||||
_, err := tx.ExecContext(ctx, `INSERT INTO run_notifications(run_id,sequence,event,channel,title,body,click,next_attempt)
|
||||
VALUES (?,?,?,'system',?,?,?,?) ON CONFLICT(run_id,sequence,event,channel) DO NOTHING`, id, sequence, status, title, body, fmt.Sprintf("pipelines://run/%d", id), nextAttempt)
|
||||
return err
|
||||
}
|
||||
|
||||
// ClaimStageRunNotification arbitrates a transition hook against the outbox worker.
|
||||
// With no transactional intent (legacy/direct stage callers), it leaves the
|
||||
// stage's existing publication path alone. Otherwise the same lease protects
|
||||
// both publishers, so only one may publish this event.
|
||||
func (s *Store) ClaimStageRunNotification(ctx context.Context, id int64, sequence int, event string, now int64) (*RunNotification, bool, error) {
|
||||
n := &RunNotification{RunID: id, Sequence: sequence, Event: event, Channel: "system"}
|
||||
err := s.db.QueryRowContext(ctx, `UPDATE run_notifications SET status='sending',attempts=attempts+1,next_attempt=?
|
||||
WHERE run_id=? AND sequence=? AND event=? AND channel='system' AND status='pending' AND attempts<3 AND (attempts=0 OR next_attempt<=?)
|
||||
RETURNING attempts`, now+60, id, sequence, event, now).Scan(&n.Attempts)
|
||||
if err == nil {
|
||||
return n, true, nil
|
||||
}
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, false, err
|
||||
}
|
||||
var exists bool
|
||||
err = s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM run_notifications WHERE run_id=? AND sequence=? AND event=? AND channel='system')`, id, sequence, event).Scan(&exists)
|
||||
return nil, !exists, err
|
||||
}
|
||||
|
||||
func (s *Store) initRunNotifications() error {
|
||||
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS run_notifications (
|
||||
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, event TEXT NOT NULL, channel TEXT NOT NULL,
|
||||
@@ -52,7 +154,8 @@ RETURNING run_id,sequence,event,channel,title,body,click,status,attempts`, now+6
|
||||
// CoverRunNotification records a successful stage publication covering the same
|
||||
// run event, without publishing a second final alert.
|
||||
func (s *Store) CoverRunNotification(ctx context.Context, id int64, sequence int, event string) error {
|
||||
_, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO run_notifications(run_id,sequence,event,channel,title,body,click,status) VALUES (?,?,?,'system','','','','covered')`, id, sequence, event)
|
||||
_, err := s.db.ExecContext(ctx, `INSERT INTO run_notifications(run_id,sequence,event,channel,title,body,click,status) VALUES (?,?,?,'system','','','','covered')
|
||||
ON CONFLICT(run_id,sequence,event,channel) DO UPDATE SET status='covered',diagnostic='' WHERE run_notifications.status IN ('pending','sending')`, id, sequence, event)
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -183,6 +183,9 @@ func (s *Store) MarkWaiting(ctx context.Context, id int64, sequence int, key str
|
||||
protocol.RunStatusWaiting, key, id)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
@@ -325,12 +328,23 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET
|
||||
status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE ? END,
|
||||
outcome=CASE WHEN cancel_requested=1 THEN '' ELSE ? END,
|
||||
diagnostic=CASE WHEN cancel_requested=1 THEN 'Canceled by operator; execution stopped' ELSE ? END,
|
||||
output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cleanup_unconfirmed=0
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, string(data), id))
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, string(data), id)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (s *Store) MarkCleanupUnconfirmed(ctx context.Context, id int64) error {
|
||||
|
||||
@@ -35,6 +35,23 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
|
||||
if !enabled {
|
||||
return
|
||||
}
|
||||
var lease *pipeline.RunNotification
|
||||
runEvent := stageAttentionEvent(event)
|
||||
if event == "success" && step.Target == protocol.SuccessEnd {
|
||||
runEvent = "succeeded"
|
||||
}
|
||||
if event == protocol.NotifyWaiting || step.Target == protocol.SuccessEnd || step.Target == protocol.FailureEnd {
|
||||
var allowed bool
|
||||
var err error
|
||||
lease, allowed, err = r.store.ClaimStageRunNotification(r.ctx, id, step.Sequence, runEvent, time.Now().Unix())
|
||||
if err != nil {
|
||||
log.Printf("claim stage notification run=%d: %v", id, err)
|
||||
return
|
||||
}
|
||||
if !allowed {
|
||||
return
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
summary := fmt.Sprintf("%s · run #%d · stage %s · %s (%s)", name, id, step.Stage, event, step.Outcome)
|
||||
@@ -134,7 +151,36 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
|
||||
covered = "succeeded"
|
||||
}
|
||||
r.markAttentionSent(id, step.Sequence, covered)
|
||||
break
|
||||
return
|
||||
}
|
||||
}
|
||||
if lease != nil {
|
||||
diagnostic := "stage notification publication failed"
|
||||
if len(receipts) > 0 && receipts[0].Diagnostic != "" {
|
||||
diagnostic = receipts[0].Diagnostic
|
||||
}
|
||||
if err := r.store.FinishRunNotification(r.ctx, *lease, time.Now().Unix(), "", fmt.Errorf("%s", diagnostic)); err != nil {
|
||||
log.Printf("release stage notification run=%d: %v", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Terminal hooks run only after FinishRun commits. They arbitrate against the
|
||||
// durable intent just like waiting hooks, never publishing a proposed success.
|
||||
func (r *runner) notifyTerminalStep(id int64, status string) {
|
||||
detail, err := r.store.GetRun(r.ctx, id)
|
||||
if err != nil || len(detail.Steps) == 0 {
|
||||
return
|
||||
}
|
||||
step := detail.Steps[len(detail.Steps)-1]
|
||||
stepStatus, _, _, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
|
||||
if !terminal || stepStatus != status {
|
||||
return
|
||||
}
|
||||
for _, definition := range detail.Graph.Stages {
|
||||
if definition.Key == step.Stage {
|
||||
r.notifyStep(id, detail.Name, detail.Workspace, definition.Notify, step)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
"git.a-wan.net/alan/pipelines/internal/summarise"
|
||||
)
|
||||
|
||||
func TestRunNotificationPublisherFailureRetriesAfterReopen(t *testing.T) {
|
||||
@@ -58,6 +59,128 @@ func TestRunNotificationPublisherFailureRetriesAfterReopen(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinishRunPersistenceFailureDoesNotAnnounceSuccess(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
s, err := pipeline.Open(filepath.Join(t.TempDir(), "not-finished.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
source := strings.Replace(pipeline.Template([]string{"one"}, true), "# stage one", "# stage one\n```notify\non=success\n```", 1)
|
||||
if _, err := s.Submit(ctx, "test", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := s.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := s.Get(ctx, items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, _, err := s.CreateRunTitled(ctx, d, "not-finished", "workspace", false, nil, nil, pipeline.RunOverrides{NotifyMode: "on"}, summarise.Title{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"id": "wrong-success", "event": "message"})
|
||||
}))
|
||||
defer server.Close()
|
||||
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
||||
r := &runner{ctx: ctx, store: s, events: newRunEventHub()}
|
||||
if err := s.BeginStep(ctx, id, 1, "one", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state := &runState{memory: map[string]memoryValue{}}
|
||||
if _, _, _, _, err := r.finishStage(id, "test", "workspace", d.Graph.Stages[0], state, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: protocol.SuccessEnd, Output: map[string]any{"markdown": "result"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls.Load() != 0 {
|
||||
t.Fatal("terminal stage success published before FinishRun")
|
||||
}
|
||||
sub := r.events.subscribe("test", "workspace", nil, func(protocol.Message) error { return nil })
|
||||
defer sub.close()
|
||||
// Marshal failure occurs before FinishRun can commit its proposed success.
|
||||
r.finishRunState(id, "test", "workspace", "succeeded", "success", "", 0, 0, "", stage.Values{"invalid": make(chan int)})
|
||||
got, err := s.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Status != "running" || len(got.Notifications) != 0 || calls.Load() != 0 {
|
||||
t.Fatalf("unpersisted success announced: %+v calls=%d", got, calls.Load())
|
||||
}
|
||||
sub.mu.Lock()
|
||||
defer sub.mu.Unlock()
|
||||
if len(sub.pending) != 0 {
|
||||
t.Fatalf("unpersisted terminal event emitted: %+v", sub.pending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitingHookSharesTransactionalRunIntent(t *testing.T) {
|
||||
for _, failFirst := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "covered", true: "retry"}[failFirst], func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
s, err := pipeline.Open(filepath.Join(t.TempDir(), "waiting-hook.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
var calls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if calls.Add(1) == 1 && failFirst {
|
||||
http.Error(w, "transient", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"id": "waiting-ack", "event": "message"})
|
||||
}))
|
||||
defer server.Close()
|
||||
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
||||
source := strings.Replace(humanPipelineSource, "# stage gate", "# stage gate\n```notify\non=waiting\n```", 1)
|
||||
if _, err := s.Submit(ctx, "waiting", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := s.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := newRunner(ctx, s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { cancel(); r.schedulerWG.Wait() }()
|
||||
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "waiting-hook", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
r.deliverRunNotifications(time.Now().Add(time.Minute))
|
||||
detail, err := s.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantCalls := int32(1)
|
||||
if failFirst {
|
||||
wantCalls = 2
|
||||
}
|
||||
if calls.Load() != wantCalls || detail.Status != "waiting" || len(detail.Notifications) != 1 {
|
||||
t.Fatalf("waiting hook duplicated/lost: calls=%d detail=%+v", calls.Load(), detail)
|
||||
}
|
||||
status := detail.Notifications[0].Status
|
||||
if status != "sent" && status != "covered" {
|
||||
t.Fatalf("missing acknowledgement: %+v", detail.Notifications)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNotificationModesAndFinalStageDedupe(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -1332,7 +1332,10 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag
|
||||
CompletedSteps: completed,
|
||||
FinishedAt: detail.FinishedAt,
|
||||
}, true)
|
||||
r.notifyRun(id, name, status, sequence, currentStage, diagnostic)
|
||||
if status != protocol.RunStatusCanceled {
|
||||
r.notifyTerminalStep(id, status)
|
||||
}
|
||||
r.deliverRunNotificationBatch(time.Now(), 1)
|
||||
// The run is terminal: nothing more can notify for it.
|
||||
r.forgetAttention(id)
|
||||
r.forgetNotifyMode(id)
|
||||
@@ -1422,9 +1425,11 @@ func (r *runner) finishStage(id int64, name, workspace string, selected protocol
|
||||
}
|
||||
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
||||
}
|
||||
r.notifyStep(id, name, workspace, selected.Notify, step)
|
||||
state.input = stage.Values(step.Output)
|
||||
status, outcome, diagnostic, terminal = terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
|
||||
if !terminal {
|
||||
r.notifyStep(id, name, workspace, selected.Notify, step)
|
||||
}
|
||||
return status, outcome, diagnostic, terminal, nil
|
||||
}
|
||||
|
||||
@@ -1588,7 +1593,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting},
|
||||
}, false)
|
||||
r.notifyStep(id, name, workspace, selected.Notify, waitStep)
|
||||
r.notifyRun(id, name, protocol.RunStatusWaiting, sequence, key, "")
|
||||
r.deliverRunNotificationBatch(time.Now(), 1)
|
||||
return
|
||||
}
|
||||
// A stage that declares a `skipped` route is short-circuited when the run
|
||||
|
||||
Reference in New Issue
Block a user