Merge main into lane-srv3: combine the review-loop control change with main's notification and chat work

Keeps this branch's advisory review edge, bounded blocked_retryable implementer
edge and their tests alongside main's run-attention/ntfy notification work, the
split of the Android chat conversation screen, and the retention/history changes.
No conflicts: the two sides touch disjoint files.
This commit is contained in:
2026-10-17 17:06:54 +01:00
28 changed files with 3403 additions and 1824 deletions
+125 -7
View File
@@ -34,15 +34,14 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
if !enabled {
return
}
// The stage's own policy already covers this attention event, so the
// run-level default must not repeat it. Recorded before delivery so a failed
// stage notification is reported by its own receipt, not silently duplicated.
r.markAttentionSent(id, step.Sequence, stageAttentionEvent(event))
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
defer cancel()
summary := fmt.Sprintf("%s · run #%d · stage %s · %s (%s)", name, id, step.Stage, event, step.Outcome)
if len(summary) > 1800 {
summary = summary[:1800]
for !utf8.ValidString(summary) {
summary = summary[:len(summary)-1]
}
}
summary = truncateUTF8(summary, 1800)
title := fmt.Sprintf("Pipeline stage %s", event)
receipts := []protocol.NotificationReceipt{}
persist := func() bool {
@@ -60,7 +59,7 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
return
}
receipt := &receipts[len(receipts)-1]
result, err := stage.PublishNtfy(ctx, "system", title, summary, filename, data)
result, err := stage.PublishNtfy(ctx, "system", title, summary, filename, data, "")
if err != nil {
receipt.Status = "failed"
receipt.Diagnostic = err.Error()
@@ -131,6 +130,125 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
}
}
// runAttentionEvent maps a run status to the operator attention notification it
// deserves. Only a run parked on a person (`waiting`) or a run that failed needs
// the operator; every other status — queued, running, succeeded, canceled,
// interrupted — returns "" so the mapping is the single place that decides
// whether a run produces a notification.
func runAttentionEvent(status string) string {
switch strings.TrimSpace(strings.ToLower(status)) {
case protocol.RunStatusWaiting:
return protocol.NotifyWaiting
case "failed":
return "failed"
}
return ""
}
// stageAttentionEvent maps a stage's notification event to the run-attention
// event it covers, or "" when the stage event is not a run-attention event (a
// success digest, for example). It is the one place the two vocabularies meet.
func stageAttentionEvent(event string) string {
switch event {
case protocol.NotifyWaiting:
return protocol.NotifyWaiting
case "failure":
return "failed"
}
return ""
}
// markAttentionSent records that [event] has already been delivered for step
// [sequence] of run [id] (by a stage policy or by the run-level default), so the
// same genuine event is never notified twice. The step sequence is part of the
// key because one run can wait on a person more than once; each wait is its own
// event.
func (r *runner) markAttentionSent(id int64, sequence int, event string) {
if event == "" {
return
}
r.attentionMu.Lock()
defer r.attentionMu.Unlock()
if r.attentionSent[id] == nil {
r.attentionSent[id] = map[string]bool{}
}
r.attentionSent[id][fmt.Sprintf("%d:%s", sequence, event)] = true
}
// attentionAlready reports whether [event] was already delivered for step
// [sequence] of run [id].
func (r *runner) attentionAlready(id int64, sequence int, event string) bool {
r.attentionMu.Lock()
defer r.attentionMu.Unlock()
return r.attentionSent[id][fmt.Sprintf("%d:%s", sequence, event)]
}
// forgetAttention drops a run's delivered-event set once it is terminal, so the
// registry is bounded by the runs still in flight.
func (r *runner) forgetAttention(id int64) {
r.attentionMu.Lock()
defer r.attentionMu.Unlock()
delete(r.attentionSent, id)
}
// runAttentionClick is the deep link a run notification carries so tapping it
// opens the run in the Android app, whose intent filter registers this scheme
// and host.
func runAttentionClick(id int64) string {
return fmt.Sprintf("pipelines://run/%d", id)
}
// notifyRun publishes one run-level attention notification to the operator's
// already-configured "system" destination. It is the run counterpart of
// notifyStep and reuses the same PublishNtfy machinery, destination and token
// handling, so no second publisher, topic or service is introduced. The run id
// and pipeline name identify the run and the click action opens it in the app.
// A run that needs nobody produces nothing, and a delivery failure is logged and
// never fails the run: this is a best-effort alert, not persisted run evidence.
func (r *runner) notifyRun(id int64, name, status string, sequence int, currentStage, diagnostic string) {
event := runAttentionEvent(status)
if event == "" {
return
}
if r.attentionAlready(id, sequence, event) {
return
}
// Record the delivery attempt before publishing so a repeated transition for
// the same run and event (a re-request, a restart-free re-entry) never
// republishes; delivery itself is never retried.
r.markAttentionSent(id, sequence, event)
title := "Run needs your decision"
summary := fmt.Sprintf("Run #%d · %s · waiting for a human decision", id, name)
if event == "failed" {
title = "Run failed"
summary = fmt.Sprintf("Run #%d · %s · run failed", id, name)
}
if currentStage != "" {
summary += " at stage " + currentStage
}
if diagnostic != "" {
summary += ": " + diagnostic
}
summary = truncateUTF8(summary, 2000)
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
defer cancel()
if _, err := stage.PublishNtfy(ctx, "system", title, summary, "", nil, runAttentionClick(id)); err != nil {
log.Printf("run attention notification run=%d: %v", id, err)
}
}
// truncateUTF8 bounds a notification string without splitting a multi-byte rune.
func truncateUTF8(text string, max int) string {
if len(text) <= max {
return text
}
text = text[:max]
for !utf8.ValidString(text) {
text = text[:len(text)-1]
}
return text
}
func notificationArtifact(workspace, name string) ([]byte, error) {
if name == "" || filepath.IsAbs(name) || filepath.Clean(name) != name || strings.ContainsAny(name, "\\\x00\r\n") {
return nil, fmt.Errorf("artifact must be a clean workspace-relative file path")
+211
View File
@@ -260,3 +260,214 @@ func TestNotificationArtifactPathSafety(t *testing.T) {
t.Fatal("safe artifact rejected", err)
}
}
// TestRunAttentionEventMapping pins the single decision of whether a run needs
// the operator: a run parked on a person (waiting) or a failed run, and nothing
// else. A queued, running, succeeded, canceled or interrupted run never notifies.
func TestRunAttentionEventMapping(t *testing.T) {
for _, test := range []struct{ status, want string }{
{"waiting", protocol.NotifyWaiting},
{"failed", "failed"},
{"Warning", ""},
{"queued", ""},
{"running", ""},
{"succeeded", ""},
{"canceled", ""},
{"interrupted", ""},
{"", ""},
} {
if got := runAttentionEvent(test.status); got != test.want {
t.Errorf("runAttentionEvent(%q) = %q, want %q", test.status, got, test.want)
}
}
}
// TestRunAttentionNotificationDelivery proves the operator-facing contract: the
// already-configured "system" destination and topic are used, the payload names
// the run and its pipeline, the click action opens the run, only waiting/failed
// publish, and a run that stays waiting is never republished.
func TestRunAttentionNotificationDelivery(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "attention.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
type delivered struct {
method, path, title, message, click, authorization string
}
deliveries := make(chan delivered, 8)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
decoder := new(mime.WordDecoder)
title, _ := decoder.DecodeHeader(r.Header.Get("Title"))
message, _ := decoder.DecodeHeader(r.Header.Get("Message"))
deliveries <- delivered{r.Method, r.URL.Path, title, message, r.Header.Get("Click"), r.Header.Get("Authorization")}
_ = json.NewEncoder(w).Encode(map[string]string{"id": "attention-receipt", "event": "message"})
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
// A run that needs nobody publishes nothing.
for _, status := range []string{"queued", "running", "succeeded", "canceled", "interrupted"} {
r.notifyRun(7, "release", status, 1, "build", "")
}
select {
case got := <-deliveries:
t.Fatalf("a run that needs nobody published: %+v", got)
default:
}
// A waiting run publishes once, with the destination, identity and click.
r.notifyRun(7, "release", protocol.RunStatusWaiting, 1, "approve", "")
// A run that stays waiting is never republished, on a later call or a re-request.
r.notifyRun(7, "release", protocol.RunStatusWaiting, 1, "approve", "")
// A failed run publishes once, with its diagnostic.
r.notifyRun(7, "release", "failed", 2, "deploy", "provider returned 500")
first := <-deliveries
if first.method != http.MethodPost || first.path != "/pipelines" {
t.Fatalf("wrong destination/topic: %+v", first)
}
if first.title != "Run needs your decision" || !strings.Contains(first.message, "Run #7") || !strings.Contains(first.message, "release") || !strings.Contains(first.message, "decision") || !strings.Contains(first.message, "approve") {
t.Fatalf("waiting notification does not identify the run/pipeline/need: %+v", first)
}
if first.click != "pipelines://run/7" {
t.Fatalf("waiting notification click does not open the run: %q", first.click)
}
if first.authorization != "" {
t.Fatalf("notification leaked an authorization header: %q", first.authorization)
}
second := <-deliveries
if second.title != "Run failed" || !strings.Contains(second.message, "Run #7") || !strings.Contains(second.message, "release") || !strings.Contains(second.message, "deploy") || !strings.Contains(second.message, "provider returned 500") {
t.Fatalf("failed notification is not actionable: %+v", second)
}
if second.click != "pipelines://run/7" {
t.Fatalf("failed notification click does not open the run: %q", second.click)
}
select {
case got := <-deliveries:
t.Fatalf("third notification delivered for two genuine events: %+v", got)
default:
}
}
// TestRunAttentionSuppressedByStagePolicy proves a stage's own notify policy and
// the run-level default cannot both fire for the same event: the author's policy
// wins, so the operator gets exactly one notification.
func TestRunAttentionSuppressedByStagePolicy(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "suppress.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.markAttentionSent(11, 5, "failed")
r.markAttentionSent(11, 6, "waiting")
if !r.attentionAlready(11, 5, "failed") || !r.attentionAlready(11, 6, "waiting") {
t.Fatal("stage-policy coverage not recorded")
}
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
calls.Add(1)
_ = json.NewEncoder(w).Encode(map[string]string{"id": "suppressed", "event": "message"})
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
r.notifyRun(11, "release", "failed", 5, "deploy", "")
r.notifyRun(11, "release", protocol.RunStatusWaiting, 6, "approve", "")
if calls.Load() != 0 {
t.Fatalf("run-level default repeated a stage-policy event %d times", calls.Load())
}
}
// TestHumanRunAttentionNotification proves the run loop publishes the run-level
// attention notification itself when the pipeline declares no notify policy: the
// transition to waiting reaches the operator's configured destination exactly
// once, with the run, pipeline and a run-opening click.
func TestHumanRunAttentionNotification(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "human-attention.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
type delivered struct{ title, message, click string }
deliveries := make(chan delivered, 4)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
decoder := new(mime.WordDecoder)
title, _ := decoder.DecodeHeader(r.Header.Get("Title"))
message, _ := decoder.DecodeHeader(r.Header.Get("Message"))
deliveries <- delivered{title, message, r.Header.Get("Click")}
_ = json.NewEncoder(w).Encode(map[string]string{"id": "human-attention", "event": "message"})
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
if _, err := store.Submit(ctx, "human", humanPipelineSource); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r.wg.Wait()
r.execute = successExecute
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "human-attention", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
got := <-deliveries
if got.title != "Run needs your decision" || !strings.Contains(got.message, fmt.Sprintf("Run #%d", run.ID)) || !strings.Contains(got.message, "human") {
t.Fatalf("run-level waiting notification wrong: %+v", got)
}
if got.click != fmt.Sprintf("pipelines://run/%d", run.ID) {
t.Fatalf("run-level waiting notification click wrong: %q", got.click)
}
select {
case extra := <-deliveries:
t.Fatalf("waiting run notified again: %+v", extra)
default:
}
}
// TestRunAttentionNotifiesEachGenuineWait proves the step sequence is part of the
// dedup key: a run that waits on a person twice notifies each time (two genuine
// events), while the same wait is never repeated.
func TestRunAttentionNotifiesEachGenuineWait(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "waits.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
calls.Add(1)
_ = json.NewEncoder(w).Encode(map[string]string{"id": "wait", "event": "message"})
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
r.notifyRun(1, "release", protocol.RunStatusWaiting, 1, "gate", "")
r.notifyRun(1, "release", protocol.RunStatusWaiting, 1, "gate", "")
r.notifyRun(1, "release", protocol.RunStatusWaiting, 5, "gate-again", "")
if calls.Load() != 2 {
t.Fatalf("two genuine waits produced %d notifications, want 2", calls.Load())
}
}
+18 -5
View File
@@ -49,6 +49,11 @@ type runner struct {
events *runEventHub
wg sync.WaitGroup
schedulerWG sync.WaitGroup
// attentionMu guards attentionSent, the per-run set of run-attention events
// already delivered — by a stage's own notify policy or by the run-level
// default — so neither republishes the same genuine event.
attentionMu sync.Mutex
attentionSent map[int64]map[string]bool
}
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
@@ -73,7 +78,7 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
return nil, err
}
root := filepath.Join(s.Directory(), "artifacts")
return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}}, nil
return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, attentionSent: map[int64]map[string]bool{}}, nil
}
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
@@ -393,6 +398,8 @@ func (r *runner) cancel(id int64) error {
if err := r.store.CancelWaiting(r.ctx, id); err != nil {
return err
}
// A canceled run is terminal: nothing more can notify for it.
r.forgetAttention(id)
if r.events != nil {
if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail != nil {
r.emitRun(protocol.RunEventFrame{
@@ -473,7 +480,7 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
original, copyErr := copyValues(stage.Values(request.Input))
if copyErr != nil {
r.finishRunState(id, workspace, "failed", "error", copyErr.Error(), 0, "", stage.Values(request.Input))
r.finishRunState(id, d.Name, workspace, "failed", "error", copyErr.Error(), 0, 0, "", stage.Values(request.Input))
return
}
input := stage.Values(request.Input)
@@ -509,7 +516,7 @@ func terminalFinish(target, stageKey, outcome, diagnostic string) (status, final
func (r *runner) resume(ctx context.Context, id int64, d *protocol.RunDetail, workspace string, step protocol.RunStep) {
// The decision may route straight to a terminal, which has no stage to run.
if status, outcome, diagnostic, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic); terminal {
r.finishRunState(id, workspace, status, outcome, diagnostic, completedSteps(d.Steps), step.Stage, stage.Values(step.Output))
r.finishRunState(id, d.Name, workspace, status, outcome, diagnostic, completedSteps(d.Steps), step.Sequence, step.Stage, stage.Values(step.Output))
return
}
state := rebuildRunState(d, step)
@@ -614,12 +621,14 @@ func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) {
r.events.emit(frame, terminal)
}
func (r *runner) finishRunState(id int64, workspace, status, outcome, diagnostic string, completed int, currentStage string, input stage.Values) {
func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diagnostic string, completed, sequence int, currentStage string, input stage.Values) {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
log.Printf("finish run %d: %v", id, err)
}
// Push the terminal delta before the attention notification, so a slow ntfy
// never delays the live update the app folds.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
@@ -629,6 +638,9 @@ func (r *runner) finishRunState(id int64, workspace, status, outcome, diagnostic
CompletedSteps: completed,
FinishedAt: time.Now().UTC().Format(time.RFC3339),
}, true)
r.notifyRun(id, name, status, sequence, currentStage, diagnostic)
// The run is terminal: nothing more can notify for it.
r.forgetAttention(id)
}
// completedSteps counts the completed steps of a run detail; resume uses it to
@@ -671,7 +683,7 @@ func (r *runner) finishStage(id int64, name, workspace string, selected protocol
func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) {
finish := func(status, outcome, diagnostic string) {
r.finishRunState(id, workspace, status, outcome, diagnostic, state.completed, state.key, state.input)
r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input)
}
defer func() {
if recovered := recover(); recovered != nil {
@@ -800,6 +812,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting},
}, false)
r.notifyStep(id, name, workspace, selected.Notify, waitStep)
r.notifyRun(id, name, protocol.RunStatusWaiting, sequence, key, "")
return
}
// A stage that declares a `skipped` route is short-circuited when the run