Files
pipelines/internal/server/notify_test.go
T
alan 890d267c71 run attention: publish to ntfy and surface what needs the operator
Nothing told the operator that a run was waiting for a person or had failed; they
found out by going looking. A run that needs attention now reaches the ntfy they
already run, and the app answers "what needs me now" in one live, ordered surface.

Server (smallest change, reusing the existing publisher):
- internal/server/notify.go adds notifyRun, called from finishRunState (failed)
  and when a run parks on a human (runner.go runLoop). It publishes to the same
  already-configured `system` destination through the unchanged stage.PublishNtfy,
  so there is no second publisher, topic or service and the token handling is the
  stage's own. runAttentionEvent maps only waiting and failed; queued, running,
  succeeded, canceled and interrupted publish nothing.
- No storms: runner.attentionSent records sequence:event per run before publishing,
  so a run that stays waiting is never republished on a re-request or restart, and
  a stage notify policy that already covers the same event marks it first so the
  run-level default is suppressed. The set is dropped when the run is terminal. A
  delivery failure is logged, never fails the run, and is not retried.
- The payload names the run and pipeline and carries a click action
  pipelines://run/<id> (validated: absolute URL, no userinfo, no fragment, no
  credentials). PublishNtfy gained the optional click parameter; existing callers
  pass "".

App:
- AndroidManifest/MainActivity declare the pipelines://run/<id> deep link
  (singleTop); runIdFromDeepLink parses it and PipelinesApp opens that run once,
  clearing the link. It opens a run; it starts nothing.
- ui/Attention.kt is the pure model (status -> Decision/Failed/Finished or
  nothing, ordered by kind then newest-first) and ui/components/AttentionSurface.kt
  is the floating indicator plus bottom sheet. It is derived from the same run
  events the server already pushes, so an entry appears with no refresh and clears
  when resolved; a finished run clears once looked at, and any entry can be
  dismissed, persisted in SettingsRepository. The existing destinations, pill and
  bottom strip are untouched and no competing navigation surface is added.

Checked before committing:
- Packaged authoring skill references: internal/author/skill/references/
  notifications.md documents the automatic run-attention notifications and the
  app surface. The only packaged "template" carrying notify is that reference; no
  pipeline definition template exists to update and install.go's managed file list
  is unchanged (notifications.md already existed).
- Repository guidelines: docs/ntfy.md gains an "Automatic run-attention
  notifications" section and no longer claims "No block means no notifications";
  the top-level README phone-notifications note now says a waiting/failed run
  notifies automatically; android/README.md gains the dated feature section and
  supersedes the previous section's note that the attention surface was absent.
- docker.md, install docs and the CLI/TUI docs describe mechanisms this change
  does not alter; left as they are.

Tests: go test ./internal/server/... ./internal/stage/... ./internal/author/...
all ok; notify_test.go covers the event mapping and the real HTTP delivery
(destination, topic, click, no duplicate). Android :app:testDebugUnitTest ->
BUILD SUCCESSFUL, 58 suites / 531 tests / 0 failures (new AttentionTest, 13 cases,
covering ordering, appearing from a push, clearing on resolution, dismissal, the
empty case, persistence and the deep link). Device screenshots and the phone's own
notification are visual review's per the brief's device policy.

Scope: android/ plus the smallest server change; no new dependency, no run started
from the app, no installer or Docker change.
2026-10-17 16:11:57 +01:00

474 lines
18 KiB
Go

package server
import (
"context"
"encoding/json"
"errors"
"fmt"
"git.a-wan.net/alan/pipelines/examples"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"io"
"mime"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"syscall"
"testing"
)
func TestStageNotificationEventsAndArtifactAudit(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "notify.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
workspace := t.TempDir()
if err := os.WriteFile(filepath.Join(workspace, "report.txt"), []byte("created file"), 0600); err != nil {
t.Fatal(err)
}
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
count := calls.Add(1)
filename, _ := new(mime.WordDecoder).DecodeHeader(r.Header.Get("Filename"))
body, _ := io.ReadAll(r.Body)
if filename == "report.txt" && string(body) != "created file" {
t.Error("wrong file content")
}
if filename == "digest.md" && string(body) != "# Digest" {
t.Error("wrong markdown")
}
ack := map[string]any{"id": fmt.Sprintf("msg%d", count), "event": "message"}
if filename != "" {
ack["attachment"] = map[string]string{"name": filename, "url": "https://ntfy.example.com/file/" + filename}
}
_ = json.NewEncoder(w).Encode(ack)
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
if _, err := store.Submit(ctx, "notifications", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
d, _ := store.Get(ctx, items[0].ID, 0)
r, _ := newRunner(ctx, store)
for i, test := range []struct {
status, outcome, target, on string
attachments bool
expected int32
}{
{"completed", "success", protocol.SuccessEnd, "success", true, 2},
{"completed", "blocked", protocol.FailureEnd, "failure", false, 1},
{"failed", "error", protocol.FailureEnd, "failure", false, 1},
{"completed", "error", protocol.FailureEnd, "success", false, 0},
} {
id, _, err := store.CreateRun(ctx, d, fmt.Sprintf("case%d", i), workspace, false, nil, map[string]any{"prompt": "task"})
if err != nil {
t.Fatal(err)
}
if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil {
t.Fatal(err)
}
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: test.status, Outcome: test.outcome, Target: test.target, Output: map[string]any{}}
policy := &protocol.NotificationPolicy{On: []string{test.on}, Destination: "anime"}
if test.attachments {
policy.Artifacts = []string{"markdown", "files"}
step.Output = map[string]any{"markdown": "# Digest", "artifacts": []any{"report.txt"}}
}
if err := store.FinishStep(ctx, id, step); err != nil {
t.Fatal(err)
}
before := calls.Load()
r.notifyStep(id, d.Name, workspace, policy, step)
detail, err := store.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
if calls.Load()-before != test.expected || len(detail.Steps[0].Notifications) != int(test.expected) {
t.Fatal("event/attachments mismatch", detail, calls.Load()-before)
}
for _, receipt := range detail.Steps[0].Notifications {
if receipt.Status != "sent" || receipt.MessageID == "" {
t.Fatal("notification receipt not persisted", receipt)
}
}
}
}
// Regress the exact research -> report terminal-path recipe from the authoring guide.
// A completion-alert request must not require native delivery or change run routing.
func TestResearchTerminalNotificationRecipe(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "recipe.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
var source string
for _, template := range examples.Catalog() {
if template.Key == "research" {
source = template.Source
}
}
if source == "" {
t.Fatal("research template missing")
}
source = strings.Replace(source, "# stage research", "# stage research\n```notify\non=failure\n```", 1)
source = strings.Replace(source, "# stage report", "# stage report\n```notify\non=success,failure\n```", 1)
if _, err = store.Submit(ctx, "research notification recipe", source); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil || len(items) != 1 {
t.Fatal("recipe missing", err)
}
id := items[0].ID
var calls atomic.Int32
var deliveryFailure atomic.Bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
calls.Add(1)
if req.Header.Get("Filename") != "" {
t.Error("event alert unexpectedly changed output to add attachments")
}
if deliveryFailure.Load() {
http.Error(w, "unavailable", http.StatusServiceUnavailable)
return
}
_ = json.NewEncoder(w).Encode(map[string]string{"id": "recipe-receipt", "event": "message"})
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
for index, test := range []struct {
name, stage, outcome, status string
executionError, deliveryError bool
}{
{"success", "report", "success", "succeeded", false, false},
{"research blocked", "research", "blocked", "failed", false, false},
{"research execution error", "research", "error", "failed", true, false},
{"report blocked", "report", "blocked", "failed", false, false},
{"report execution error", "report", "error", "failed", true, false},
{"delivery error preserves success", "report", "success", "succeeded", false, true},
} {
t.Run(test.name, func(t *testing.T) {
deliveryFailure.Store(test.deliveryError)
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Key == test.stage {
if test.executionError {
return stage.Result{}, errors.New("fixture execution failed")
}
return stage.Result{Outcome: test.outcome, Output: stage.Values{"report": "retained evidence"}}, nil
}
return stage.Result{Outcome: "success", Output: stage.Values{"question": "topic", "evidence": []any{"source fact"}}}, nil
}
before := calls.Load()
run, err := r.start(protocol.Message{PipelineID: id, Version: 1, RequestKey: fmt.Sprintf("recipe-%d", index), Workspace: t.TempDir(), Input: map[string]any{"prompt": "research a topic"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != test.status || calls.Load()-before != 1 {
t.Fatal("wrong verdict or duplicate/missing terminal alert", detail.Status, calls.Load()-before)
}
last := detail.Steps[len(detail.Steps)-1]
if last.Stage != test.stage || len(last.Notifications) != 1 {
t.Fatal("notification not retained on terminal stage", last)
}
for _, step := range detail.Steps[:len(detail.Steps)-1] {
if len(step.Notifications) != 0 {
t.Fatal("intermediate success alerted", step)
}
}
want := "sent"
if test.deliveryError {
want = "failed"
}
if last.Notifications[0].Status != want {
t.Fatal("delivery receipt wrong", last.Notifications)
}
if !test.executionError && detail.Output["report"] != "retained evidence" {
t.Fatal("notification changed output handoff", detail.Output)
}
})
}
}
func TestNotificationFailureDoesNotChangeRunRouting(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "routing.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := strings.Replace(pipeline.Template([]string{"one"}, true), "# stage one", "# stage one\n```notify\non=success\ndestination=anime\nartifacts=markdown\n```", 1)
if _, err := store.Submit(ctx, "test", source); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, _ := newRunner(ctx, store)
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "# Result"}}, nil
}
t.Setenv("PIPELINES_NTFY_URL", "")
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "once", Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, _ := store.GetRun(ctx, run.ID)
if detail.Status != "succeeded" || len(detail.Steps[0].Notifications) != 1 || detail.Steps[0].Notifications[0].Status != "failed" {
t.Fatal("notification altered routing or lost failure audit", detail)
}
}
func TestNotificationArtifactPathSafety(t *testing.T) {
root := t.TempDir()
os.WriteFile(filepath.Join(root, "safe.txt"), []byte("safe"), 0600)
os.WriteFile(filepath.Join(root, ".env"), []byte("secret"), 0600)
os.Symlink(filepath.Join(root, ".env"), filepath.Join(root, "alias.txt"))
outside := filepath.Join(t.TempDir(), "outside.txt")
os.WriteFile(outside, []byte("secret"), 0600)
os.Symlink(outside, filepath.Join(root, "external.txt"))
os.Mkdir(filepath.Join(root, "directory"), 0700)
os.WriteFile(filepath.Join(root, "large.bin"), []byte(strings.Repeat("x", (1<<20)+1)), 0600)
if err := syscall.Mkfifo(filepath.Join(root, "pipe"), 0600); err != nil {
t.Fatal(err)
}
for _, name := range []string{"../outside.txt", outside, ".env", "alias.txt", "external.txt", "directory", "large.bin", "pipe", "pipelines.db", "secret.key", "secrets/file.txt", "sub/../safe.txt"} {
if _, err := notificationArtifact(root, name); err == nil {
t.Fatal("unsafe artifact accepted", name)
}
}
if data, err := notificationArtifact(root, "safe.txt"); err != nil || string(data) != "safe" {
t.Fatal("safe artifact rejected", err)
}
}
// TestRunAttentionEventMapping pins the single decision of whether a run needs
// the operator: a run parked on a person (waiting) or a failed run, and nothing
// else. A queued, running, succeeded, canceled or interrupted run never notifies.
func TestRunAttentionEventMapping(t *testing.T) {
for _, test := range []struct{ status, want string }{
{"waiting", protocol.NotifyWaiting},
{"failed", "failed"},
{"Warning", ""},
{"queued", ""},
{"running", ""},
{"succeeded", ""},
{"canceled", ""},
{"interrupted", ""},
{"", ""},
} {
if got := runAttentionEvent(test.status); got != test.want {
t.Errorf("runAttentionEvent(%q) = %q, want %q", test.status, got, test.want)
}
}
}
// TestRunAttentionNotificationDelivery proves the operator-facing contract: the
// already-configured "system" destination and topic are used, the payload names
// the run and its pipeline, the click action opens the run, only waiting/failed
// publish, and a run that stays waiting is never republished.
func TestRunAttentionNotificationDelivery(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "attention.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
type delivered struct {
method, path, title, message, click, authorization string
}
deliveries := make(chan delivered, 8)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
decoder := new(mime.WordDecoder)
title, _ := decoder.DecodeHeader(r.Header.Get("Title"))
message, _ := decoder.DecodeHeader(r.Header.Get("Message"))
deliveries <- delivered{r.Method, r.URL.Path, title, message, r.Header.Get("Click"), r.Header.Get("Authorization")}
_ = json.NewEncoder(w).Encode(map[string]string{"id": "attention-receipt", "event": "message"})
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
// A run that needs nobody publishes nothing.
for _, status := range []string{"queued", "running", "succeeded", "canceled", "interrupted"} {
r.notifyRun(7, "release", status, 1, "build", "")
}
select {
case got := <-deliveries:
t.Fatalf("a run that needs nobody published: %+v", got)
default:
}
// A waiting run publishes once, with the destination, identity and click.
r.notifyRun(7, "release", protocol.RunStatusWaiting, 1, "approve", "")
// A run that stays waiting is never republished, on a later call or a re-request.
r.notifyRun(7, "release", protocol.RunStatusWaiting, 1, "approve", "")
// A failed run publishes once, with its diagnostic.
r.notifyRun(7, "release", "failed", 2, "deploy", "provider returned 500")
first := <-deliveries
if first.method != http.MethodPost || first.path != "/pipelines" {
t.Fatalf("wrong destination/topic: %+v", first)
}
if first.title != "Run needs your decision" || !strings.Contains(first.message, "Run #7") || !strings.Contains(first.message, "release") || !strings.Contains(first.message, "decision") || !strings.Contains(first.message, "approve") {
t.Fatalf("waiting notification does not identify the run/pipeline/need: %+v", first)
}
if first.click != "pipelines://run/7" {
t.Fatalf("waiting notification click does not open the run: %q", first.click)
}
if first.authorization != "" {
t.Fatalf("notification leaked an authorization header: %q", first.authorization)
}
second := <-deliveries
if second.title != "Run failed" || !strings.Contains(second.message, "Run #7") || !strings.Contains(second.message, "release") || !strings.Contains(second.message, "deploy") || !strings.Contains(second.message, "provider returned 500") {
t.Fatalf("failed notification is not actionable: %+v", second)
}
if second.click != "pipelines://run/7" {
t.Fatalf("failed notification click does not open the run: %q", second.click)
}
select {
case got := <-deliveries:
t.Fatalf("third notification delivered for two genuine events: %+v", got)
default:
}
}
// TestRunAttentionSuppressedByStagePolicy proves a stage's own notify policy and
// the run-level default cannot both fire for the same event: the author's policy
// wins, so the operator gets exactly one notification.
func TestRunAttentionSuppressedByStagePolicy(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "suppress.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.markAttentionSent(11, 5, "failed")
r.markAttentionSent(11, 6, "waiting")
if !r.attentionAlready(11, 5, "failed") || !r.attentionAlready(11, 6, "waiting") {
t.Fatal("stage-policy coverage not recorded")
}
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
calls.Add(1)
_ = json.NewEncoder(w).Encode(map[string]string{"id": "suppressed", "event": "message"})
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
r.notifyRun(11, "release", "failed", 5, "deploy", "")
r.notifyRun(11, "release", protocol.RunStatusWaiting, 6, "approve", "")
if calls.Load() != 0 {
t.Fatalf("run-level default repeated a stage-policy event %d times", calls.Load())
}
}
// TestHumanRunAttentionNotification proves the run loop publishes the run-level
// attention notification itself when the pipeline declares no notify policy: the
// transition to waiting reaches the operator's configured destination exactly
// once, with the run, pipeline and a run-opening click.
func TestHumanRunAttentionNotification(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "human-attention.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
type delivered struct{ title, message, click string }
deliveries := make(chan delivered, 4)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
decoder := new(mime.WordDecoder)
title, _ := decoder.DecodeHeader(r.Header.Get("Title"))
message, _ := decoder.DecodeHeader(r.Header.Get("Message"))
deliveries <- delivered{title, message, r.Header.Get("Click")}
_ = json.NewEncoder(w).Encode(map[string]string{"id": "human-attention", "event": "message"})
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
if _, err := store.Submit(ctx, "human", humanPipelineSource); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r.wg.Wait()
r.execute = successExecute
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "human-attention", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
got := <-deliveries
if got.title != "Run needs your decision" || !strings.Contains(got.message, fmt.Sprintf("Run #%d", run.ID)) || !strings.Contains(got.message, "human") {
t.Fatalf("run-level waiting notification wrong: %+v", got)
}
if got.click != fmt.Sprintf("pipelines://run/%d", run.ID) {
t.Fatalf("run-level waiting notification click wrong: %q", got.click)
}
select {
case extra := <-deliveries:
t.Fatalf("waiting run notified again: %+v", extra)
default:
}
}
// TestRunAttentionNotifiesEachGenuineWait proves the step sequence is part of the
// dedup key: a run that waits on a person twice notifies each time (two genuine
// events), while the same wait is never repeated.
func TestRunAttentionNotifiesEachGenuineWait(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "waits.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
calls.Add(1)
_ = json.NewEncoder(w).Encode(map[string]string{"id": "wait", "event": "message"})
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
r.notifyRun(1, "release", protocol.RunStatusWaiting, 1, "gate", "")
r.notifyRun(1, "release", protocol.RunStatusWaiting, 1, "gate", "")
r.notifyRun(1, "release", protocol.RunStatusWaiting, 5, "gate-again", "")
if calls.Load() != 2 {
t.Fatalf("two genuine waits produced %d notifications, want 2", calls.Load())
}
}