Files
pipelines/internal/server/notify_test.go
T
alan a48e4a5b6a Replace starters with four graded examples seeded into new databases
The template catalog is now Example 1-4: one stage, two stages (handoff),
feedback loop (runner-owned limits, retries, timeout) and human decision
(type=human approve/revise/abort with notify hooks). All read-only or leave
changes uncommitted; none commits, pushes or deploys.

A brand-new database (schema created by this Open) gets the examples once at
server start, bound to the configured default model and harness. A
server_settings marker records the decision, so removed examples never come
back and existing installs are marked without any pipeline being added or
rewritten.
2026-11-03 19:35:47 +00:00

483 lines
19 KiB
Go

package server
import (
"context"
"encoding/json"
"errors"
"fmt"
"git.a-wan.net/alan/pipelines/examples"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"io"
"mime"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"syscall"
"testing"
)
func TestStageNotificationEventsAndArtifactAudit(t *testing.T) {
t.Setenv("PIPELINES_WEB_PUBLIC_URL", "https://pipelines.example.com/")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "notify.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
workspace := t.TempDir()
if err := os.WriteFile(filepath.Join(workspace, "report.txt"), []byte("created file"), 0600); err != nil {
t.Fatal(err)
}
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
count := calls.Add(1)
if click, actions := r.Header.Get("Click"), r.Header.Get("Actions"); !strings.HasPrefix(click, "pipelines://run/") || actions != "view, Open in browser, https://pipelines.example.com/runs/"+strings.TrimPrefix(click, "pipelines://run/") {
t.Errorf("stage notification lacks app click/web action: %q %q", click, actions)
}
filename, _ := new(mime.WordDecoder).DecodeHeader(r.Header.Get("Filename"))
body, _ := io.ReadAll(r.Body)
if filename == "report.txt" && string(body) != "created file" {
t.Error("wrong file content")
}
if filename == "digest.md" && string(body) != "# Digest" {
t.Error("wrong markdown")
}
ack := map[string]any{"id": fmt.Sprintf("msg%d", count), "event": "message"}
if filename != "" {
ack["attachment"] = map[string]string{"name": filename, "url": "https://ntfy.example.com/file/" + filename}
}
_ = json.NewEncoder(w).Encode(ack)
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
if _, err := store.Submit(ctx, "notifications", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
d, _ := store.Get(ctx, items[0].ID, 0)
r, _ := newRunner(ctx, store)
for i, test := range []struct {
status, outcome, target, on string
attachments bool
expected int32
}{
{"completed", "success", protocol.SuccessEnd, "success", true, 2},
{"completed", "blocked", protocol.FailureEnd, "failure", false, 1},
{"failed", "error", protocol.FailureEnd, "failure", false, 1},
{"completed", "error", protocol.FailureEnd, "success", false, 0},
} {
id, _, err := store.CreateRun(ctx, d, fmt.Sprintf("case%d", i), workspace, false, nil, map[string]any{"prompt": "task"})
if err != nil {
t.Fatal(err)
}
if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil {
t.Fatal(err)
}
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: test.status, Outcome: test.outcome, Target: test.target, Output: map[string]any{}}
policy := &protocol.NotificationPolicy{On: []string{test.on}, Destination: "anime"}
if test.attachments {
policy.Artifacts = []string{"markdown", "files"}
step.Output = map[string]any{"markdown": "# Digest", "artifacts": []any{"report.txt"}}
}
if err := store.FinishStep(ctx, id, step); err != nil {
t.Fatal(err)
}
before := calls.Load()
r.notifyStep(id, d.Name, workspace, policy, step)
detail, err := store.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
if calls.Load()-before != test.expected || len(detail.Steps[0].Notifications) != int(test.expected) {
t.Fatal("event/attachments mismatch", detail, calls.Load()-before)
}
for _, receipt := range detail.Steps[0].Notifications {
if receipt.Status != "sent" || receipt.MessageID == "" {
t.Fatal("notification receipt not persisted", receipt)
}
}
}
}
// Regress the terminal-path recipe from the authoring guide (research -> report
// there) on the bundled two-stage example's draft -> check.
// A completion-alert request must not require native delivery or change run routing.
func TestTerminalNotificationRecipe(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "recipe.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
var source string
for _, template := range examples.Catalog() {
if template.Key == "two-stages" {
source = template.Source
}
}
if source == "" {
t.Fatal("two-stages template missing")
}
source = strings.Replace(source, "# stage draft", "# stage draft\n```notify\non=failure\n```", 1)
source = strings.Replace(source, "# stage check", "# stage check\n```notify\non=success,failure\n```", 1)
if _, err = store.Submit(ctx, "research notification recipe", source); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil || len(items) != 1 {
t.Fatal("recipe missing", err)
}
id := items[0].ID
var calls atomic.Int32
var deliveryFailure atomic.Bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
calls.Add(1)
if req.Header.Get("Filename") != "" {
t.Error("event alert unexpectedly changed output to add attachments")
}
if deliveryFailure.Load() {
http.Error(w, "unavailable", http.StatusServiceUnavailable)
return
}
_ = json.NewEncoder(w).Encode(map[string]string{"id": "recipe-receipt", "event": "message"})
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
for index, test := range []struct {
name, stage, outcome, status string
executionError, deliveryError bool
}{
{"success", "check", "success", "succeeded", false, false},
{"draft blocked", "draft", "blocked", "failed", false, false},
{"draft execution error", "draft", "error", "failed", true, false},
{"check blocked", "check", "blocked", "failed", false, false},
{"check execution error", "check", "error", "failed", true, false},
{"delivery error preserves success", "check", "success", "succeeded", false, true},
} {
t.Run(test.name, func(t *testing.T) {
deliveryFailure.Store(test.deliveryError)
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Key == test.stage {
if test.executionError {
return stage.Result{}, errors.New("fixture execution failed")
}
return stage.Result{Outcome: test.outcome, Output: stage.Values{"report": "retained evidence"}}, nil
}
return stage.Result{Outcome: "success", Output: stage.Values{"question": "topic", "evidence": []any{"source fact"}}}, nil
}
before := calls.Load()
run, err := r.start(protocol.Message{PipelineID: id, Version: 1, RequestKey: fmt.Sprintf("recipe-%d", index), Workspace: t.TempDir(), Input: map[string]any{"prompt": "research a topic"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != test.status || calls.Load()-before != 1 {
t.Fatal("wrong verdict or duplicate/missing terminal alert", detail.Status, calls.Load()-before)
}
last := detail.Steps[len(detail.Steps)-1]
if last.Stage != test.stage || len(last.Notifications) != 1 {
t.Fatal("notification not retained on terminal stage", last)
}
for _, step := range detail.Steps[:len(detail.Steps)-1] {
if len(step.Notifications) != 0 {
t.Fatal("intermediate success alerted", step)
}
}
want := "sent"
if test.deliveryError {
want = "failed"
}
if last.Notifications[0].Status != want {
t.Fatal("delivery receipt wrong", last.Notifications)
}
if !test.executionError && detail.Output["report"] != "retained evidence" {
t.Fatal("notification changed output handoff", detail.Output)
}
})
}
}
func TestNotificationFailureDoesNotChangeRunRouting(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "routing.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := strings.Replace(pipeline.Template([]string{"one"}, true), "# stage one", "# stage one\n```notify\non=success\ndestination=anime\nartifacts=markdown\n```", 1)
if _, err := store.Submit(ctx, "test", source); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, _ := newRunner(ctx, store)
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "# Result"}}, nil
}
t.Setenv("PIPELINES_NTFY_URL", "")
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "once", Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, _ := store.GetRun(ctx, run.ID)
if detail.Status != "succeeded" || len(detail.Steps[0].Notifications) != 1 || detail.Steps[0].Notifications[0].Status != "failed" {
t.Fatal("notification altered routing or lost failure audit", detail)
}
}
func TestNotificationArtifactPathSafety(t *testing.T) {
root := t.TempDir()
os.WriteFile(filepath.Join(root, "safe.txt"), []byte("safe"), 0600)
os.WriteFile(filepath.Join(root, ".env"), []byte("secret"), 0600)
os.Symlink(filepath.Join(root, ".env"), filepath.Join(root, "alias.txt"))
outside := filepath.Join(t.TempDir(), "outside.txt")
os.WriteFile(outside, []byte("secret"), 0600)
os.Symlink(outside, filepath.Join(root, "external.txt"))
os.Mkdir(filepath.Join(root, "directory"), 0700)
os.WriteFile(filepath.Join(root, "large.bin"), []byte(strings.Repeat("x", (1<<20)+1)), 0600)
if err := syscall.Mkfifo(filepath.Join(root, "pipe"), 0600); err != nil {
t.Fatal(err)
}
for _, name := range []string{"../outside.txt", outside, ".env", "alias.txt", "external.txt", "directory", "large.bin", "pipe", "pipelines.db", "secret.key", "secrets/file.txt", "sub/../safe.txt"} {
if _, err := notificationArtifact(root, name); err == nil {
t.Fatal("unsafe artifact accepted", name)
}
}
if data, err := notificationArtifact(root, "safe.txt"); err != nil || string(data) != "safe" {
t.Fatal("safe artifact rejected", err)
}
}
// TestRunAttentionEventMapping pins the single decision of whether a run needs
// the operator: a run parked on a person (waiting) or a failed run, and nothing
// else. A queued, running, succeeded, canceled or interrupted run never notifies.
func TestRunAttentionEventMapping(t *testing.T) {
for _, test := range []struct{ status, want string }{
{"waiting", protocol.NotifyWaiting},
{"failed", "failed"},
{"Warning", ""},
{"queued", ""},
{"running", ""},
{"succeeded", ""},
{"canceled", ""},
{"interrupted", ""},
{"", ""},
} {
if got := runAttentionEvent(test.status); got != test.want {
t.Errorf("runAttentionEvent(%q) = %q, want %q", test.status, got, test.want)
}
}
}
// TestRunAttentionNotificationDelivery proves the operator-facing contract: the
// already-configured "system" destination and topic are used, the payload names
// the run and its pipeline, the click action opens the run, only waiting/failed
// publish, and a run that stays waiting is never republished.
func TestRunAttentionNotificationDelivery(t *testing.T) {
t.Setenv("PIPELINES_WEB_PUBLIC_URL", "https://pipelines.example.com")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "attention.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
type delivered struct {
method, path, title, message, click, authorization, actions string
}
deliveries := make(chan delivered, 8)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
decoder := new(mime.WordDecoder)
title, _ := decoder.DecodeHeader(r.Header.Get("Title"))
message, _ := decoder.DecodeHeader(r.Header.Get("Message"))
deliveries <- delivered{r.Method, r.URL.Path, title, message, r.Header.Get("Click"), r.Header.Get("Authorization"), r.Header.Get("Actions")}
_ = json.NewEncoder(w).Encode(map[string]string{"id": "attention-receipt", "event": "message"})
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
// A run that needs nobody publishes nothing.
for _, status := range []string{"queued", "running", "succeeded", "canceled", "interrupted"} {
r.notifyRun(7, "release", status, 1, "build", "")
}
select {
case got := <-deliveries:
t.Fatalf("a run that needs nobody published: %+v", got)
default:
}
// A waiting run publishes once, with the destination, identity and click.
r.notifyRun(7, "release", protocol.RunStatusWaiting, 1, "approve", "")
// A run that stays waiting is never republished, on a later call or a re-request.
r.notifyRun(7, "release", protocol.RunStatusWaiting, 1, "approve", "")
// An auto run's failure is silent: only runs marked "on" announce their end.
r.notifyRun(8, "release", "failed", 2, "deploy", "provider returned 500")
// A failed run marked "on" publishes once, with its diagnostic.
r.setNotifyMode(7, "on")
r.notifyRun(7, "release", "failed", 2, "deploy", "provider returned 500")
first := <-deliveries
if first.method != http.MethodPost || first.path != "/pipelines" {
t.Fatalf("wrong destination/topic: %+v", first)
}
if first.title != "Run needs your decision" || !strings.Contains(first.message, "Run #7") || !strings.Contains(first.message, "release") || !strings.Contains(first.message, "decision") || !strings.Contains(first.message, "approve") {
t.Fatalf("waiting notification does not identify the run/pipeline/need: %+v", first)
}
if first.click != "pipelines://run/7" || first.actions != "view, Open in browser, https://pipelines.example.com/runs/7" {
t.Fatalf("waiting notification click does not open the run: %q", first.click)
}
if first.authorization != "" {
t.Fatalf("notification leaked an authorization header: %q", first.authorization)
}
second := <-deliveries
if second.title != "Run failed" || !strings.Contains(second.message, "Run #7") || !strings.Contains(second.message, "release") || !strings.Contains(second.message, "deploy") || !strings.Contains(second.message, "provider returned 500") {
t.Fatalf("failed notification is not actionable: %+v", second)
}
if second.click != "pipelines://run/7" || second.actions != "view, Open in browser, https://pipelines.example.com/runs/7" {
t.Fatalf("failed notification click does not open the run: %q", second.click)
}
select {
case got := <-deliveries:
t.Fatalf("third notification delivered for two genuine events: %+v", got)
default:
}
}
// TestRunAttentionSuppressedByStagePolicy proves a stage's own notify policy and
// the run-level default cannot both fire for the same event: the author's policy
// wins, so the operator gets exactly one notification.
func TestRunAttentionSuppressedByStagePolicy(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "suppress.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.markAttentionSent(11, 5, "failed")
r.markAttentionSent(11, 6, "waiting")
if !r.attentionAlready(11, 5, "failed") || !r.attentionAlready(11, 6, "waiting") {
t.Fatal("stage-policy coverage not recorded")
}
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
calls.Add(1)
_ = json.NewEncoder(w).Encode(map[string]string{"id": "suppressed", "event": "message"})
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
r.notifyRun(11, "release", "failed", 5, "deploy", "")
r.notifyRun(11, "release", protocol.RunStatusWaiting, 6, "approve", "")
if calls.Load() != 0 {
t.Fatalf("run-level default repeated a stage-policy event %d times", calls.Load())
}
}
// TestHumanRunAttentionNotification proves the run loop publishes the run-level
// attention notification itself when the pipeline declares no notify policy: the
// transition to waiting reaches the operator's configured destination exactly
// once, with the run, pipeline and a run-opening click.
func TestHumanRunAttentionNotification(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "human-attention.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
type delivered struct{ title, message, click string }
deliveries := make(chan delivered, 4)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
decoder := new(mime.WordDecoder)
title, _ := decoder.DecodeHeader(r.Header.Get("Title"))
message, _ := decoder.DecodeHeader(r.Header.Get("Message"))
deliveries <- delivered{title, message, r.Header.Get("Click")}
_ = json.NewEncoder(w).Encode(map[string]string{"id": "human-attention", "event": "message"})
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
if _, err := store.Submit(ctx, "human", humanPipelineSource); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer r.wg.Wait()
r.execute = successExecute
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "human-attention", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
got := <-deliveries
if got.title != "Run needs your decision" || !strings.Contains(got.message, fmt.Sprintf("Run #%d", run.ID)) || !strings.Contains(got.message, "human") {
t.Fatalf("run-level waiting notification wrong: %+v", got)
}
if got.click != fmt.Sprintf("pipelines://run/%d", run.ID) {
t.Fatalf("run-level waiting notification click wrong: %q", got.click)
}
select {
case extra := <-deliveries:
t.Fatalf("waiting run notified again: %+v", extra)
default:
}
}
// TestRunAttentionNotifiesEachGenuineWait proves the step sequence is part of the
// dedup key: a run that waits on a person twice notifies each time (two genuine
// events), while the same wait is never repeated.
func TestRunAttentionNotifiesEachGenuineWait(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "waits.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
calls.Add(1)
_ = json.NewEncoder(w).Encode(map[string]string{"id": "wait", "event": "message"})
}))
defer srv.Close()
t.Setenv("PIPELINES_NTFY_URL", srv.URL+"/pipelines")
r.notifyRun(1, "release", protocol.RunStatusWaiting, 1, "gate", "")
r.notifyRun(1, "release", protocol.RunStatusWaiting, 1, "gate", "")
r.notifyRun(1, "release", protocol.RunStatusWaiting, 5, "gate-again", "")
if calls.Load() != 2 {
t.Fatalf("two genuine waits produced %d notifications, want 2", calls.Load())
}
}