From d1846441cc02daede572c1571d60875de7fe5c18 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 1 Nov 2026 02:03:58 +0000 Subject: [PATCH] Stream ntfy notifications to app clients over /connect Record every notification published to ntfy (run outbox, stage notify hooks, chat question/approval/reply) in a persisted app_notifications outbox and stream it to subscribe_notifications clients with an after cursor; clear_notifications and ack_attention clear across devices, as do runs leaving waiting, deleted runs and answered chat questions. Add the notify.ntfy_enabled setting (default on) to stop only the ntfy publish. --- docs/cli.md | 7 +- docs/ntfy.md | 54 ++- internal/pipeline/app_notifications.go | 238 ++++++++++++ .../pipeline/notification_settings_test.go | 4 +- internal/pipeline/store.go | 4 + internal/protocol/message.go | 9 + internal/protocol/notification_settings.go | 12 +- internal/protocol/notifications.go | 107 +++++ internal/server/app_notifications.go | 323 ++++++++++++++++ internal/server/app_notifications_test.go | 365 ++++++++++++++++++ internal/server/attention_acks.go | 10 +- internal/server/chat_attention.go | 61 ++- internal/server/chat_hub.go | 4 +- internal/server/chat_permissions.go | 1 + internal/server/chat_reply_attention.go | 9 +- internal/server/notification_settings_test.go | 2 +- internal/server/notify.go | 44 ++- internal/server/run_delete.go | 7 + internal/server/runner.go | 10 + internal/server/websocket.go | 14 +- 20 files changed, 1265 insertions(+), 20 deletions(-) create mode 100644 internal/pipeline/app_notifications.go create mode 100644 internal/protocol/notifications.go create mode 100644 internal/server/app_notifications.go create mode 100644 internal/server/app_notifications_test.go diff --git a/docs/cli.md b/docs/cli.md index 9ede8d3e..4242d51d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -319,7 +319,12 @@ and matched field. `list_runs` also returns all-history `usage` aggregates. `attention_acks` reads and `ack_attention` adds to the operator's shared in-app acknowledgements (dismissed Needs-you entries and chat read watermarks; see [ntfy.md](ntfy.md#clearing-notifications-across-devices)); other -connections receive an `attention_acks_changed` push. +connections receive an `attention_acks_changed` push. `subscribe_notifications` +(optional `after` cursor) streams the same notifications the server publishes to +ntfy as `notification` pushes, `clear_notifications` (`ids`) clears them on every +device, and `notifications_cleared` reports clears; `ntfy_enabled` in the +notification settings turns the ntfy publish off without stopping the stream (see +[ntfy.md](ntfy.md#app-notification-stream-no-ntfy-app-needed)). Every run summary (`list_runs` rows, `get_run`, subscribe snapshots and terminal `run_event` frames) carries `size_bytes`: bytes on disk for the run (artifact tree diff --git a/docs/ntfy.md b/docs/ntfy.md index 689cc7bf..61a4ce31 100644 --- a/docs/ntfy.md +++ b/docs/ntfy.md @@ -169,7 +169,7 @@ reading history never publishes. ## Notification settings -Three global, server-side switches choose which of these events are published. +Global, server-side switches choose which of these events are published. They are stored in the server database (`server_settings`), survive restarts and apply to every device subscribed to the topic. Per-stage `notify` hooks and explicit `type=ntfy` stages are not affected. @@ -179,13 +179,14 @@ explicit `type=ntfy` stages are not affected. | `run_finished` — Notify when any run ends | off | every **auto**-mode run that fails or succeeds notifies; otherwise only runs started with **Notify me** (`on`) announce their end | | `chat_needs_you` — Chat needs your answer or approval | on | chat question/form and approval requests notify | | `chat_reply_finished` — Chat reply finished | off | each finished assistant turn in a chat notifies | +| `ntfy_enabled` — Publish to ntfy | on | when off, nothing is published to ntfy; the app notification stream (below) still delivers every notification | Change them in the app under **Settings → Phone notifications** (checkboxes; a refused change reverts and shows the server's reason). Over the websocket protocol, `{"type":"notification_settings"}` reads them and `{"type":"set_notification_settings","notification_settings":{"chat_reply_finished":true}}` -changes any subset; both reply with type `notification_settings` carrying all -three keys. Unknown keys and non-boolean values are refused with an `error` +changes any subset; both reply with type `notification_settings` carrying every +key. Unknown keys and non-boolean values are refused with an `error` reply and nothing is written. `run_finished` changed meaning: it once meant "auto runs announce success" and @@ -209,6 +210,53 @@ This does **not** reach notifications the ntfy app has already shown on a phone: once ntfy delivered a message, the server cannot retract it, so those must still be swiped away in the ntfy app (or the Android notification shade). +## App notification stream (no ntfy app needed) + +Every notification the server publishes to ntfy — run waiting/failed/succeeded +alerts from the run outbox, stage `notify` hook events, and chat +question/approval/reply alerts — is first recorded in the `app_notifications` +table (kept 7 days) and streamed over `/connect` to clients that subscribe, so +the Pipelines app can post them itself. The same settings decide what is +recorded, so the stream carries exactly the set ntfy gets, with the same title, +message and `click` deep link. A stage hook with several attachments is one +stream notification per step event; explicit `type=ntfy` stages are not +streamed. + +```text +→ {"type":"subscribe_notifications","id":"1","after":"run:7:3:waiting"} +← {"type":"notifications_subscribed","id":"1","notifications":[N...],"ids":["",...]} +← {"type":"notification","notification":N} (push) +← {"type":"notifications_cleared","ids":["run:7:3:waiting"]} (push) +→ {"type":"clear_notifications","id":"2","ids":["run:7:3:waiting"]} +← {"type":"notifications_cleared","id":"2","ids":[]} + +N = {"id":"run:7:3:waiting","kind":"run_waiting","title":"Run needs your decision", + "message":"Run #7 · release · waiting for a human decision at stage approve", + "click":"pipelines://run/7","priority":3,"tags":[], + "created_at":"2026-11-01T10:00:00Z","group":"run:7"} +``` + +- `id` is stable: `run:::` (event `waiting`, `failed`, + `succeeded`, or `success` for a mid-run stage hook) or + `chat:::`. `kind` is one of `run_waiting`, + `run_failed`, `run_succeeded`, `stage_failed`, `stage_succeeded`, + `chat_question`, `chat_approval`, `chat_reply`. `group` is `run:` or + `chat:`. `priority` is 3 and `tags` empty, matching what ntfy receives. +- The subscribe reply holds the active (uncleared) notifications newer than the + `after` cursor (the last id the client holds), oldest first, at most 100; + an empty, unknown or pruned cursor replays them all. Its `ids` lists + notifications cleared within the 7 days (newest first, at most 500) so a + reconnecting device can dismiss ones it still shows. Pushes follow in order; + subscribing again replaces the subscription. +- `clear_notifications` (the app's dismiss or tap) has the same effect as the + matching `ack_attention`: `run_waiting` dismisses `:Decision`, + `run_failed`/`stage_failed` `:Failed`, and a chat notification marks the + chat seen (clearing its other notifications too). Other subscribers receive + `notifications_cleared`; connections receive `attention_acks_changed`. +- Notifications are also cleared, with a push, by `ack_attention` from any + client, by a run leaving `waiting` (its waiting notifications), by deleting a + run, and by an answered or cancelled chat question or approval. + ## Explicit delivery stage (optional) The preceding stage must output three nonempty UTF-8 strings: `title`, `summary`, diff --git a/internal/pipeline/app_notifications.go b/internal/pipeline/app_notifications.go new file mode 100644 index 00000000..2cc50158 --- /dev/null +++ b/internal/pipeline/app_notifications.go @@ -0,0 +1,238 @@ +package pipeline + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "strings" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// AppNotificationRetention bounds how long an app notification is kept and +// replayed to a reconnecting client. +const AppNotificationRetention = 7 * 24 * time.Hour + +// maxReplayedClearedIDs bounds the cleared ids a subscribe reply carries. +const maxReplayedClearedIDs = 500 + +// AppNotificationRecord is one app notification plus the run or chat it +// belongs to, which the clearing paths match on. +type AppNotificationRecord struct { + protocol.AppNotification + RunID int64 + SessionID string +} + +// AppNotificationFilter selects active notifications to clear: a run or a +// chat session, optionally narrowed to kinds or one exact id. An empty filter +// matches nothing. +type AppNotificationFilter struct { + RunID int64 + SessionID string + Kinds []string + ID string +} + +// initAppNotifications creates the app notification outbox. seq is the +// replay cursor order; id is the stable, client-visible identity. +func (s *Store) initAppNotifications() error { + _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS app_notifications ( +seq INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT NOT NULL UNIQUE, kind TEXT NOT NULL, +title TEXT NOT NULL, message TEXT NOT NULL, click TEXT NOT NULL, priority INTEGER NOT NULL, +tags TEXT NOT NULL, grp TEXT NOT NULL, run_id INTEGER NOT NULL DEFAULT 0, +session_id TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, cleared_at INTEGER NOT NULL DEFAULT 0); +CREATE INDEX IF NOT EXISTS app_notifications_run ON app_notifications(run_id, cleared_at); +CREATE INDEX IF NOT EXISTS app_notifications_session ON app_notifications(session_id, cleared_at); +CREATE INDEX IF NOT EXISTS app_notifications_created ON app_notifications(created_at);`) + return err +} + +// RecordAppNotification stores n once (by id) and reports whether it is new +// and active, returning it with its created_at. A repeated id (a retried +// publication) is ignored. A run_waiting notification for a run that is no +// longer waiting is stored already cleared, so a delayed publication cannot +// resurrect a stale decision. Rows older than AppNotificationRetention are +// pruned. +func (s *Store) RecordAppNotification(ctx context.Context, n AppNotificationRecord, now time.Time) (protocol.AppNotification, bool, error) { + if n.Tags == nil { + n.Tags = []string{} + } + if n.Priority == 0 { + n.Priority = protocol.DefaultNotificationPriority + } + n.CreatedAt = now.UTC().Format(time.RFC3339) + tags, err := json.Marshal(n.Tags) + if err != nil { + return n.AppNotification, false, err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return n.AppNotification, false, err + } + defer tx.Rollback() + nowMillis := now.UnixMilli() + if _, err := tx.ExecContext(ctx, `DELETE FROM app_notifications WHERE created_at < ?`, now.Add(-AppNotificationRetention).UnixMilli()); err != nil { + return n.AppNotification, false, err + } + var cleared int64 + if n.Kind == protocol.NotificationRunWaiting && n.RunID > 0 { + var status string + switch err := tx.QueryRowContext(ctx, `SELECT status FROM pipeline_runs WHERE id=?`, n.RunID).Scan(&status); { + case errors.Is(err, sql.ErrNoRows): + case err != nil: + return n.AppNotification, false, err + case status != protocol.RunStatusWaiting: + cleared = nowMillis + } + } + result, err := tx.ExecContext(ctx, `INSERT INTO app_notifications(id,kind,title,message,click,priority,tags,grp,run_id,session_id,created_at,cleared_at) +VALUES (?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO NOTHING`, n.ID, n.Kind, n.Title, n.Message, n.Click, n.Priority, string(tags), n.Group, n.RunID, n.SessionID, nowMillis, cleared) + if err != nil { + return n.AppNotification, false, err + } + inserted, _ := result.RowsAffected() + if err := tx.Commit(); err != nil { + return n.AppNotification, false, err + } + return n.AppNotification, inserted > 0 && cleared == 0, nil +} + +// ActiveAppNotifications returns the active (not cleared) notifications newer +// than the notification named after, oldest first, bounded to the newest +// protocol.MaxReplayedNotifications within the retention window. An empty or +// unknown cursor replays the whole bounded set. cleared lists notifications +// cleared within the window (newest first, bounded). +func (s *Store) ActiveAppNotifications(ctx context.Context, after string, now time.Time) (active []protocol.AppNotification, cleared []string, err error) { + cutoff := now.Add(-AppNotificationRetention).UnixMilli() + var afterSeq int64 + if after != "" { + if err := s.db.QueryRowContext(ctx, `SELECT seq FROM app_notifications WHERE id=?`, after).Scan(&afterSeq); err != nil && !errors.Is(err, sql.ErrNoRows) { + return nil, nil, err + } + } + rows, err := s.db.QueryContext(ctx, `SELECT id,kind,title,message,click,priority,tags,grp,created_at FROM app_notifications +WHERE cleared_at=0 AND seq>? AND created_at>=? ORDER BY seq DESC LIMIT ?`, afterSeq, cutoff, protocol.MaxReplayedNotifications) + if err != nil { + return nil, nil, err + } + active = []protocol.AppNotification{} + for rows.Next() { + var n protocol.AppNotification + var tags string + var created int64 + if err := rows.Scan(&n.ID, &n.Kind, &n.Title, &n.Message, &n.Click, &n.Priority, &tags, &n.Group, &created); err != nil { + rows.Close() + return nil, nil, err + } + if json.Unmarshal([]byte(tags), &n.Tags) != nil || n.Tags == nil { + n.Tags = []string{} + } + n.CreatedAt = time.UnixMilli(created).UTC().Format(time.RFC3339) + active = append(active, n) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, nil, err + } + for i, j := 0, len(active)-1; i < j; i, j = i+1, j-1 { + active[i], active[j] = active[j], active[i] + } + rows, err = s.db.QueryContext(ctx, `SELECT id FROM app_notifications WHERE cleared_at>0 AND created_at>=? ORDER BY cleared_at DESC, seq DESC LIMIT ?`, cutoff, maxReplayedClearedIDs) + if err != nil { + return nil, nil, err + } + defer rows.Close() + cleared = []string{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, nil, err + } + cleared = append(cleared, id) + } + return active, cleared, rows.Err() +} + +// ClearAppNotifications clears the named active notifications and returns the +// ones it changed, in request order. Unknown or already cleared ids are +// skipped. +func (s *Store) ClearAppNotifications(ctx context.Context, ids []string, now time.Time) ([]AppNotificationRecord, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + defer tx.Rollback() + changed := []AppNotificationRecord{} + for _, id := range ids { + var n AppNotificationRecord + err := tx.QueryRowContext(ctx, `UPDATE app_notifications SET cleared_at=? WHERE id=? AND cleared_at=0 +RETURNING id,kind,grp,run_id,session_id`, now.UnixMilli(), id).Scan(&n.ID, &n.Kind, &n.Group, &n.RunID, &n.SessionID) + if errors.Is(err, sql.ErrNoRows) { + continue + } + if err != nil { + return nil, err + } + changed = append(changed, n) + } + return changed, tx.Commit() +} + +// ClearAppNotificationsFor clears every active notification matching filter +// and returns the ids it changed, oldest first. +func (s *Store) ClearAppNotificationsFor(ctx context.Context, filter AppNotificationFilter, now time.Time) ([]string, error) { + var where []string + var args []any + switch { + case filter.RunID > 0: + where, args = append(where, "run_id=?"), append(args, filter.RunID) + case filter.SessionID != "": + where, args = append(where, "session_id=?"), append(args, filter.SessionID) + default: + return []string{}, nil + } + if len(filter.Kinds) > 0 { + where = append(where, "kind IN (?"+strings.Repeat(",?", len(filter.Kinds)-1)+")") + for _, kind := range filter.Kinds { + args = append(args, kind) + } + } + if filter.ID != "" { + where, args = append(where, "id=?"), append(args, filter.ID) + } + rows, err := s.db.QueryContext(ctx, `UPDATE app_notifications SET cleared_at=? +WHERE cleared_at=0 AND `+strings.Join(where, " AND ")+` RETURNING seq,id`, append([]any{now.UnixMilli()}, args...)...) + if err != nil { + return nil, err + } + defer rows.Close() + type cleared struct { + seq int64 + id string + } + var found []cleared + for rows.Next() { + var c cleared + if err := rows.Scan(&c.seq, &c.id); err != nil { + return nil, err + } + found = append(found, c) + } + if err := rows.Err(); err != nil { + return nil, err + } + // RETURNING order is unspecified; report oldest first. + for i := 1; i < len(found); i++ { + for j := i; j > 0 && found[j].seq < found[j-1].seq; j-- { + found[j], found[j-1] = found[j-1], found[j] + } + } + ids := make([]string, 0, len(found)) + for _, c := range found { + ids = append(ids, c.id) + } + return ids, nil +} diff --git a/internal/pipeline/notification_settings_test.go b/internal/pipeline/notification_settings_test.go index 126cd79f..b35ed9ae 100644 --- a/internal/pipeline/notification_settings_test.go +++ b/internal/pipeline/notification_settings_test.go @@ -26,12 +26,12 @@ func TestNotificationSettingsDefaultPersistAndPartialUpdate(t *testing.T) { t.Fatalf("default values changed: %+v", got) } got, err = s.UpdateNotificationSettings(ctx, map[string]any{protocol.NotifyChatReplyFinished: true}) - want := protocol.NotificationSettings{RunFinished: false, ChatNeedsYou: true, ChatReplyFinished: true} + want := protocol.NotificationSettings{RunFinished: false, ChatNeedsYou: true, ChatReplyFinished: true, NtfyEnabled: true} if err != nil || got != want { t.Fatalf("partial update = %+v, %v", got, err) } got, err = s.UpdateNotificationSettings(ctx, map[string]any{protocol.NotifyRunFinished: true, protocol.NotifyChatNeedsYou: false}) - want = protocol.NotificationSettings{RunFinished: true, ChatNeedsYou: false, ChatReplyFinished: true} + want = protocol.NotificationSettings{RunFinished: true, ChatNeedsYou: false, ChatReplyFinished: true, NtfyEnabled: true} if err != nil || got != want { t.Fatalf("second update = %+v, %v", got, err) } diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go index 3a2af589..7e3f9a13 100644 --- a/internal/pipeline/store.go +++ b/internal/pipeline/store.go @@ -81,6 +81,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU db.Close() return nil, err } + if err := s.initAppNotifications(); err != nil { + db.Close() + return nil, err + } if err := s.initRunSizes(); err != nil { db.Close() return nil, err diff --git a/internal/protocol/message.go b/internal/protocol/message.go index d5f8ba85..be475185 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -248,4 +248,13 @@ type Message struct { // whole set) and the attention_acks_changed push (the delta). See // attention_acks.go. AttentionAcks *AttentionAcks `json:"attention_acks,omitempty"` + // App notification stream (see notifications.go): After is the + // subscribe_notifications cursor, Notifications the subscribe reply, + // Notification one push, and IDs the clear_notifications request, the + // notifications_cleared reply/push and the subscribe reply's cleared ids. + // Notifications and IDs are present (possibly empty) whenever set. + After string `json:"after,omitempty"` + Notifications []AppNotification `json:"notifications,omitzero"` + Notification *AppNotification `json:"notification,omitempty"` + IDs []string `json:"ids,omitzero"` } diff --git a/internal/protocol/notification_settings.go b/internal/protocol/notification_settings.go index a87e00b9..5e1cd7cf 100644 --- a/internal/protocol/notification_settings.go +++ b/internal/protocol/notification_settings.go @@ -23,6 +23,7 @@ const ( NotifyRunFinished = "run_finished" NotifyChatNeedsYou = "chat_needs_you" NotifyChatReplyFinished = "chat_reply_finished" + NotifyNtfyEnabled = "ntfy_enabled" ) // NotificationSettings is the typed form of the global notification choices. @@ -35,14 +36,18 @@ type NotificationSettings struct { ChatNeedsYou bool // ChatReplyFinished notifies when an assistant turn in a chat ends. ChatReplyFinished bool + // NtfyEnabled publishes notifications to ntfy. On by default; when off the + // server still records and streams every notification to app clients + // (subscribe_notifications) but stops publishing to ntfy. + NtfyEnabled bool } // NotificationSettingKeys lists every setting key in a stable order. -var NotificationSettingKeys = []string{NotifyRunFinished, NotifyChatNeedsYou, NotifyChatReplyFinished} +var NotificationSettingKeys = []string{NotifyRunFinished, NotifyChatNeedsYou, NotifyChatReplyFinished, NotifyNtfyEnabled} // DefaultNotificationSettings are the values before the operator changes any. func DefaultNotificationSettings() NotificationSettings { - return NotificationSettings{RunFinished: false, ChatNeedsYou: true, ChatReplyFinished: false} + return NotificationSettings{RunFinished: false, ChatNeedsYou: true, ChatReplyFinished: false, NtfyEnabled: true} } // Map returns the wire form with every key present. @@ -51,6 +56,7 @@ func (s NotificationSettings) Map() map[string]any { NotifyRunFinished: s.RunFinished, NotifyChatNeedsYou: s.ChatNeedsYou, NotifyChatReplyFinished: s.ChatReplyFinished, + NotifyNtfyEnabled: s.NtfyEnabled, } } @@ -63,6 +69,8 @@ func (s *NotificationSettings) Field(key string) *bool { return &s.ChatNeedsYou case NotifyChatReplyFinished: return &s.ChatReplyFinished + case NotifyNtfyEnabled: + return &s.NtfyEnabled } return nil } diff --git a/internal/protocol/notifications.go b/internal/protocol/notifications.go new file mode 100644 index 00000000..4581c104 --- /dev/null +++ b/internal/protocol/notifications.go @@ -0,0 +1,107 @@ +package protocol + +import "fmt" + +// App notifications are the same operator notifications the server publishes +// to ntfy, streamed over /connect so the Pipelines app can post them itself +// without the ntfy app. Every publication point records one notification +// (persisted, with a stable id) before it publishes to ntfy, so the stream +// carries exactly the set ntfy gets, governed by the same notification +// settings; notify.ntfy_enabled=false stops only the ntfy publish. +// +// → {"type":"subscribe_notifications","id":"1","after":"run:7:3:waiting"} +// ← {"type":"notifications_subscribed","id":"1","notifications":[{...}],"ids":["chat:ses_x:form:f1"]} +// ← {"type":"notification","notification":{...}} (push) +// ← {"type":"notifications_cleared","ids":["run:7:3:waiting"]} (push) +// → {"type":"clear_notifications","id":"2","ids":["run:7:3:waiting"]} +// ← {"type":"notifications_cleared","id":"2","ids":[...]} (reply: the ids it cleared) +// +// subscribe_notifications replies with every still-active notification newer +// than the optional "after" cursor (a notification id the client already +// holds), oldest first, bounded to the newest MaxReplayedNotifications within +// the store's retention window (7 days); an unknown or pruned cursor replays the whole bounded +// set. The reply's "ids" lists notifications cleared within the retention +// window (bounded) so a reconnecting client can dismiss ones it still shows. +// After the reply the connection receives "notification" and +// "notifications_cleared" pushes in order. Subscribing again replaces the +// subscription. +// +// clear_notifications is the app's dismiss/tap: it clears the named ids, has +// the same effect as the matching ack_attention (a run_waiting id dismisses +// ":Decision", run_failed/stage_failed ":Failed", a chat id marks +// the chat seen) and pushes notifications_cleared to every other subscriber. +// An ack_attention from any client, a run leaving "waiting", a deleted run and +// an answered chat question/approval also clear and push. +const ( + SubscribeNotifications = "subscribe_notifications" + NotificationsSubscribed = "notifications_subscribed" + NotificationPush = "notification" + NotificationsCleared = "notifications_cleared" + ClearNotifications = "clear_notifications" +) + +// App notification kinds. +const ( + NotificationRunWaiting = "run_waiting" // a run needs a human decision + NotificationRunFailed = "run_failed" // a run (or its final stage) failed + NotificationRunSucceeded = "run_succeeded" // a run succeeded + NotificationStageFailed = "stage_failed" // a stage notify hook reported a failure mid-run + NotificationStageSucceeded = "stage_succeeded" // a stage notify hook reported a success mid-run + NotificationChatQuestion = "chat_question" // a chat asked a question + NotificationChatApproval = "chat_approval" // a chat asks for permission/approval + NotificationChatReply = "chat_reply" // a chat finished its reply +) + +// DefaultNotificationPriority is ntfy's default priority, which every ntfy +// publication uses today (the server sends no Priority header). +const DefaultNotificationPriority = 3 + +// MaxReplayedNotifications bounds a subscribe reply and MaxClearNotificationIDs +// one clear_notifications request. +const ( + MaxReplayedNotifications = 100 + MaxClearNotificationIDs = 500 + maxNotificationIDLength = 512 +) + +// AppNotification is one operator notification as streamed to app clients. +type AppNotification struct { + // ID is stable and is the id used to clear it: "run:::" + // or "chat:::". + ID string `json:"id"` + Kind string `json:"kind"` + Title string `json:"title"` + Message string `json:"message"` + // Click is the same deep link ntfy gets (pipelines://run/, + // pipelines://chat/). + Click string `json:"click"` + Priority int `json:"priority"` + Tags []string `json:"tags"` + // CreatedAt is RFC3339 (UTC). + CreatedAt string `json:"created_at"` + // Group is "run:" or "chat:" for Android notification grouping. + Group string `json:"group"` +} + +// ValidateClearNotifications checks a clear_notifications id list. +func ValidateClearNotifications(ids []string) ([]string, error) { + if len(ids) == 0 { + return nil, fmt.Errorf("clear_notifications must name at least one id") + } + if len(ids) > MaxClearNotificationIDs { + return nil, fmt.Errorf("clear_notifications may carry at most %d ids", MaxClearNotificationIDs) + } + seen := make(map[string]struct{}, len(ids)) + result := make([]string, 0, len(ids)) + for _, id := range ids { + if id == "" || len(id) > maxNotificationIDLength { + return nil, fmt.Errorf("invalid notification id %q", id) + } + if _, dup := seen[id]; dup { + continue + } + seen[id] = struct{}{} + result = append(result, id) + } + return result, nil +} diff --git a/internal/server/app_notifications.go b/internal/server/app_notifications.go new file mode 100644 index 00000000..9a3b6e86 --- /dev/null +++ b/internal/server/app_notifications.go @@ -0,0 +1,323 @@ +package server + +import ( + "context" + "fmt" + "log" + "strconv" + "strings" + "sync" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// appNotificationStore persists the app notification outbox. +// *pipeline.Store implements it. +type appNotificationStore interface { + RecordAppNotification(context.Context, pipeline.AppNotificationRecord, time.Time) (protocol.AppNotification, bool, error) + ActiveAppNotifications(context.Context, string, time.Time) ([]protocol.AppNotification, []string, error) + ClearAppNotifications(context.Context, []string, time.Time) ([]pipeline.AppNotificationRecord, error) + ClearAppNotificationsFor(context.Context, pipeline.AppNotificationFilter, time.Time) ([]string, error) +} + +// appNotificationQueue bounds one subscriber's undelivered frames. A client +// that falls this far behind is dropped; it recovers with a resubscribe. +const appNotificationQueue = 512 + +// appNotifier records every operator notification (the same ones published to +// ntfy) and streams them to subscribed connections. One mutex orders store +// writes with fan-out, so each subscriber sees its snapshot reply and then +// every later push exactly in commit order. +type appNotifier struct { + store appNotificationStore + now func() time.Time + mu sync.Mutex + subs map[string]*appNotificationSub +} + +type appNotificationSub struct { + queue chan protocol.Message + done chan struct{} + once sync.Once +} + +func newAppNotifier(store appNotificationStore) *appNotifier { + return &appNotifier{store: store, now: time.Now, subs: map[string]*appNotificationSub{}} +} + +func (s *appNotificationSub) stop() { s.once.Do(func() { close(s.done) }) } + +func (s *appNotificationSub) run(write func(protocol.Message) error) { + for { + select { + case <-s.done: + return + case msg := <-s.queue: + if write(msg) != nil { + s.stop() + return + } + } + } +} + +// enqueueLocked queues msg for every subscriber except exclude. Callers hold +// a.mu. +func (a *appNotifier) enqueueLocked(msg protocol.Message, exclude string) { + for id, sub := range a.subs { + if id == exclude { + continue + } + select { + case sub.queue <- msg: + case <-sub.done: + delete(a.subs, id) + default: + log.Printf("app notifications: dropping slow subscriber %s", id) + sub.stop() + delete(a.subs, id) + } + } +} + +// subscribe replaces clientID's subscription with one that first delivers the +// replay reply and then every later push. +func (a *appNotifier) subscribe(ctx context.Context, clientID, requestID, after string, write func(protocol.Message) error) error { + a.mu.Lock() + defer a.mu.Unlock() + active, cleared, err := a.store.ActiveAppNotifications(ctx, after, a.now()) + if err != nil { + return err + } + if old := a.subs[clientID]; old != nil { + old.stop() + } + sub := &appNotificationSub{queue: make(chan protocol.Message, appNotificationQueue), done: make(chan struct{})} + sub.queue <- protocol.Message{Type: protocol.NotificationsSubscribed, ID: requestID, Notifications: active, IDs: cleared} + a.subs[clientID] = sub + go sub.run(write) + return nil +} + +func (a *appNotifier) detach(clientID string) { + a.mu.Lock() + defer a.mu.Unlock() + if sub := a.subs[clientID]; sub != nil { + sub.stop() + delete(a.subs, clientID) + } +} + +// record persists n and pushes it when it is new. Failures are logged only: +// recording must never block or fail the ntfy publication beside it. +func (a *appNotifier) record(ctx context.Context, n pipeline.AppNotificationRecord) { + if a == nil { + return + } + a.mu.Lock() + defer a.mu.Unlock() + stored, fresh, err := a.store.RecordAppNotification(ctx, n, a.now()) + if err != nil { + log.Printf("record app notification %s: %v", n.ID, err) + return + } + if fresh { + a.enqueueLocked(protocol.Message{Type: protocol.NotificationPush, Notification: &stored}, "") + } +} + +// clear clears ids and pushes the ones that changed to every subscriber but +// exclude (the requester, which receives them in its reply). +func (a *appNotifier) clear(ctx context.Context, ids []string, exclude string) ([]pipeline.AppNotificationRecord, error) { + a.mu.Lock() + defer a.mu.Unlock() + changed, err := a.store.ClearAppNotifications(ctx, ids, a.now()) + if err != nil { + return nil, err + } + if len(changed) > 0 { + a.enqueueLocked(protocol.Message{Type: protocol.NotificationsCleared, IDs: recordIDs(changed)}, exclude) + } + return changed, nil +} + +// clearFor clears every active notification matching filter and pushes the +// change to every subscriber. +func (a *appNotifier) clearFor(ctx context.Context, filter pipeline.AppNotificationFilter) { + if a == nil { + return + } + a.mu.Lock() + defer a.mu.Unlock() + ids, err := a.store.ClearAppNotificationsFor(ctx, filter, a.now()) + if err != nil { + log.Printf("clear app notifications: %v", err) + return + } + if len(ids) > 0 { + a.enqueueLocked(protocol.Message{Type: protocol.NotificationsCleared, IDs: ids}, "") + } +} + +// clearForAcks clears the notifications an attention acknowledgement covers: +// ":Decision" the run's waiting notifications, ":Failed" its failure +// notifications, and a chat_seen watermark every notification of that chat. +func (a *appNotifier) clearForAcks(ctx context.Context, ack protocol.AttentionAcks) { + if a == nil { + return + } + for _, key := range ack.Dismissed { + run, kind, ok := strings.Cut(key, ":") + id, err := strconv.ParseInt(run, 10, 64) + if !ok || err != nil { + continue + } + kinds := []string{protocol.NotificationRunWaiting} + if kind == "Failed" { + kinds = []string{protocol.NotificationRunFailed, protocol.NotificationStageFailed} + } + a.clearFor(ctx, pipeline.AppNotificationFilter{RunID: id, Kinds: kinds}) + } + for session := range ack.ChatSeen { + a.clearFor(ctx, pipeline.AppNotificationFilter{SessionID: session}) + } +} + +func recordIDs(records []pipeline.AppNotificationRecord) []string { + ids := make([]string, 0, len(records)) + for _, r := range records { + ids = append(ids, r.ID) + } + return ids +} + +// acksForCleared maps notifications cleared in the app to the equivalent +// attention acknowledgement, so a dismissal in the notification shade has the +// same effect as one in the Needs-you list. +func acksForCleared(records []pipeline.AppNotificationRecord) protocol.AttentionAcks { + ack := protocol.AttentionAcks{Dismissed: []string{}, ChatSeen: map[string]int64{}} + for _, r := range records { + switch { + case r.RunID > 0 && r.Kind == protocol.NotificationRunWaiting: + ack.Dismissed = append(ack.Dismissed, fmt.Sprintf("%d:Decision", r.RunID)) + case r.RunID > 0 && (r.Kind == protocol.NotificationRunFailed || r.Kind == protocol.NotificationStageFailed): + ack.Dismissed = append(ack.Dismissed, fmt.Sprintf("%d:Failed", r.RunID)) + case r.SessionID != "": + // The store raises a watermark to the session's own recency, so 1 + // acknowledges everything the chat holds now. + ack.ChatSeen[r.SessionID] = 1 + } + } + return ack +} + +// handleAppNotifications answers subscribe_notifications and +// clear_notifications. It reports whether request was one of them. +func handleAppNotifications(ctx context.Context, request protocol.Message, notes *appNotifier, ackStore attentionAckStore, acks *attentionAckHub, clientID string, write func(protocol.Message) error) bool { + if request.Type != protocol.SubscribeNotifications && request.Type != protocol.ClearNotifications { + return false + } + if notes == nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "notifications are unavailable: no store"}) + return true + } + if request.Type == protocol.SubscribeNotifications { + if err := notes.subscribe(ctx, clientID, request.ID, request.After, write); err != nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) + } + return true + } + ids, err := protocol.ValidateClearNotifications(request.IDs) + if err != nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) + return true + } + changed, err := notes.clear(ctx, ids, clientID) + if err != nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) + return true + } + _ = write(protocol.Message{Type: protocol.NotificationsCleared, ID: request.ID, IDs: recordIDs(changed)}) + ack := acksForCleared(changed) + if ack.Empty() { + return true + } + if ackStore != nil { + if valid, err := protocol.ValidateAttentionAck(&ack); err == nil { + delta, _, err := ackStore.AckAttention(ctx, &valid, time.Now()) + if err != nil { + log.Printf("ack cleared notifications: %v", err) + } else if acks != nil && !delta.Empty() { + // The requester's Needs-you list changes too, so it is included. + acks.broadcast("", delta) + } + } + } + // The equivalent ack also covers sibling notifications (the chat's other + // notifications, the run's earlier waits); push those to everyone. + notes.clearForAcks(ctx, ack) + return true +} + +// ntfyEnabled reads notify.ntfy_enabled. Without a store, or when the read +// fails, ntfy stays enabled (the default) so a transient error never silences +// the operator. +func ntfyEnabled(ctx context.Context, store notificationSettingsStore) bool { + if store == nil { + return true + } + settings, err := store.NotificationSettings(ctx) + if err != nil { + log.Printf("read notification settings: %v", err) + return true + } + return settings.NtfyEnabled +} + +// ntfyDisabledResult is what a publication reports when ntfy is turned off: +// the notification reached app clients only. +func ntfyDisabledResult() stage.Values { + return stage.Values{"destination": "system", "message_id": "", "ntfy": "disabled"} +} + +// runNotificationKind maps a run outbox event to its app notification kind. +func runNotificationKind(event string) string { + switch event { + case protocol.NotifyWaiting: + return protocol.NotificationRunWaiting + case "failed": + return protocol.NotificationRunFailed + case "succeeded": + return protocol.NotificationRunSucceeded + } + return "run_" + event +} + +// runNotificationRecord builds the app notification for one run event: +// identity "run:::", deep link and group from the run id. +func runNotificationRecord(runID int64, sequence int, event, kind, title, message string) pipeline.AppNotificationRecord { + return pipeline.AppNotificationRecord{ + AppNotification: protocol.AppNotification{ + ID: fmt.Sprintf("run:%d:%d:%s", runID, sequence, event), + Kind: kind, + Title: title, + Message: message, + Click: runAttentionClick(runID), + Priority: protocol.DefaultNotificationPriority, + Tags: []string{}, + Group: fmt.Sprintf("run:%d", runID), + }, + RunID: runID, + } +} + +// settingsStore returns the runner's store as a settings reader, or nil. +func (r *runner) settingsStore() notificationSettingsStore { + if r.store == nil { + return nil + } + return r.store +} diff --git a/internal/server/app_notifications_test.go b/internal/server/app_notifications_test.go new file mode 100644 index 00000000..cb55ecfa --- /dev/null +++ b/internal/server/app_notifications_test.go @@ -0,0 +1,365 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "net/http/httptest" + "testing" + "time" + + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" + "git.a-wan.net/alan/pipelines/internal/summarise" +) + +// appNotificationFixture is a handler over a temp store with one pipeline, +// plus helpers to make a run end with a durable notification intent. +type appNotificationFixture struct { + t *testing.T + ctx context.Context + store *pipeline.Store + runner *runner + url string + def *protocol.PipelineDetail +} + +func newAppNotificationFixture(t *testing.T, ctx context.Context) *appNotificationFixture { + t.Helper() + store := newOperatorStore(t) + if _, err := store.Submit(ctx, "notes", pipeline.Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + def, err := store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + handlerCtx, stop := context.WithCancel(ctx) + handler := Handler(handlerCtx, store) + t.Cleanup(func() { stop(); WaitRuns(handler) }) + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + return &appNotificationFixture{t: t, ctx: ctx, store: store, runner: handler.(*runHandler).runner, url: srv.URL, def: def} +} + +// failRun creates a "Notify me" run and fails it, which commits a durable run +// notification intent in the same transaction, then drains the outbox. +func (f *appNotificationFixture) failRun(name string) int64 { + f.t.Helper() + id, _, err := f.store.CreateRunTitled(f.ctx, f.def, name, f.t.TempDir(), false, nil, nil, pipeline.RunOverrides{NotifyMode: "on"}, summarise.Title{}) + if err != nil { + f.t.Fatal(err) + } + if err := f.store.FinishRun(f.ctx, id, "failed", "error", "boom", nil); err != nil { + f.t.Fatal(err) + } + f.runner.deliverRunNotifications(time.Now()) + return id +} + +func readFrame(t *testing.T, ctx context.Context, conn *websocket.Conn, want string) protocol.Message { + t.Helper() + var msg protocol.Message + if err := wsjson.Read(ctx, conn, &msg); err != nil { + t.Fatal(err) + } + if msg.Type != want { + t.Fatalf("frame type = %q, want %q: %+v", msg.Type, want, msg) + } + return msg +} + +func subscribeNotifications(t *testing.T, ctx context.Context, conn *websocket.Conn, after string) protocol.Message { + t.Helper() + if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.SubscribeNotifications, ID: "sub", After: after}); err != nil { + t.Fatal(err) + } + reply := readFrame(t, ctx, conn, protocol.NotificationsSubscribed) + if reply.ID != "sub" { + t.Fatalf("subscribe reply id = %q", reply.ID) + } + return reply +} + +func TestAppNotificationStreamMatchesNtfy(t *testing.T) { + t.Setenv("PIPELINES_WEB_PUBLIC_URL", "") + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + captured := newNtfyCapture(t) + f := newAppNotificationFixture(t, ctx) + phone := dialAckClient(t, ctx, f.url) + initial := subscribeNotifications(t, ctx, phone, "") + if initial.Notifications == nil || len(initial.Notifications) != 0 || initial.IDs == nil { + t.Fatalf("initial reply = %+v", initial) + } + run := f.failRun("stream") + push := readFrame(t, ctx, phone, protocol.NotificationPush) + got := captured() + if len(got) != 1 { + t.Fatalf("ntfy deliveries = %+v", got) + } + n := push.Notification + if n == nil || n.Title != got[0].Title || n.Message != got[0].Message || n.Click != got[0].Click { + t.Fatalf("stream %+v differs from ntfy %+v", n, got[0]) + } + if n.ID != fmt.Sprintf("run:%d:0:failed", run) || n.Kind != protocol.NotificationRunFailed || n.Group != fmt.Sprintf("run:%d", run) || + n.Priority != 3 || n.Tags == nil || len(n.Tags) != 0 || n.Click != fmt.Sprintf("pipelines://run/%d", run) { + t.Fatalf("notification fields = %+v", n) + } + if _, err := time.Parse(time.RFC3339, n.CreatedAt); err != nil { + t.Fatalf("created_at = %q", n.CreatedAt) + } + // The wire shape the app parses. + raw, _ := json.Marshal(push) + var wire map[string]any + _ = json.Unmarshal(raw, &wire) + body, _ := wire["notification"].(map[string]any) + for _, key := range []string{"id", "kind", "title", "message", "click", "priority", "tags", "created_at", "group"} { + if _, ok := body[key]; !ok { + t.Fatalf("notification wire lacks %q: %s", key, raw) + } + } + // A retried publication of the same event is not streamed twice. + f.runner.deliverRunNotifications(time.Now().Add(time.Hour)) + reply := subscribeNotifications(t, ctx, phone, "") + if len(reply.Notifications) != 1 || reply.Notifications[0].ID != n.ID || reply.Notifications[0].CreatedAt != n.CreatedAt { + t.Fatalf("replay = %+v", reply.Notifications) + } +} + +func TestAppNotificationReplayAfterCursorAndCrossDeviceClear(t *testing.T) { + t.Setenv("PIPELINES_NTFY_URL", "") + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + f := newAppNotificationFixture(t, ctx) + // Offline: two runs fail with no app connected (ntfy unconfigured too). + first := f.failRun("first") + second := f.failRun("second") + firstID, secondID := fmt.Sprintf("run:%d:0:failed", first), fmt.Sprintf("run:%d:0:failed", second) + + phone := dialAckClient(t, ctx, f.url) + all := subscribeNotifications(t, ctx, phone, "") + if len(all.Notifications) != 2 || all.Notifications[0].ID != firstID || all.Notifications[1].ID != secondID { + t.Fatalf("full replay = %+v", all.Notifications) + } + // Reconnect with the cursor of the first: only the missed second replays. + phone.CloseNow() + phone = dialAckClient(t, ctx, f.url) + missed := subscribeNotifications(t, ctx, phone, firstID) + if len(missed.Notifications) != 1 || missed.Notifications[0].ID != secondID { + t.Fatalf("replay after %s = %+v", firstID, missed.Notifications) + } + // An unknown cursor replays the bounded active set. + web := dialAckClient(t, ctx, f.url) + if unknown := subscribeNotifications(t, ctx, web, "run:999:0:failed"); len(unknown.Notifications) != 2 { + t.Fatalf("unknown cursor replay = %+v", unknown.Notifications) + } + + // Clearing on the web broadcasts to the phone and acks the run. + if err := wsjson.Write(ctx, web, protocol.Message{Type: protocol.ClearNotifications, ID: "clr", IDs: []string{firstID, "unknown"}}); err != nil { + t.Fatal(err) + } + reply := readFrame(t, ctx, web, protocol.NotificationsCleared) + if reply.ID != "clr" || len(reply.IDs) != 1 || reply.IDs[0] != firstID { + t.Fatalf("clear reply = %+v", reply) + } + pushed := readFrame(t, ctx, phone, protocol.NotificationsCleared) + if pushed.ID != "" || len(pushed.IDs) != 1 || pushed.IDs[0] != firstID { + t.Fatalf("phone cleared push = %+v", pushed) + } + // Same effect as the ack: the phone's Needs-you list hears it too. + changed := readFrame(t, ctx, phone, protocol.AttentionAcksChanged) + if changed.AttentionAcks == nil || len(changed.AttentionAcks.Dismissed) != 1 || changed.AttentionAcks.Dismissed[0] != fmt.Sprintf("%d:Failed", first) { + t.Fatalf("ack push = %+v", changed) + } + + // An ack_attention from the phone clears the second on the web. + ack := ackExchange(t, ctx, phone, protocol.Message{Type: protocol.AckAttention, ID: "ack", AttentionAcks: &protocol.AttentionAcks{Dismissed: []string{fmt.Sprintf("%d:Failed", second)}}}) + if ack.Type != protocol.AttentionAcksReply { + t.Fatalf("ack reply = %+v", ack) + } + // The web receives the ack push (from the ack hub) and the clear push in + // either order. + seen := map[string]protocol.Message{} + for len(seen) < 2 { + var msg protocol.Message + if err := wsjson.Read(ctx, web, &msg); err != nil { + t.Fatal(err) + } + seen[msg.Type] = msg + } + if c := seen[protocol.NotificationsCleared]; len(c.IDs) != 1 || c.IDs[0] != secondID { + t.Fatalf("web cleared push = %+v", seen) + } + phoneClear := readFrame(t, ctx, phone, protocol.NotificationsCleared) + if len(phoneClear.IDs) != 1 || phoneClear.IDs[0] != secondID { + t.Fatalf("phone cleared push = %+v", phoneClear) + } + // Nothing active is left; both ids are reported cleared on resubscribe. + final := subscribeNotifications(t, ctx, phone, "") + if len(final.Notifications) != 0 || len(final.IDs) != 2 { + t.Fatalf("final replay = %+v", final) + } + // A malformed clear is refused. + if err := wsjson.Write(ctx, phone, protocol.Message{Type: protocol.ClearNotifications, ID: "bad"}); err != nil { + t.Fatal(err) + } + if bad := readFrame(t, ctx, phone, protocol.Error); bad.ID != "bad" { + t.Fatalf("bad clear = %+v", bad) + } +} + +func TestNtfyDisabledStillStreams(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + captured := newNtfyCapture(t) + f := newAppNotificationFixture(t, ctx) + phone := dialAckClient(t, ctx, f.url) + set := ackExchange(t, ctx, phone, protocol.Message{Type: protocol.SetNotificationSettings, ID: "off", NotificationSettings: map[string]any{protocol.NotifyNtfyEnabled: false}}) + if set.Type != protocol.NotificationSettingsReply || set.NotificationSettings[protocol.NotifyNtfyEnabled] != false { + t.Fatalf("settings reply = %+v", set) + } + subscribeNotifications(t, ctx, phone, "") + run := f.failRun("quiet") + push := readFrame(t, ctx, phone, protocol.NotificationPush) + if push.Notification == nil || push.Notification.ID != fmt.Sprintf("run:%d:0:failed", run) || push.Notification.Title != "Run failed" { + t.Fatalf("push = %+v", push) + } + if got := captured(); len(got) != 0 { + t.Fatalf("ntfy published while disabled: %+v", got) + } + // The outbox is settled, not retried against ntfy. + rows, err := f.store.RunNotifications(ctx, run) + if err != nil || len(rows) != 1 || rows[0].Status != "sent" { + t.Fatalf("outbox = %+v %v", rows, err) + } +} + +func TestChatAppNotificationsRespectNtfyEnabledAndClearOnAnswer(t *testing.T) { + t.Setenv("PIPELINES_NTFY_URL", "http://ntfy.invalid/pipelines") + t.Setenv("PIPELINES_WEB_PUBLIC_URL", "") + var published []ntfyCapture + original := chatAttentionPublish + chatAttentionPublish = func(_ context.Context, _, title, message, _ string, _ []byte, click string, _ ...stage.NtfyView) (stage.Values, error) { + published = append(published, ntfyCapture{Title: title, Message: message, Click: click}) + return stage.Values{}, nil + } + defer func() { chatAttentionPublish = original }() + ctx := context.Background() + store := newOperatorStore(t) + h := chatAttentionHub(t, "ses_a") + h.settings = store + h.notes = newAppNotifier(store) + form := protocol.ChatForm{ID: "que_1", SessionID: "ses_a", Kind: "question", Title: "Ship it?"} + h.emit(protocol.ChatEventFrame{SessionID: "ses_a", Kind: protocol.ChatEventForm, Form: &form}) + h.attentionWG.Wait() + active, _, err := store.ActiveAppNotifications(ctx, "", time.Now()) + if err != nil || len(published) != 1 || len(active) != 1 { + t.Fatalf("published=%+v active=%+v err=%v", published, active, err) + } + n := active[0] + if n.ID != "chat:ses_a:form:que_1" || n.Kind != protocol.NotificationChatQuestion || n.Group != "chat:ses_a" || + n.Title != published[0].Title || n.Message != published[0].Message || n.Click != published[0].Click { + t.Fatalf("chat notification %+v vs ntfy %+v", n, published[0]) + } + // Answering the question clears it. + form.Resolved = true + h.emit(protocol.ChatEventFrame{SessionID: "ses_a", Kind: protocol.ChatEventForm, Form: &form}) + h.attentionWG.Wait() + if active, _, _ := store.ActiveAppNotifications(ctx, "", time.Now()); len(active) != 0 { + t.Fatalf("answered question still active: %+v", active) + } + // With ntfy off, the approval is streamed but not published. + if _, err := store.UpdateNotificationSettings(ctx, map[string]any{protocol.NotifyNtfyEnabled: false}); err != nil { + t.Fatal(err) + } + h.pushPermission("ses_a", protocol.ChatPermissionFrame{SessionID: "ses_a", RequestID: "per_1", Message: "Allow rm?"}) + h.attentionWG.Wait() + active, _, _ = store.ActiveAppNotifications(ctx, "", time.Now()) + if len(published) != 1 || len(active) != 1 || active[0].Kind != protocol.NotificationChatApproval || active[0].ID != "chat:ses_a:permission:per_1" { + t.Fatalf("published=%+v active=%+v", published, active) + } + h.confirmPermission("ses_a", "per_1", "phone", "once") + h.attentionWG.Wait() + if active, _, _ := store.ActiveAppNotifications(ctx, "", time.Now()); len(active) != 0 { + t.Fatalf("answered approval still active: %+v", active) + } +} + +func TestRunLeavingWaitingClearsWaitingNotification(t *testing.T) { + ctx := context.Background() + store := newOperatorStore(t) + notes := newAppNotifier(store) + r := &runner{ctx: ctx, store: store, notes: notes} + notes.record(ctx, runNotificationRecord(5, 2, protocol.NotifyWaiting, protocol.NotificationRunWaiting, "Run needs your decision", "Run #5")) + notes.record(ctx, runNotificationRecord(5, 1, "success", protocol.NotificationStageSucceeded, "Pipeline stage success", "digest")) + r.emitRun(protocol.RunEventFrame{RunID: 5, Status: protocol.RunStatusWaiting}, false) + if active, _, _ := store.ActiveAppNotifications(ctx, "", time.Now()); len(active) != 2 { + t.Fatalf("waiting frame cleared: %+v", active) + } + r.emitRun(protocol.RunEventFrame{RunID: 5, Status: "running"}, false) + active, cleared, _ := store.ActiveAppNotifications(ctx, "", time.Now()) + if len(active) != 1 || active[0].Kind != protocol.NotificationStageSucceeded || len(cleared) != 1 || cleared[0] != "run:5:2:waiting" { + t.Fatalf("active=%+v cleared=%+v", active, cleared) + } +} + +// A stage notify hook streams one notification per step event (artifacts stay +// ntfy attachments) with the hook's title and summary, and with ntfy disabled +// it still streams while publishing nothing. +func TestStageHookAppNotification(t *testing.T) { + t.Setenv("PIPELINES_WEB_PUBLIC_URL", "") + captured := newNtfyCapture(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + store := newOperatorStore(t) + if _, err := store.Submit(ctx, "hooks", pipeline.Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + items, _ := store.List(ctx) + d, _ := store.Get(ctx, items[0].ID, 0) + workspace := t.TempDir() + r := &runner{ctx: ctx, store: store, notes: newAppNotifier(store)} + policy := &protocol.NotificationPolicy{On: []string{"success"}, Destination: "system", Artifacts: []string{"markdown"}} + for i, ntfyOn := range []bool{true, false} { + if _, err := store.UpdateNotificationSettings(ctx, map[string]any{protocol.NotifyNtfyEnabled: ntfyOn}); err != nil { + t.Fatal(err) + } + id, _, err := store.CreateRun(ctx, d, fmt.Sprintf("hook%d", i), workspace, false, nil, map[string]any{"prompt": "task"}) + if err != nil { + t.Fatal(err) + } + if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil { + t.Fatal(err) + } + step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"markdown": "# Digest"}} + if err := store.FinishStep(ctx, id, step); err != nil { + t.Fatal(err) + } + before := len(captured()) + r.notifyStep(id, d.Name, workspace, policy, step) + active, _, err := store.ActiveAppNotifications(ctx, "", time.Now()) + if err != nil || len(active) != i+1 { + t.Fatalf("active = %+v %v", active, err) + } + n := active[i] + if n.ID != fmt.Sprintf("run:%d:1:success", id) || n.Kind != protocol.NotificationStageSucceeded || n.Click != fmt.Sprintf("pipelines://run/%d", id) { + t.Fatalf("hook notification = %+v", n) + } + got := captured() + if ntfyOn && (len(got) != before+1 || got[before].Title != n.Title || got[before].Message != n.Message || got[before].Click != n.Click) { + t.Fatalf("ntfy %+v vs stream %+v", got, n) + } + if !ntfyOn && len(got) != before { + t.Fatalf("ntfy published while disabled: %+v", got) + } + } +} diff --git a/internal/server/attention_acks.go b/internal/server/attention_acks.go index 280d5962..82ff2e01 100644 --- a/internal/server/attention_acks.go +++ b/internal/server/attention_acks.go @@ -63,7 +63,7 @@ func (h *attentionAckHub) broadcast(sender string, delta protocol.AttentionAcks) // whether request was one of them. An ack that changes nothing (already // dismissed, or a watermark that does not advance) replies with the whole set // but pushes nothing. -func handleAttentionAcks(ctx context.Context, request protocol.Message, store attentionAckStore, hub *attentionAckHub, clientID string, write func(protocol.Message) error) bool { +func handleAttentionAcks(ctx context.Context, request protocol.Message, store attentionAckStore, hub *attentionAckHub, notes *appNotifier, clientID string, write func(protocol.Message) error) bool { if request.Type != protocol.GetAttentionAcks && request.Type != protocol.AckAttention { return false } @@ -86,5 +86,13 @@ func handleAttentionAcks(ctx context.Context, request protocol.Message, store at if hub != nil && !delta.Empty() { hub.broadcast(clientID, delta) } + // The acknowledged items' app notifications clear on every device. The + // request (not only the delta) decides, so an ack repeated after a new + // notification still clears it. + if request.Type == protocol.AckAttention && notes != nil { + if valid, err := protocol.ValidateAttentionAck(request.AttentionAcks); err == nil { + notes.clearForAcks(ctx, valid) + } + } return true } diff --git a/internal/server/chat_attention.go b/internal/server/chat_attention.go index c6470b6e..15d02f6b 100644 --- a/internal/server/chat_attention.go +++ b/internal/server/chat_attention.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) @@ -27,6 +28,13 @@ type chatAttention struct { // rather than a question/approval (gated by chat_needs_you). reply bool body string + // id is the stable app notification id ("chat:::"). + id string +} + +// chatNotificationID is the app notification id for one chat attention item. +func chatNotificationID(sessionID, what, id string) string { + return "chat:" + sessionID + ":" + what + ":" + id } // claimChatAttentionLocked records key in the hub-wide dedupe set and reports @@ -53,11 +61,15 @@ func (h *chatHub) claimChatAttentionLocked(key string) bool { // question/form. A form first seen resolved or cancelled is only recorded, so a // later stale copy cannot announce it. Callers must hold h.mu. func (h *chatHub) noteFormAttentionLocked(frame protocol.ChatEventFrame) { - if frame.Kind != protocol.ChatEventForm || frame.Form == nil || frame.Form.ID == "" || frame.SessionID == "" || !chatAttentionConfigured() { + if frame.Kind != protocol.ChatEventForm || frame.Form == nil || frame.Form.ID == "" || frame.SessionID == "" || !h.attentionWanted() { return } form := frame.Form if !h.claimChatAttentionLocked("form\x00" + frame.SessionID + "\x00" + form.ID) { + // An answered or cancelled question no longer needs the operator. + if form.Resolved || form.Cancelled { + h.clearChatAttention(frame.SessionID, chatNotificationID(frame.SessionID, "form", form.ID)) + } return } if form.Resolved || form.Cancelled { @@ -79,13 +91,13 @@ func (h *chatHub) noteFormAttentionLocked(frame protocol.ChatEventFrame) { body = "The agent asked a question" } kind := strings.ToLower(form.Kind) - h.startChatAttention(chatAttention{sessionID: frame.SessionID, approval: strings.Contains(kind, "permission") || strings.Contains(kind, "approval"), body: body}) + h.startChatAttention(chatAttention{sessionID: frame.SessionID, approval: strings.Contains(kind, "permission") || strings.Contains(kind, "approval"), body: body, id: chatNotificationID(frame.SessionID, "form", form.ID)}) } // notePermissionAttentionLocked notifies the operator once about a pending // approval request. Callers must hold h.mu. func (h *chatHub) notePermissionAttentionLocked(frame protocol.ChatPermissionFrame) { - if frame.RequestID == "" || frame.SessionID == "" || frame.Resolved || !chatAttentionConfigured() { + if frame.RequestID == "" || frame.SessionID == "" || frame.Resolved || !h.attentionWanted() { return } if !h.claimChatAttentionLocked("permission\x00" + frame.SessionID + "\x00" + frame.RequestID) { @@ -101,13 +113,39 @@ func (h *chatHub) notePermissionAttentionLocked(frame protocol.ChatPermissionFra if strings.TrimSpace(body) == "" { body = "The agent asked for permission" } - h.startChatAttention(chatAttention{sessionID: frame.SessionID, approval: true, body: body}) + h.startChatAttention(chatAttention{sessionID: frame.SessionID, approval: true, body: body, id: chatNotificationID(frame.SessionID, "permission", frame.RequestID)}) } func chatAttentionConfigured() bool { return os.Getenv("PIPELINES_NTFY_URL") != "" } +// attentionWanted reports whether a chat attention item has anywhere to go: +// the ntfy connection or the app notification stream. +func (h *chatHub) attentionWanted() bool { + return chatAttentionConfigured() || h.notes != nil +} + +// clearChatAttention clears one chat app notification asynchronously, off the +// hub mutex. +func (h *chatHub) clearChatAttention(sessionID, id string) { + if h.notes == nil { + return + } + base := h.baseCtx + if base == nil { + base = context.Background() + } + notes := h.notes + h.attentionWG.Add(1) + go func() { + defer h.attentionWG.Done() + ctx, cancel := context.WithTimeout(base, 5*time.Second) + defer cancel() + notes.clearFor(ctx, pipeline.AppNotificationFilter{SessionID: sessionID, ID: id}) + }() +} + // startChatAttention publishes asynchronously within the 30-second notification // budget. It never blocks or fails chat handling; failures are only logged. func (h *chatHub) startChatAttention(a chatAttention) { @@ -117,6 +155,7 @@ func (h *chatHub) startChatAttention(a chatAttention) { } registry := h.registry settingsStore := h.settings + notes := h.notes h.attentionWG.Add(1) go func() { defer h.attentionWG.Done() @@ -138,6 +177,20 @@ func (h *chatHub) startChatAttention(a chatAttention) { title, body = chatReplyText(name, a.body) } click, views := chatAttentionLinks(a.sessionID) + kind := protocol.NotificationChatQuestion + if a.reply { + kind = protocol.NotificationChatReply + } else if a.approval { + kind = protocol.NotificationChatApproval + } + notes.record(ctx, pipeline.AppNotificationRecord{ + AppNotification: protocol.AppNotification{ID: a.id, Kind: kind, Title: title, Message: body, Click: click, + Priority: protocol.DefaultNotificationPriority, Tags: []string{}, Group: "chat:" + a.sessionID}, + SessionID: a.sessionID, + }) + if !chatAttentionConfigured() || !settings.NtfyEnabled { + return + } if _, err := chatAttentionPublish(ctx, "system", title, body, "", nil, click, views...); err != nil { log.Printf("chat attention notification session=%s: %v", a.sessionID, err) } diff --git a/internal/server/chat_hub.go b/internal/server/chat_hub.go index c284ae84..57645fbf 100644 --- a/internal/server/chat_hub.go +++ b/internal/server/chat_hub.go @@ -39,7 +39,9 @@ type chatHub struct { registry operatorRegistry // settings holds the operator's notification choices; nil means defaults. settings notificationSettingsStore - now func() time.Time + // notes records and streams app notifications; nil records nothing. + notes *appNotifier + now func() time.Time // userSeq is a monotonic source of ids for the user turns the server echoes // to the other clients, so each retained user message has a stable id. userSeq int diff --git a/internal/server/chat_permissions.go b/internal/server/chat_permissions.go index 32d006ba..a5a636b8 100644 --- a/internal/server/chat_permissions.go +++ b/internal/server/chat_permissions.go @@ -105,6 +105,7 @@ func (h *chatHub) confirmPermission(sessionID, requestID, client, decision strin } } h.mu.Unlock() + h.clearChatAttention(sessionID, chatNotificationID(sessionID, "permission", requestID)) frame.Resolved = true frame.Decision = decision frame.ResolvedBy = client diff --git a/internal/server/chat_reply_attention.go b/internal/server/chat_reply_attention.go index 4eec0137..75c239c3 100644 --- a/internal/server/chat_reply_attention.go +++ b/internal/server/chat_reply_attention.go @@ -1,6 +1,7 @@ package server import ( + "strconv" "strings" "git.a-wan.net/alan/pipelines/internal/protocol" @@ -28,7 +29,7 @@ func (h *chatHub) noteReplyAttentionLocked(prior *protocol.ChatStep, frame proto if prior == nil || (prior.Status == protocol.ChatStepEnded && !prior.Intermediate) { return } - if !chatAttentionConfigured() { + if !h.attentionWanted() { return } if step.TurnID != "" && !h.claimChatAttentionLocked("reply\x00"+frame.SessionID+"\x00"+step.TurnID) { @@ -38,7 +39,11 @@ func (h *chatHub) noteReplyAttentionLocked(prior *protocol.ChatStep, frame proto if body == "" { body = chatReplyFallbackBody } - h.startChatAttention(chatAttention{sessionID: frame.SessionID, reply: true, body: body}) + turnID := step.TurnID + if turnID == "" { + turnID = strconv.FormatInt(h.clock().UnixNano(), 36) + } + h.startChatAttention(chatAttention{sessionID: frame.SessionID, reply: true, body: body, id: chatNotificationID(frame.SessionID, "reply", turnID)}) } // finalReplyTextLocked returns a bounded single-line excerpt of the latest diff --git a/internal/server/notification_settings_test.go b/internal/server/notification_settings_test.go index bd4ac9fe..3762302b 100644 --- a/internal/server/notification_settings_test.go +++ b/internal/server/notification_settings_test.go @@ -45,7 +45,7 @@ func notificationSettingsRoundTrip(t *testing.T, ctx context.Context, store *pip func wantSettings(t *testing.T, reply protocol.Message, want protocol.NotificationSettings) { t.Helper() - if reply.Type != protocol.NotificationSettingsReply || len(reply.NotificationSettings) != 3 { + if reply.Type != protocol.NotificationSettingsReply || len(reply.NotificationSettings) != len(protocol.NotificationSettingKeys) { t.Fatalf("reply = %+v", reply) } for key, value := range want.Map() { diff --git a/internal/server/notify.go b/internal/server/notify.go index c4f9a475..1306cb6e 100644 --- a/internal/server/notify.go +++ b/internal/server/notify.go @@ -67,13 +67,28 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N } return true } + // The app stream gets one notification per step event, recorded before + // the first ntfy publication; artifacts stay ntfy attachments. + appEvent, appKind := stageNotification(event, step.Target) + recorded := false + ntfyOn := ntfyEnabled(ctx, r.settingsStore()) send := func(filename string, data []byte) { receipts = append(receipts, protocol.NotificationReceipt{Destination: "system", Status: "sending", Filename: filename}) if !persist() { return } receipt := &receipts[len(receipts)-1] - result, err := stage.PublishNtfy(ctx, "system", title, summary, filename, data, runAttentionClick(id), runAttentionViews(id)...) + if !recorded { + recorded = true + r.notes.record(ctx, runNotificationRecord(id, step.Sequence, appEvent, appKind, title, summary)) + } + if !ntfyOn { + receipt.Status = "sent" + receipt.Diagnostic = "ntfy disabled; delivered to app clients only" + persist() + return + } + result, err := ntfyPublish(ctx, "system", title, summary, filename, data, runAttentionClick(id), runAttentionViews(id)...) if err != nil { receipt.Status = "failed" receipt.Diagnostic = err.Error() @@ -165,6 +180,26 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N } } +// ntfyPublish is the run notification publisher; tests may replace it. +var ntfyPublish = stage.PublishNtfy + +// stageNotification maps a stage hook event to the app notification event +// (the id suffix, shared with the run outbox for the same run event so the +// two never both appear) and kind. +func stageNotification(event, target string) (string, string) { + switch { + case event == protocol.NotifyWaiting: + return protocol.NotifyWaiting, protocol.NotificationRunWaiting + case event == "failure" && target == protocol.FailureEnd: + return "failed", protocol.NotificationRunFailed + case event == "failure": + return "failed", protocol.NotificationStageFailed + case target == protocol.SuccessEnd: + return "succeeded", protocol.NotificationRunSucceeded + } + return "success", protocol.NotificationStageSucceeded +} + // Terminal hooks run only after FinishRun commits. They arbitrate against the // durable intent just like waiting hooks, never publishing a proposed success. func (r *runner) notifyTerminalStep(id int64, status string) { @@ -412,7 +447,12 @@ func (r *runner) deliverRunNotificationBatch(now time.Time, limit int) { // and the web origin is server configuration that can change. n.Click = runAttentionClick(n.RunID) ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second) - result, publishErr := stage.PublishNtfy(ctx, n.Channel, n.Title, n.Body, "", nil, n.Click, runAttentionViews(n.RunID)...) + r.notes.record(ctx, runNotificationRecord(n.RunID, n.Sequence, n.Event, runNotificationKind(n.Event), n.Title, n.Body)) + var result stage.Values + var publishErr error + if ntfyEnabled(ctx, r.settingsStore()) { + result, publishErr = ntfyPublish(ctx, n.Channel, n.Title, n.Body, "", nil, n.Click, runAttentionViews(n.RunID)...) + } cancel() messageID, _ := result["message_id"].(string) save, done := context.WithTimeout(context.Background(), 5*time.Second) diff --git a/internal/server/run_delete.go b/internal/server/run_delete.go index 791d1170..d2f80076 100644 --- a/internal/server/run_delete.go +++ b/internal/server/run_delete.go @@ -5,6 +5,7 @@ import ( "log" "time" + "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" ) @@ -123,6 +124,12 @@ func (r *runner) deleteRuns(ids []int64) (*protocol.RunsDeletedResult, error) { r.notifyMu.Lock() delete(r.notifyModes, id) r.notifyMu.Unlock() + // A deleted run's notifications are gone on every device. + if r.notes != nil { + c, cancel := context.WithTimeout(r.ctx, 5*time.Second) + r.notes.clearFor(c, pipeline.AppNotificationFilter{RunID: id}) + cancel() + } } if len(result.Deleted) > 0 { log.Printf("deleted runs %v (refused %d, freed %d bytes)", result.Deleted, len(result.Refused), result.FreedBytes) diff --git a/internal/server/runner.go b/internal/server/runner.go index 77880952..2b9c13eb 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -53,6 +53,9 @@ type runner struct { // events pushes run state deltas to subscribed clients. When nil (tests that // build a runner directly) emits are dropped, so nothing else changes. events *runEventHub + // notes records and streams app notifications beside every ntfy + // publication. Nil (a runner built directly in a test) records nothing. + notes *appNotifier // identMu guards idents: the pipeline identity and stored title (see // runIdentity) that every emitted frame of a run carries, so a client with no // cached row renders a brand-new run without re-reading the list. Populated @@ -1335,6 +1338,13 @@ func (r *runner) stampRunIdentity(frame *protocol.RunEventFrame) { // and does nothing otherwise (a runner built directly in a test has no hub). // terminal marks a final transition, which the hub flushes without coalescing. func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) { + // A run that is no longer waiting has no decision pending: its waiting + // notifications are cleared on every device. + if r.notes != nil && frame.Status != "" && frame.Status != protocol.RunStatusWaiting { + c, cancel := context.WithTimeout(context.Background(), 5*time.Second) + r.notes.clearFor(c, pipeline.AppNotificationFilter{RunID: frame.RunID, Kinds: []string{protocol.NotificationRunWaiting}}) + cancel() + } if r.events == nil { return } diff --git a/internal/server/websocket.go b/internal/server/websocket.go index ed10cee5..a8194cfb 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -120,8 +120,14 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin hub.settings = settingsStore acks := newAttentionAckHub() var ackStore attentionAckStore + var notes *appNotifier if len(stores) > 0 { ackStore = stores[0] + notes = newAppNotifier(stores[0]) + hub.notes = notes + if runs != nil { + runs.notes = notes + } } mux.HandleFunc("GET /artifacts/{run}/{step}/{name...}", func(w http.ResponseWriter, r *http.Request) { if runs == nil { @@ -194,6 +200,9 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin // the welcome frame so a push never precedes it. acks.attach(clientID, write) defer acks.detach(clientID) + if notes != nil { + defer notes.detach(clientID) + } for { var request protocol.Message if err := wsjson.Read(connCtx, conn, &request); err != nil { @@ -210,7 +219,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin if handleNotificationSettings(ctx, request, settingsStore, write) { continue } - if handleAttentionAcks(ctx, request, ackStore, acks, clientID, write) { + if handleAttentionAcks(ctx, request, ackStore, acks, notes, clientID, write) { + continue + } + if handleAppNotifications(ctx, request, notes, ackStore, acks, clientID, write) { continue } if handleRunEvents(request, events, runs, clientID, write) {