d1846441cc
Record every notification published to ntfy (run outbox, stage notify hooks, chat question/approval/reply) in a persisted app_notifications outbox and stream it to subscribe_notifications clients with an after cursor; clear_notifications and ack_attention clear across devices, as do runs leaving waiting, deleted runs and answered chat questions. Add the notify.ntfy_enabled setting (default on) to stop only the ntfy publish.
324 lines
11 KiB
Go
324 lines
11 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
// appNotificationStore persists the app notification outbox.
|
|
// *pipeline.Store implements it.
|
|
type appNotificationStore interface {
|
|
RecordAppNotification(context.Context, pipeline.AppNotificationRecord, time.Time) (protocol.AppNotification, bool, error)
|
|
ActiveAppNotifications(context.Context, string, time.Time) ([]protocol.AppNotification, []string, error)
|
|
ClearAppNotifications(context.Context, []string, time.Time) ([]pipeline.AppNotificationRecord, error)
|
|
ClearAppNotificationsFor(context.Context, pipeline.AppNotificationFilter, time.Time) ([]string, error)
|
|
}
|
|
|
|
// appNotificationQueue bounds one subscriber's undelivered frames. A client
|
|
// that falls this far behind is dropped; it recovers with a resubscribe.
|
|
const appNotificationQueue = 512
|
|
|
|
// appNotifier records every operator notification (the same ones published to
|
|
// ntfy) and streams them to subscribed connections. One mutex orders store
|
|
// writes with fan-out, so each subscriber sees its snapshot reply and then
|
|
// every later push exactly in commit order.
|
|
type appNotifier struct {
|
|
store appNotificationStore
|
|
now func() time.Time
|
|
mu sync.Mutex
|
|
subs map[string]*appNotificationSub
|
|
}
|
|
|
|
type appNotificationSub struct {
|
|
queue chan protocol.Message
|
|
done chan struct{}
|
|
once sync.Once
|
|
}
|
|
|
|
func newAppNotifier(store appNotificationStore) *appNotifier {
|
|
return &appNotifier{store: store, now: time.Now, subs: map[string]*appNotificationSub{}}
|
|
}
|
|
|
|
func (s *appNotificationSub) stop() { s.once.Do(func() { close(s.done) }) }
|
|
|
|
func (s *appNotificationSub) run(write func(protocol.Message) error) {
|
|
for {
|
|
select {
|
|
case <-s.done:
|
|
return
|
|
case msg := <-s.queue:
|
|
if write(msg) != nil {
|
|
s.stop()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// enqueueLocked queues msg for every subscriber except exclude. Callers hold
|
|
// a.mu.
|
|
func (a *appNotifier) enqueueLocked(msg protocol.Message, exclude string) {
|
|
for id, sub := range a.subs {
|
|
if id == exclude {
|
|
continue
|
|
}
|
|
select {
|
|
case sub.queue <- msg:
|
|
case <-sub.done:
|
|
delete(a.subs, id)
|
|
default:
|
|
log.Printf("app notifications: dropping slow subscriber %s", id)
|
|
sub.stop()
|
|
delete(a.subs, id)
|
|
}
|
|
}
|
|
}
|
|
|
|
// subscribe replaces clientID's subscription with one that first delivers the
|
|
// replay reply and then every later push.
|
|
func (a *appNotifier) subscribe(ctx context.Context, clientID, requestID, after string, write func(protocol.Message) error) error {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
active, cleared, err := a.store.ActiveAppNotifications(ctx, after, a.now())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if old := a.subs[clientID]; old != nil {
|
|
old.stop()
|
|
}
|
|
sub := &appNotificationSub{queue: make(chan protocol.Message, appNotificationQueue), done: make(chan struct{})}
|
|
sub.queue <- protocol.Message{Type: protocol.NotificationsSubscribed, ID: requestID, Notifications: active, IDs: cleared}
|
|
a.subs[clientID] = sub
|
|
go sub.run(write)
|
|
return nil
|
|
}
|
|
|
|
func (a *appNotifier) detach(clientID string) {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
if sub := a.subs[clientID]; sub != nil {
|
|
sub.stop()
|
|
delete(a.subs, clientID)
|
|
}
|
|
}
|
|
|
|
// record persists n and pushes it when it is new. Failures are logged only:
|
|
// recording must never block or fail the ntfy publication beside it.
|
|
func (a *appNotifier) record(ctx context.Context, n pipeline.AppNotificationRecord) {
|
|
if a == nil {
|
|
return
|
|
}
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
stored, fresh, err := a.store.RecordAppNotification(ctx, n, a.now())
|
|
if err != nil {
|
|
log.Printf("record app notification %s: %v", n.ID, err)
|
|
return
|
|
}
|
|
if fresh {
|
|
a.enqueueLocked(protocol.Message{Type: protocol.NotificationPush, Notification: &stored}, "")
|
|
}
|
|
}
|
|
|
|
// clear clears ids and pushes the ones that changed to every subscriber but
|
|
// exclude (the requester, which receives them in its reply).
|
|
func (a *appNotifier) clear(ctx context.Context, ids []string, exclude string) ([]pipeline.AppNotificationRecord, error) {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
changed, err := a.store.ClearAppNotifications(ctx, ids, a.now())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(changed) > 0 {
|
|
a.enqueueLocked(protocol.Message{Type: protocol.NotificationsCleared, IDs: recordIDs(changed)}, exclude)
|
|
}
|
|
return changed, nil
|
|
}
|
|
|
|
// clearFor clears every active notification matching filter and pushes the
|
|
// change to every subscriber.
|
|
func (a *appNotifier) clearFor(ctx context.Context, filter pipeline.AppNotificationFilter) {
|
|
if a == nil {
|
|
return
|
|
}
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
ids, err := a.store.ClearAppNotificationsFor(ctx, filter, a.now())
|
|
if err != nil {
|
|
log.Printf("clear app notifications: %v", err)
|
|
return
|
|
}
|
|
if len(ids) > 0 {
|
|
a.enqueueLocked(protocol.Message{Type: protocol.NotificationsCleared, IDs: ids}, "")
|
|
}
|
|
}
|
|
|
|
// clearForAcks clears the notifications an attention acknowledgement covers:
|
|
// "<run>:Decision" the run's waiting notifications, "<run>:Failed" its failure
|
|
// notifications, and a chat_seen watermark every notification of that chat.
|
|
func (a *appNotifier) clearForAcks(ctx context.Context, ack protocol.AttentionAcks) {
|
|
if a == nil {
|
|
return
|
|
}
|
|
for _, key := range ack.Dismissed {
|
|
run, kind, ok := strings.Cut(key, ":")
|
|
id, err := strconv.ParseInt(run, 10, 64)
|
|
if !ok || err != nil {
|
|
continue
|
|
}
|
|
kinds := []string{protocol.NotificationRunWaiting}
|
|
if kind == "Failed" {
|
|
kinds = []string{protocol.NotificationRunFailed, protocol.NotificationStageFailed}
|
|
}
|
|
a.clearFor(ctx, pipeline.AppNotificationFilter{RunID: id, Kinds: kinds})
|
|
}
|
|
for session := range ack.ChatSeen {
|
|
a.clearFor(ctx, pipeline.AppNotificationFilter{SessionID: session})
|
|
}
|
|
}
|
|
|
|
func recordIDs(records []pipeline.AppNotificationRecord) []string {
|
|
ids := make([]string, 0, len(records))
|
|
for _, r := range records {
|
|
ids = append(ids, r.ID)
|
|
}
|
|
return ids
|
|
}
|
|
|
|
// acksForCleared maps notifications cleared in the app to the equivalent
|
|
// attention acknowledgement, so a dismissal in the notification shade has the
|
|
// same effect as one in the Needs-you list.
|
|
func acksForCleared(records []pipeline.AppNotificationRecord) protocol.AttentionAcks {
|
|
ack := protocol.AttentionAcks{Dismissed: []string{}, ChatSeen: map[string]int64{}}
|
|
for _, r := range records {
|
|
switch {
|
|
case r.RunID > 0 && r.Kind == protocol.NotificationRunWaiting:
|
|
ack.Dismissed = append(ack.Dismissed, fmt.Sprintf("%d:Decision", r.RunID))
|
|
case r.RunID > 0 && (r.Kind == protocol.NotificationRunFailed || r.Kind == protocol.NotificationStageFailed):
|
|
ack.Dismissed = append(ack.Dismissed, fmt.Sprintf("%d:Failed", r.RunID))
|
|
case r.SessionID != "":
|
|
// The store raises a watermark to the session's own recency, so 1
|
|
// acknowledges everything the chat holds now.
|
|
ack.ChatSeen[r.SessionID] = 1
|
|
}
|
|
}
|
|
return ack
|
|
}
|
|
|
|
// handleAppNotifications answers subscribe_notifications and
|
|
// clear_notifications. It reports whether request was one of them.
|
|
func handleAppNotifications(ctx context.Context, request protocol.Message, notes *appNotifier, ackStore attentionAckStore, acks *attentionAckHub, clientID string, write func(protocol.Message) error) bool {
|
|
if request.Type != protocol.SubscribeNotifications && request.Type != protocol.ClearNotifications {
|
|
return false
|
|
}
|
|
if notes == nil {
|
|
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "notifications are unavailable: no store"})
|
|
return true
|
|
}
|
|
if request.Type == protocol.SubscribeNotifications {
|
|
if err := notes.subscribe(ctx, clientID, request.ID, request.After, write); err != nil {
|
|
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
|
|
}
|
|
return true
|
|
}
|
|
ids, err := protocol.ValidateClearNotifications(request.IDs)
|
|
if err != nil {
|
|
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
|
|
return true
|
|
}
|
|
changed, err := notes.clear(ctx, ids, clientID)
|
|
if err != nil {
|
|
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
|
|
return true
|
|
}
|
|
_ = write(protocol.Message{Type: protocol.NotificationsCleared, ID: request.ID, IDs: recordIDs(changed)})
|
|
ack := acksForCleared(changed)
|
|
if ack.Empty() {
|
|
return true
|
|
}
|
|
if ackStore != nil {
|
|
if valid, err := protocol.ValidateAttentionAck(&ack); err == nil {
|
|
delta, _, err := ackStore.AckAttention(ctx, &valid, time.Now())
|
|
if err != nil {
|
|
log.Printf("ack cleared notifications: %v", err)
|
|
} else if acks != nil && !delta.Empty() {
|
|
// The requester's Needs-you list changes too, so it is included.
|
|
acks.broadcast("", delta)
|
|
}
|
|
}
|
|
}
|
|
// The equivalent ack also covers sibling notifications (the chat's other
|
|
// notifications, the run's earlier waits); push those to everyone.
|
|
notes.clearForAcks(ctx, ack)
|
|
return true
|
|
}
|
|
|
|
// ntfyEnabled reads notify.ntfy_enabled. Without a store, or when the read
|
|
// fails, ntfy stays enabled (the default) so a transient error never silences
|
|
// the operator.
|
|
func ntfyEnabled(ctx context.Context, store notificationSettingsStore) bool {
|
|
if store == nil {
|
|
return true
|
|
}
|
|
settings, err := store.NotificationSettings(ctx)
|
|
if err != nil {
|
|
log.Printf("read notification settings: %v", err)
|
|
return true
|
|
}
|
|
return settings.NtfyEnabled
|
|
}
|
|
|
|
// ntfyDisabledResult is what a publication reports when ntfy is turned off:
|
|
// the notification reached app clients only.
|
|
func ntfyDisabledResult() stage.Values {
|
|
return stage.Values{"destination": "system", "message_id": "", "ntfy": "disabled"}
|
|
}
|
|
|
|
// runNotificationKind maps a run outbox event to its app notification kind.
|
|
func runNotificationKind(event string) string {
|
|
switch event {
|
|
case protocol.NotifyWaiting:
|
|
return protocol.NotificationRunWaiting
|
|
case "failed":
|
|
return protocol.NotificationRunFailed
|
|
case "succeeded":
|
|
return protocol.NotificationRunSucceeded
|
|
}
|
|
return "run_" + event
|
|
}
|
|
|
|
// runNotificationRecord builds the app notification for one run event:
|
|
// identity "run:<run>:<step>:<event>", deep link and group from the run id.
|
|
func runNotificationRecord(runID int64, sequence int, event, kind, title, message string) pipeline.AppNotificationRecord {
|
|
return pipeline.AppNotificationRecord{
|
|
AppNotification: protocol.AppNotification{
|
|
ID: fmt.Sprintf("run:%d:%d:%s", runID, sequence, event),
|
|
Kind: kind,
|
|
Title: title,
|
|
Message: message,
|
|
Click: runAttentionClick(runID),
|
|
Priority: protocol.DefaultNotificationPriority,
|
|
Tags: []string{},
|
|
Group: fmt.Sprintf("run:%d", runID),
|
|
},
|
|
RunID: runID,
|
|
}
|
|
}
|
|
|
|
// settingsStore returns the runner's store as a settings reader, or nil.
|
|
func (r *runner) settingsStore() notificationSettingsStore {
|
|
if r.store == nil {
|
|
return nil
|
|
}
|
|
return r.store
|
|
}
|