package server import ( "context" "fmt" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" "io" "log" "os" "path/filepath" "strings" "syscall" "time" "unicode/utf8" ) func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.NotificationPolicy, step protocol.RunStep) { if r.notifyMode(id) == "off" || policy == nil { return } event := "success" if step.Status == protocol.StepStatusWaiting || step.Outcome == protocol.StepStatusWaiting { event = protocol.NotifyWaiting } else if step.Status != "completed" || step.Outcome == "error" || step.Target == protocol.FailureEnd { event = "failure" } enabled := false for _, on := range policy.On { if on == event { enabled = true } } if !enabled { return } var lease *pipeline.RunNotification runEvent := stageAttentionEvent(event) if event == "success" && step.Target == protocol.SuccessEnd { runEvent = "succeeded" } if event == protocol.NotifyWaiting || step.Target == protocol.SuccessEnd || step.Target == protocol.FailureEnd { var allowed bool var err error lease, allowed, err = r.store.ClaimStageRunNotification(r.ctx, id, step.Sequence, runEvent, time.Now().Unix()) if err != nil { log.Printf("claim stage notification run=%d: %v", id, err) return } if !allowed { return } } ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second) defer cancel() summary := stageNotificationSummary(name, id, step.Stage, event, step.Outcome) summary = truncateUTF8(summary, 1800) title := fmt.Sprintf("Pipeline stage %s", event) receipts := []protocol.NotificationReceipt{} persist := func() bool { save, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := r.store.SaveNotifications(save, id, step.Sequence, receipts); err != nil { log.Printf("notification audit run=%d step=%d: %v", id, step.Sequence, err) return false } return true } // The app stream gets one notification per step event, recorded before // the first ntfy publication; artifacts stay ntfy attachments. appEvent, appKind := stageNotification(event, step.Target) recorded := false ntfyOn := ntfyEnabled(ctx, r.settingsStore()) send := func(filename string, data []byte) { receipts = append(receipts, protocol.NotificationReceipt{Destination: "system", Status: "sending", Filename: filename}) if !persist() { return } receipt := &receipts[len(receipts)-1] if !recorded { recorded = true r.notes.record(ctx, runNotificationRecord(id, step.Sequence, appEvent, appKind, title, summary)) } if !ntfyOn { receipt.Status = "sent" receipt.Diagnostic = "ntfy disabled; delivered to app clients only" persist() return } result, err := ntfyPublish(ctx, "system", title, summary, filename, data, runAttentionClick(id), runAttentionViews(id)...) if err != nil { receipt.Status = "failed" receipt.Diagnostic = err.Error() } else { receipt.Status = "sent" receipt.MessageID, _ = result["message_id"].(string) receipt.AttachmentURL, _ = result["attachment_url"].(string) } persist() } reject := func(filename, diagnostic string) { receipts = append(receipts, protocol.NotificationReceipt{Destination: "system", Status: "failed", Filename: filename, Diagnostic: diagnostic}) persist() } sentArtifact := false for _, kind := range policy.Artifacts { switch kind { case "markdown": if markdown, ok := step.Output["markdown"].(string); ok && strings.TrimSpace(markdown) != "" { sentArtifact = true send("digest.md", []byte(markdown)) } case "files": raw, exists := step.Output["artifacts"] if !exists { continue } var names []string switch files := raw.(type) { case []string: names = files case []any: for _, value := range files { name, ok := value.(string) if !ok { reject("", "output.artifacts must be an array of relative file paths") names = nil break } names = append(names, name) } default: reject("", "output.artifacts must be an array of relative file paths") } if len(names) > 10 { reject("", "at most 10 file artifacts can be notified") continue } seen := map[string]bool{} for _, name := range names { if seen[name] { continue } seen[name] = true data, err := notificationArtifact(workspace, name) if err != nil { reject(name, err.Error()) continue } sentArtifact = true send(filepath.Base(name), data) } } } // No produced artifacts still merits an event, especially on failures. if !sentArtifact { send("", nil) } // Only a successful publish covers the run alert. A failed stage publisher // must not swallow the durable run-level fallback and its retries. for _, receipt := range receipts { if receipt.Status == "sent" { covered := stageAttentionEvent(event) if event == "success" && step.Target == protocol.SuccessEnd { covered = "succeeded" } r.markAttentionSent(id, step.Sequence, covered) return } } if lease != nil { diagnostic := "stage notification publication failed" if len(receipts) > 0 && receipts[0].Diagnostic != "" { diagnostic = receipts[0].Diagnostic } if err := r.store.FinishRunNotification(r.ctx, *lease, time.Now().Unix(), "", fmt.Errorf("%s", diagnostic)); err != nil { log.Printf("release stage notification run=%d: %v", id, err) } } } // ntfyPublish is the run notification publisher; tests may replace it. var ntfyPublish = stage.PublishNtfy // stageNotification maps a stage hook event to the app notification event // (the id suffix, shared with the run outbox for the same run event so the // two never both appear) and kind. func stageNotification(event, target string) (string, string) { switch { case event == protocol.NotifyWaiting: return protocol.NotifyWaiting, protocol.NotificationRunWaiting case event == "failure" && target == protocol.FailureEnd: return "failed", protocol.NotificationRunFailed case event == "failure": return "failed", protocol.NotificationStageFailed case target == protocol.SuccessEnd: return "succeeded", protocol.NotificationRunSucceeded } return "success", protocol.NotificationStageSucceeded } // Terminal hooks run only after FinishRun commits. They arbitrate against the // durable intent just like waiting hooks, never publishing a proposed success. func (r *runner) notifyTerminalStep(id int64, status string) { detail, err := r.store.GetRun(r.ctx, id) if err != nil || len(detail.Steps) == 0 { return } step := detail.Steps[len(detail.Steps)-1] stepStatus, _, _, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic) if !terminal || stepStatus != status { return } for _, definition := range detail.Graph.Stages { if definition.Key == step.Stage { r.notifyStep(id, detail.Name, detail.Workspace, definition.Notify, step) return } } } // runAttentionEvent maps a run status to the operator attention notification it // deserves. Only a run parked on a person (`waiting`) or a run that failed needs // the operator; every other status — queued, running, succeeded, canceled, // interrupted — returns "" so the mapping is the single place that decides // whether a run produces a notification. func runAttentionEvent(status string) string { switch strings.TrimSpace(strings.ToLower(status)) { case protocol.RunStatusWaiting: return protocol.NotifyWaiting case "failed": return "failed" } return "" } // stageAttentionEvent maps a stage's notification event to the run-attention // event it covers, or "" when the stage event is not a run-attention event (a // success digest, for example). It is the one place the two vocabularies meet. func stageAttentionEvent(event string) string { switch event { case protocol.NotifyWaiting: return protocol.NotifyWaiting case "failure": return "failed" } return "" } // markAttentionSent records that [event] has already been delivered for step // [sequence] of run [id] (by a stage policy or by the run-level default), so the // same genuine event is never notified twice. The step sequence is part of the // key because one run can wait on a person more than once; each wait is its own // event. func (r *runner) markAttentionSent(id int64, sequence int, event string) { if event == "" { return } if r.store != nil { if err := r.store.CoverRunNotification(r.ctx, id, sequence, event); err != nil { log.Printf("record stage notification coverage run=%d: %v", id, err) } } r.attentionMu.Lock() defer r.attentionMu.Unlock() if r.attentionSent == nil { r.attentionSent = map[int64]map[string]bool{} } if r.attentionSent[id] == nil { r.attentionSent[id] = map[string]bool{} } r.attentionSent[id][fmt.Sprintf("%d:%s", sequence, event)] = true } // attentionAlready reports whether [event] was already delivered for step // [sequence] of run [id]. func (r *runner) attentionAlready(id int64, sequence int, event string) bool { r.attentionMu.Lock() defer r.attentionMu.Unlock() return r.attentionSent[id][fmt.Sprintf("%d:%s", sequence, event)] } func (r *runner) setNotifyMode(id int64, mode string) { mode = strings.ToLower(strings.TrimSpace(mode)) if mode != "on" && mode != "off" { mode = "auto" } r.notifyMu.Lock() if r.notifyModes == nil { r.notifyModes = map[int64]string{} } r.notifyModes[id] = mode r.notifyMu.Unlock() } func (r *runner) notifyMode(id int64) string { r.notifyMu.Lock() mode := r.notifyModes[id] r.notifyMu.Unlock() if mode != "" { return mode } if r.store != nil { if stored, err := r.store.RunNotifyMode(r.ctx, id); err == nil && stored != "" { r.notifyMu.Lock() if r.notifyModes == nil { r.notifyModes = map[int64]string{} } r.notifyModes[id] = stored r.notifyMu.Unlock() return stored } } return "auto" } func (r *runner) forgetNotifyMode(id int64) { r.notifyMu.Lock() delete(r.notifyModes, id) r.notifyMu.Unlock() } // forgetAttention drops a run's delivered-event set once it is terminal, so the // registry is bounded by the runs still in flight. func (r *runner) forgetAttention(id int64) { r.attentionMu.Lock() defer r.attentionMu.Unlock() delete(r.attentionSent, id) } // runAttentionClick is the notification tap target for a run: always the native // app deep link, so a tap on the phone opens the Pipelines app. func runAttentionClick(id int64) string { return fmt.Sprintf("pipelines://run/%d", id) } // runAttentionViews is the optional "Open in browser" action for a run. func runAttentionViews(id int64) []stage.NtfyView { return webAttentionViews(fmt.Sprintf("/runs/%d", id)) } // webAttentionViews returns an "Open in browser" ntfy action for path on the // operator's authenticated web origin, or nothing when none is configured. // Never infer a public address from a request Host, the unauthenticated API // listener or the ntfy server URL. func webAttentionViews(path string) []stage.NtfyView { raw := os.Getenv("PIPELINES_WEB_PUBLIC_URL") if raw == "" { return nil } base := publicNtfyBase(raw) if base == "" { log.Printf("notification web link unavailable: PIPELINES_WEB_PUBLIC_URL must be an HTTP(S) origin without credentials, query or fragment") return nil } return []stage.NtfyView{{Label: "Open in browser", URL: base + path}} } // notifyRun publishes one run-level attention notification to the operator's // already-configured "system" destination. It is the run counterpart of // notifyStep and reuses the same PublishNtfy machinery, destination and token // handling, so no second publisher, topic or service is introduced. The run id // and pipeline name identify the run and the click action opens it in the app. // Attempts are persisted before publication. Publisher acknowledgement is not // proof of device reception; a crash in that acknowledgement window can repeat. func (r *runner) notifyRun(id int64, name, status string, sequence int, currentStage, diagnostic string) { if r.notifyMode(id) == "off" { return } event := runAttentionEvent(status) if status == "succeeded" { event = "succeeded" } if event == "" { return } // A run's end notifies only runs marked "on", unless the operator opted // into every run end (run_finished); waiting notifies any non-off run. if event != protocol.NotifyWaiting && r.notifyMode(id) != "on" { settings, err := r.store.NotificationSettings(r.ctx) if err != nil || !settings.RunFinished { return } } if r.attentionAlready(id, sequence, event) { return } title := "Run needs your decision" summary := fmt.Sprintf("Run #%d · %s · waiting for a human decision", id, name) if event == "failed" { title = "Run failed" summary = fmt.Sprintf("Run #%d · %s · run failed", id, name) } if event == "succeeded" { title = "Run succeeded" summary = fmt.Sprintf("Run #%d · %s · run succeeded", id, name) } if currentStage != "" { summary += " at stage " + currentStage } if diagnostic != "" { summary += ": " + diagnostic } summary = truncateUTF8(summary, 2000) if err := r.store.EnqueueRunNotification(r.ctx, pipeline.RunNotification{RunID: id, Sequence: sequence, Event: event, Channel: "system", Title: title, Body: summary, Click: runAttentionClick(id)}); err != nil { log.Printf("enqueue run notification run=%d: %v", id, err) return } r.deliverRunNotificationBatch(time.Now(), 1) } // resumeRunNotifications also recovers pending/expired leases after restart. func (r *runner) resumeRunNotifications() { ticker := time.NewTicker(time.Second) defer ticker.Stop() for { select { case <-r.ctx.Done(): return case <-r.store.Done(): return case <-ticker.C: r.deliverRunNotifications(time.Now()) } } } func (r *runner) deliverRunNotifications(now time.Time) { r.deliverRunNotificationBatch(now, 32) } func (r *runner) deliverRunNotificationBatch(now time.Time, limit int) { // Bound one drain so a queue cannot monopolize the runner. started := time.Now() for i := 0; i < limit; i++ { n, err := r.store.ClaimRunNotification(r.ctx, now.Add(time.Since(started)).Unix()) if err != nil { log.Printf("claim run notification: %v", err) return } if n == nil { return } // Links are derived from the run id at delivery, not from the stored // click: rows queued by older servers may hold a web URL (or nothing), // and the web origin is server configuration that can change. n.Click = runAttentionClick(n.RunID) ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second) r.notes.record(ctx, runNotificationRecord(n.RunID, n.Sequence, n.Event, runNotificationKind(n.Event), n.Title, n.Body)) var result stage.Values var publishErr error if ntfyEnabled(ctx, r.settingsStore()) { result, publishErr = ntfyPublish(ctx, n.Channel, n.Title, n.Body, "", nil, n.Click, runAttentionViews(n.RunID)...) } cancel() messageID, _ := result["message_id"].(string) save, done := context.WithTimeout(context.Background(), 5*time.Second) err = r.store.FinishRunNotification(save, *n, now.Add(time.Since(started)).Unix(), messageID, publishErr) done() if err != nil { log.Printf("save run notification run=%d: %v", n.RunID, err) } } } // truncateUTF8 bounds a notification string without splitting a multi-byte rune. func truncateUTF8(text string, max int) string { if len(text) <= max { return text } text = text[:max] for !utf8.ValidString(text) { text = text[:len(text)-1] } return text } func notificationArtifact(workspace, name string) ([]byte, error) { if name == "" || filepath.IsAbs(name) || filepath.Clean(name) != name || strings.ContainsAny(name, "\\\x00\r\n") { return nil, fmt.Errorf("artifact must be a clean workspace-relative file path") } for _, part := range strings.Split(filepath.ToSlash(name), "/") { lower := strings.ToLower(part) if strings.HasPrefix(part, ".") || lower == "secrets" || lower == "node_modules" || strings.HasSuffix(lower, ".pem") || strings.HasSuffix(lower, ".key") || strings.HasSuffix(lower, ".db") || strings.HasSuffix(lower, ".db-wal") || strings.HasSuffix(lower, ".db-shm") || strings.HasSuffix(lower, ".sqlite") || strings.HasSuffix(lower, ".sqlite3") { return nil, fmt.Errorf("artifact path is protected") } } root, err := filepath.EvalSymlinks(workspace) if err != nil { return nil, fmt.Errorf("artifact workspace unavailable") } // os.Root prevents symlink traversal outside the workspace; disallow symlinks // altogether so aliases cannot bypass protected path names within it either. current := root for _, part := range strings.Split(name, string(os.PathSeparator)) { current = filepath.Join(current, part) info, err := os.Lstat(current) if err != nil || info.Mode()&os.ModeSymlink != 0 { return nil, fmt.Errorf("artifact is missing or uses a symlink") } } scoped, err := os.OpenRoot(root) if err != nil { return nil, fmt.Errorf("artifact workspace unavailable") } defer scoped.Close() file, err := scoped.OpenFile(name, os.O_RDONLY|syscall.O_NONBLOCK|syscall.O_NOFOLLOW, 0) if err != nil { return nil, fmt.Errorf("cannot read artifact") } defer file.Close() info, err := file.Stat() if err != nil || !info.Mode().IsRegular() || info.Size() > 1<<20 { return nil, fmt.Errorf("artifact must be a regular file at most 1 MiB") } data, err := io.ReadAll(io.LimitReader(file, (1<<20)+1)) if err != nil || len(data) > 1<<20 { return nil, fmt.Errorf("cannot read artifact within size limit") } return data, nil } // stageNotificationSummary is a stage notification's one-line body. The // outcome is shown only when the stage recorded one (a waiting human stage has // none yet), and a waiting stage says what it waits for. func stageNotificationSummary(name string, runID int64, stage, event, outcome string) string { state := event if event == "waiting" { state = "waiting for your decision" } if outcome = strings.TrimSpace(outcome); outcome != "" { state += " (" + outcome + ")" } return fmt.Sprintf("%s · run #%d · stage %s · %s", name, runID, stage, state) }