40ba2ac117
Run notifications now always click through to pipelines://run/<id>; a valid PIPELINES_WEB_PUBLIC_URL adds an "Open in browser" ntfy view action instead of replacing the click. Queued rows derive links from the run id at delivery, so legacy rows holding a web URL still open the app. Unresolved chat questions/forms and permission requests publish one async, bounded notification to the system destination (pipelines://chat/<id> plus the browser action), deduplicated per form/request id across reconnects and replays. stage.PublishNtfy gains optional, strictly validated view actions. Server tests now clear the operator's real ntfy environment.
475 lines
15 KiB
Go
475 lines
15 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.NotificationPolicy, step protocol.RunStep) {
|
|
if r.notifyMode(id) == "off" || policy == nil {
|
|
return
|
|
}
|
|
event := "success"
|
|
if step.Status == protocol.StepStatusWaiting || step.Outcome == protocol.StepStatusWaiting {
|
|
event = protocol.NotifyWaiting
|
|
} else if step.Status != "completed" || step.Outcome == "error" || step.Target == protocol.FailureEnd {
|
|
event = "failure"
|
|
}
|
|
enabled := false
|
|
for _, on := range policy.On {
|
|
if on == event {
|
|
enabled = true
|
|
}
|
|
}
|
|
if !enabled {
|
|
return
|
|
}
|
|
var lease *pipeline.RunNotification
|
|
runEvent := stageAttentionEvent(event)
|
|
if event == "success" && step.Target == protocol.SuccessEnd {
|
|
runEvent = "succeeded"
|
|
}
|
|
if event == protocol.NotifyWaiting || step.Target == protocol.SuccessEnd || step.Target == protocol.FailureEnd {
|
|
var allowed bool
|
|
var err error
|
|
lease, allowed, err = r.store.ClaimStageRunNotification(r.ctx, id, step.Sequence, runEvent, time.Now().Unix())
|
|
if err != nil {
|
|
log.Printf("claim stage notification run=%d: %v", id, err)
|
|
return
|
|
}
|
|
if !allowed {
|
|
return
|
|
}
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
|
|
defer cancel()
|
|
summary := fmt.Sprintf("%s · run #%d · stage %s · %s (%s)", name, id, step.Stage, event, step.Outcome)
|
|
summary = truncateUTF8(summary, 1800)
|
|
title := fmt.Sprintf("Pipeline stage %s", event)
|
|
receipts := []protocol.NotificationReceipt{}
|
|
persist := func() bool {
|
|
save, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
if err := r.store.SaveNotifications(save, id, step.Sequence, receipts); err != nil {
|
|
log.Printf("notification audit run=%d step=%d: %v", id, step.Sequence, err)
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
send := func(filename string, data []byte) {
|
|
receipts = append(receipts, protocol.NotificationReceipt{Destination: "system", Status: "sending", Filename: filename})
|
|
if !persist() {
|
|
return
|
|
}
|
|
receipt := &receipts[len(receipts)-1]
|
|
result, err := stage.PublishNtfy(ctx, "system", title, summary, filename, data, runAttentionClick(id), runAttentionViews(id)...)
|
|
if err != nil {
|
|
receipt.Status = "failed"
|
|
receipt.Diagnostic = err.Error()
|
|
} else {
|
|
receipt.Status = "sent"
|
|
receipt.MessageID, _ = result["message_id"].(string)
|
|
receipt.AttachmentURL, _ = result["attachment_url"].(string)
|
|
}
|
|
persist()
|
|
}
|
|
reject := func(filename, diagnostic string) {
|
|
receipts = append(receipts, protocol.NotificationReceipt{Destination: "system", Status: "failed", Filename: filename, Diagnostic: diagnostic})
|
|
persist()
|
|
}
|
|
sentArtifact := false
|
|
for _, kind := range policy.Artifacts {
|
|
switch kind {
|
|
case "markdown":
|
|
if markdown, ok := step.Output["markdown"].(string); ok && strings.TrimSpace(markdown) != "" {
|
|
sentArtifact = true
|
|
send("digest.md", []byte(markdown))
|
|
}
|
|
case "files":
|
|
raw, exists := step.Output["artifacts"]
|
|
if !exists {
|
|
continue
|
|
}
|
|
var names []string
|
|
switch files := raw.(type) {
|
|
case []string:
|
|
names = files
|
|
case []any:
|
|
for _, value := range files {
|
|
name, ok := value.(string)
|
|
if !ok {
|
|
reject("", "output.artifacts must be an array of relative file paths")
|
|
names = nil
|
|
break
|
|
}
|
|
names = append(names, name)
|
|
}
|
|
default:
|
|
reject("", "output.artifacts must be an array of relative file paths")
|
|
}
|
|
if len(names) > 10 {
|
|
reject("", "at most 10 file artifacts can be notified")
|
|
continue
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, name := range names {
|
|
if seen[name] {
|
|
continue
|
|
}
|
|
seen[name] = true
|
|
data, err := notificationArtifact(workspace, name)
|
|
if err != nil {
|
|
reject(name, err.Error())
|
|
continue
|
|
}
|
|
sentArtifact = true
|
|
send(filepath.Base(name), data)
|
|
}
|
|
}
|
|
}
|
|
// No produced artifacts still merits an event, especially on failures.
|
|
if !sentArtifact {
|
|
send("", nil)
|
|
}
|
|
// Only a successful publish covers the run alert. A failed stage publisher
|
|
// must not swallow the durable run-level fallback and its retries.
|
|
for _, receipt := range receipts {
|
|
if receipt.Status == "sent" {
|
|
covered := stageAttentionEvent(event)
|
|
if event == "success" && step.Target == protocol.SuccessEnd {
|
|
covered = "succeeded"
|
|
}
|
|
r.markAttentionSent(id, step.Sequence, covered)
|
|
return
|
|
}
|
|
}
|
|
if lease != nil {
|
|
diagnostic := "stage notification publication failed"
|
|
if len(receipts) > 0 && receipts[0].Diagnostic != "" {
|
|
diagnostic = receipts[0].Diagnostic
|
|
}
|
|
if err := r.store.FinishRunNotification(r.ctx, *lease, time.Now().Unix(), "", fmt.Errorf("%s", diagnostic)); err != nil {
|
|
log.Printf("release stage notification run=%d: %v", id, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Terminal hooks run only after FinishRun commits. They arbitrate against the
|
|
// durable intent just like waiting hooks, never publishing a proposed success.
|
|
func (r *runner) notifyTerminalStep(id int64, status string) {
|
|
detail, err := r.store.GetRun(r.ctx, id)
|
|
if err != nil || len(detail.Steps) == 0 {
|
|
return
|
|
}
|
|
step := detail.Steps[len(detail.Steps)-1]
|
|
stepStatus, _, _, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
|
|
if !terminal || stepStatus != status {
|
|
return
|
|
}
|
|
for _, definition := range detail.Graph.Stages {
|
|
if definition.Key == step.Stage {
|
|
r.notifyStep(id, detail.Name, detail.Workspace, definition.Notify, step)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// runAttentionEvent maps a run status to the operator attention notification it
|
|
// deserves. Only a run parked on a person (`waiting`) or a run that failed needs
|
|
// the operator; every other status — queued, running, succeeded, canceled,
|
|
// interrupted — returns "" so the mapping is the single place that decides
|
|
// whether a run produces a notification.
|
|
func runAttentionEvent(status string) string {
|
|
switch strings.TrimSpace(strings.ToLower(status)) {
|
|
case protocol.RunStatusWaiting:
|
|
return protocol.NotifyWaiting
|
|
case "failed":
|
|
return "failed"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// stageAttentionEvent maps a stage's notification event to the run-attention
|
|
// event it covers, or "" when the stage event is not a run-attention event (a
|
|
// success digest, for example). It is the one place the two vocabularies meet.
|
|
func stageAttentionEvent(event string) string {
|
|
switch event {
|
|
case protocol.NotifyWaiting:
|
|
return protocol.NotifyWaiting
|
|
case "failure":
|
|
return "failed"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// markAttentionSent records that [event] has already been delivered for step
|
|
// [sequence] of run [id] (by a stage policy or by the run-level default), so the
|
|
// same genuine event is never notified twice. The step sequence is part of the
|
|
// key because one run can wait on a person more than once; each wait is its own
|
|
// event.
|
|
func (r *runner) markAttentionSent(id int64, sequence int, event string) {
|
|
if event == "" {
|
|
return
|
|
}
|
|
if r.store != nil {
|
|
if err := r.store.CoverRunNotification(r.ctx, id, sequence, event); err != nil {
|
|
log.Printf("record stage notification coverage run=%d: %v", id, err)
|
|
}
|
|
}
|
|
r.attentionMu.Lock()
|
|
defer r.attentionMu.Unlock()
|
|
if r.attentionSent == nil {
|
|
r.attentionSent = map[int64]map[string]bool{}
|
|
}
|
|
if r.attentionSent[id] == nil {
|
|
r.attentionSent[id] = map[string]bool{}
|
|
}
|
|
r.attentionSent[id][fmt.Sprintf("%d:%s", sequence, event)] = true
|
|
}
|
|
|
|
// attentionAlready reports whether [event] was already delivered for step
|
|
// [sequence] of run [id].
|
|
func (r *runner) attentionAlready(id int64, sequence int, event string) bool {
|
|
r.attentionMu.Lock()
|
|
defer r.attentionMu.Unlock()
|
|
return r.attentionSent[id][fmt.Sprintf("%d:%s", sequence, event)]
|
|
}
|
|
|
|
func (r *runner) setNotifyMode(id int64, mode string) {
|
|
mode = strings.ToLower(strings.TrimSpace(mode))
|
|
if mode != "on" && mode != "off" {
|
|
mode = "auto"
|
|
}
|
|
r.notifyMu.Lock()
|
|
if r.notifyModes == nil {
|
|
r.notifyModes = map[int64]string{}
|
|
}
|
|
r.notifyModes[id] = mode
|
|
r.notifyMu.Unlock()
|
|
}
|
|
|
|
func (r *runner) notifyMode(id int64) string {
|
|
r.notifyMu.Lock()
|
|
mode := r.notifyModes[id]
|
|
r.notifyMu.Unlock()
|
|
if mode != "" {
|
|
return mode
|
|
}
|
|
if r.store != nil {
|
|
if stored, err := r.store.RunNotifyMode(r.ctx, id); err == nil && stored != "" {
|
|
r.notifyMu.Lock()
|
|
if r.notifyModes == nil {
|
|
r.notifyModes = map[int64]string{}
|
|
}
|
|
r.notifyModes[id] = stored
|
|
r.notifyMu.Unlock()
|
|
return stored
|
|
}
|
|
}
|
|
return "auto"
|
|
}
|
|
|
|
func (r *runner) forgetNotifyMode(id int64) {
|
|
r.notifyMu.Lock()
|
|
delete(r.notifyModes, id)
|
|
r.notifyMu.Unlock()
|
|
}
|
|
|
|
// forgetAttention drops a run's delivered-event set once it is terminal, so the
|
|
// registry is bounded by the runs still in flight.
|
|
func (r *runner) forgetAttention(id int64) {
|
|
r.attentionMu.Lock()
|
|
defer r.attentionMu.Unlock()
|
|
delete(r.attentionSent, id)
|
|
}
|
|
|
|
// runAttentionClick is the notification tap target for a run: always the native
|
|
// app deep link, so a tap on the phone opens the Pipelines app.
|
|
func runAttentionClick(id int64) string {
|
|
return fmt.Sprintf("pipelines://run/%d", id)
|
|
}
|
|
|
|
// runAttentionViews is the optional "Open in browser" action for a run.
|
|
func runAttentionViews(id int64) []stage.NtfyView {
|
|
return webAttentionViews(fmt.Sprintf("/runs/%d", id))
|
|
}
|
|
|
|
// webAttentionViews returns an "Open in browser" ntfy action for path on the
|
|
// operator's authenticated web origin, or nothing when none is configured.
|
|
// Never infer a public address from a request Host, the unauthenticated API
|
|
// listener or the ntfy server URL.
|
|
func webAttentionViews(path string) []stage.NtfyView {
|
|
raw := os.Getenv("PIPELINES_WEB_PUBLIC_URL")
|
|
if raw == "" {
|
|
return nil
|
|
}
|
|
base := publicNtfyBase(raw)
|
|
if base == "" {
|
|
log.Printf("notification web link unavailable: PIPELINES_WEB_PUBLIC_URL must be an HTTP(S) origin without credentials, query or fragment")
|
|
return nil
|
|
}
|
|
return []stage.NtfyView{{Label: "Open in browser", URL: base + path}}
|
|
}
|
|
|
|
// notifyRun publishes one run-level attention notification to the operator's
|
|
// already-configured "system" destination. It is the run counterpart of
|
|
// notifyStep and reuses the same PublishNtfy machinery, destination and token
|
|
// handling, so no second publisher, topic or service is introduced. The run id
|
|
// and pipeline name identify the run and the click action opens it in the app.
|
|
// Attempts are persisted before publication. Publisher acknowledgement is not
|
|
// proof of device reception; a crash in that acknowledgement window can repeat.
|
|
func (r *runner) notifyRun(id int64, name, status string, sequence int, currentStage, diagnostic string) {
|
|
if r.notifyMode(id) == "off" {
|
|
return
|
|
}
|
|
event := runAttentionEvent(status)
|
|
if status == "succeeded" && r.notifyMode(id) == "on" {
|
|
event = "succeeded"
|
|
}
|
|
if event == "" {
|
|
return
|
|
}
|
|
if r.attentionAlready(id, sequence, event) {
|
|
return
|
|
}
|
|
title := "Run needs your decision"
|
|
summary := fmt.Sprintf("Run #%d · %s · waiting for a human decision", id, name)
|
|
if event == "failed" {
|
|
title = "Run failed"
|
|
summary = fmt.Sprintf("Run #%d · %s · run failed", id, name)
|
|
}
|
|
if event == "succeeded" {
|
|
title = "Run succeeded"
|
|
summary = fmt.Sprintf("Run #%d · %s · run succeeded", id, name)
|
|
}
|
|
if currentStage != "" {
|
|
summary += " at stage " + currentStage
|
|
}
|
|
if diagnostic != "" {
|
|
summary += ": " + diagnostic
|
|
}
|
|
summary = truncateUTF8(summary, 2000)
|
|
if err := r.store.EnqueueRunNotification(r.ctx, pipeline.RunNotification{RunID: id, Sequence: sequence, Event: event, Channel: "system", Title: title, Body: summary, Click: runAttentionClick(id)}); err != nil {
|
|
log.Printf("enqueue run notification run=%d: %v", id, err)
|
|
return
|
|
}
|
|
r.deliverRunNotificationBatch(time.Now(), 1)
|
|
}
|
|
|
|
// resumeRunNotifications also recovers pending/expired leases after restart.
|
|
func (r *runner) resumeRunNotifications() {
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-r.ctx.Done():
|
|
return
|
|
case <-r.store.Done():
|
|
return
|
|
case <-ticker.C:
|
|
r.deliverRunNotifications(time.Now())
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *runner) deliverRunNotifications(now time.Time) {
|
|
r.deliverRunNotificationBatch(now, 32)
|
|
}
|
|
|
|
func (r *runner) deliverRunNotificationBatch(now time.Time, limit int) {
|
|
// Bound one drain so a queue cannot monopolize the runner.
|
|
started := time.Now()
|
|
for i := 0; i < limit; i++ {
|
|
n, err := r.store.ClaimRunNotification(r.ctx, now.Add(time.Since(started)).Unix())
|
|
if err != nil {
|
|
log.Printf("claim run notification: %v", err)
|
|
return
|
|
}
|
|
if n == nil {
|
|
return
|
|
}
|
|
// Links are derived from the run id at delivery, not from the stored
|
|
// click: rows queued by older servers may hold a web URL (or nothing),
|
|
// and the web origin is server configuration that can change.
|
|
n.Click = runAttentionClick(n.RunID)
|
|
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
|
|
result, publishErr := stage.PublishNtfy(ctx, n.Channel, n.Title, n.Body, "", nil, n.Click, runAttentionViews(n.RunID)...)
|
|
cancel()
|
|
messageID, _ := result["message_id"].(string)
|
|
save, done := context.WithTimeout(context.Background(), 5*time.Second)
|
|
err = r.store.FinishRunNotification(save, *n, now.Add(time.Since(started)).Unix(), messageID, publishErr)
|
|
done()
|
|
if err != nil {
|
|
log.Printf("save run notification run=%d: %v", n.RunID, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// truncateUTF8 bounds a notification string without splitting a multi-byte rune.
|
|
func truncateUTF8(text string, max int) string {
|
|
if len(text) <= max {
|
|
return text
|
|
}
|
|
text = text[:max]
|
|
for !utf8.ValidString(text) {
|
|
text = text[:len(text)-1]
|
|
}
|
|
return text
|
|
}
|
|
|
|
func notificationArtifact(workspace, name string) ([]byte, error) {
|
|
if name == "" || filepath.IsAbs(name) || filepath.Clean(name) != name || strings.ContainsAny(name, "\\\x00\r\n") {
|
|
return nil, fmt.Errorf("artifact must be a clean workspace-relative file path")
|
|
}
|
|
for _, part := range strings.Split(filepath.ToSlash(name), "/") {
|
|
lower := strings.ToLower(part)
|
|
if strings.HasPrefix(part, ".") || lower == "secrets" || lower == "node_modules" || strings.HasSuffix(lower, ".pem") || strings.HasSuffix(lower, ".key") || strings.HasSuffix(lower, ".db") || strings.HasSuffix(lower, ".db-wal") || strings.HasSuffix(lower, ".db-shm") || strings.HasSuffix(lower, ".sqlite") || strings.HasSuffix(lower, ".sqlite3") {
|
|
return nil, fmt.Errorf("artifact path is protected")
|
|
}
|
|
}
|
|
root, err := filepath.EvalSymlinks(workspace)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("artifact workspace unavailable")
|
|
}
|
|
// os.Root prevents symlink traversal outside the workspace; disallow symlinks
|
|
// altogether so aliases cannot bypass protected path names within it either.
|
|
current := root
|
|
for _, part := range strings.Split(name, string(os.PathSeparator)) {
|
|
current = filepath.Join(current, part)
|
|
info, err := os.Lstat(current)
|
|
if err != nil || info.Mode()&os.ModeSymlink != 0 {
|
|
return nil, fmt.Errorf("artifact is missing or uses a symlink")
|
|
}
|
|
}
|
|
scoped, err := os.OpenRoot(root)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("artifact workspace unavailable")
|
|
}
|
|
defer scoped.Close()
|
|
file, err := scoped.OpenFile(name, os.O_RDONLY|syscall.O_NONBLOCK|syscall.O_NOFOLLOW, 0)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot read artifact")
|
|
}
|
|
defer file.Close()
|
|
info, err := file.Stat()
|
|
if err != nil || !info.Mode().IsRegular() || info.Size() > 1<<20 {
|
|
return nil, fmt.Errorf("artifact must be a regular file at most 1 MiB")
|
|
}
|
|
data, err := io.ReadAll(io.LimitReader(file, (1<<20)+1))
|
|
if err != nil || len(data) > 1<<20 {
|
|
return nil, fmt.Errorf("cannot read artifact within size limit")
|
|
}
|
|
return data, nil
|
|
}
|