815f7c2a5a
A waiting human stage has no outcome yet, so its notification read "… · waiting ()". The outcome is now shown only when recorded, and a waiting stage reads "waiting for your decision".
537 lines
18 KiB
Go
537 lines
18 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.NotificationPolicy, step protocol.RunStep) {
|
|
if r.notifyMode(id) == "off" || policy == nil {
|
|
return
|
|
}
|
|
event := "success"
|
|
if step.Status == protocol.StepStatusWaiting || step.Outcome == protocol.StepStatusWaiting {
|
|
event = protocol.NotifyWaiting
|
|
} else if step.Status != "completed" || step.Outcome == "error" || step.Target == protocol.FailureEnd {
|
|
event = "failure"
|
|
}
|
|
enabled := false
|
|
for _, on := range policy.On {
|
|
if on == event {
|
|
enabled = true
|
|
}
|
|
}
|
|
if !enabled {
|
|
return
|
|
}
|
|
var lease *pipeline.RunNotification
|
|
runEvent := stageAttentionEvent(event)
|
|
if event == "success" && step.Target == protocol.SuccessEnd {
|
|
runEvent = "succeeded"
|
|
}
|
|
if event == protocol.NotifyWaiting || step.Target == protocol.SuccessEnd || step.Target == protocol.FailureEnd {
|
|
var allowed bool
|
|
var err error
|
|
lease, allowed, err = r.store.ClaimStageRunNotification(r.ctx, id, step.Sequence, runEvent, time.Now().Unix())
|
|
if err != nil {
|
|
log.Printf("claim stage notification run=%d: %v", id, err)
|
|
return
|
|
}
|
|
if !allowed {
|
|
return
|
|
}
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
|
|
defer cancel()
|
|
summary := stageNotificationSummary(name, id, step.Stage, event, step.Outcome)
|
|
summary = truncateUTF8(summary, 1800)
|
|
title := fmt.Sprintf("Pipeline stage %s", event)
|
|
receipts := []protocol.NotificationReceipt{}
|
|
persist := func() bool {
|
|
save, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
if err := r.store.SaveNotifications(save, id, step.Sequence, receipts); err != nil {
|
|
log.Printf("notification audit run=%d step=%d: %v", id, step.Sequence, err)
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
// The app stream gets one notification per step event, recorded before
|
|
// the first ntfy publication; artifacts stay ntfy attachments.
|
|
appEvent, appKind := stageNotification(event, step.Target)
|
|
recorded := false
|
|
ntfyOn := ntfyEnabled(ctx, r.settingsStore())
|
|
send := func(filename string, data []byte) {
|
|
receipts = append(receipts, protocol.NotificationReceipt{Destination: "system", Status: "sending", Filename: filename})
|
|
if !persist() {
|
|
return
|
|
}
|
|
receipt := &receipts[len(receipts)-1]
|
|
if !recorded {
|
|
recorded = true
|
|
r.notes.record(ctx, runNotificationRecord(id, step.Sequence, appEvent, appKind, title, summary))
|
|
}
|
|
if !ntfyOn {
|
|
receipt.Status = "sent"
|
|
receipt.Diagnostic = "ntfy disabled; delivered to app clients only"
|
|
persist()
|
|
return
|
|
}
|
|
result, err := ntfyPublish(ctx, "system", title, summary, filename, data, runAttentionClick(id), runAttentionViews(id)...)
|
|
if err != nil {
|
|
receipt.Status = "failed"
|
|
receipt.Diagnostic = err.Error()
|
|
} else {
|
|
receipt.Status = "sent"
|
|
receipt.MessageID, _ = result["message_id"].(string)
|
|
receipt.AttachmentURL, _ = result["attachment_url"].(string)
|
|
}
|
|
persist()
|
|
}
|
|
reject := func(filename, diagnostic string) {
|
|
receipts = append(receipts, protocol.NotificationReceipt{Destination: "system", Status: "failed", Filename: filename, Diagnostic: diagnostic})
|
|
persist()
|
|
}
|
|
sentArtifact := false
|
|
for _, kind := range policy.Artifacts {
|
|
switch kind {
|
|
case "markdown":
|
|
if markdown, ok := step.Output["markdown"].(string); ok && strings.TrimSpace(markdown) != "" {
|
|
sentArtifact = true
|
|
send("digest.md", []byte(markdown))
|
|
}
|
|
case "files":
|
|
raw, exists := step.Output["artifacts"]
|
|
if !exists {
|
|
continue
|
|
}
|
|
var names []string
|
|
switch files := raw.(type) {
|
|
case []string:
|
|
names = files
|
|
case []any:
|
|
for _, value := range files {
|
|
name, ok := value.(string)
|
|
if !ok {
|
|
reject("", "output.artifacts must be an array of relative file paths")
|
|
names = nil
|
|
break
|
|
}
|
|
names = append(names, name)
|
|
}
|
|
default:
|
|
reject("", "output.artifacts must be an array of relative file paths")
|
|
}
|
|
if len(names) > 10 {
|
|
reject("", "at most 10 file artifacts can be notified")
|
|
continue
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, name := range names {
|
|
if seen[name] {
|
|
continue
|
|
}
|
|
seen[name] = true
|
|
data, err := notificationArtifact(workspace, name)
|
|
if err != nil {
|
|
reject(name, err.Error())
|
|
continue
|
|
}
|
|
sentArtifact = true
|
|
send(filepath.Base(name), data)
|
|
}
|
|
}
|
|
}
|
|
// No produced artifacts still merits an event, especially on failures.
|
|
if !sentArtifact {
|
|
send("", nil)
|
|
}
|
|
// Only a successful publish covers the run alert. A failed stage publisher
|
|
// must not swallow the durable run-level fallback and its retries.
|
|
for _, receipt := range receipts {
|
|
if receipt.Status == "sent" {
|
|
covered := stageAttentionEvent(event)
|
|
if event == "success" && step.Target == protocol.SuccessEnd {
|
|
covered = "succeeded"
|
|
}
|
|
r.markAttentionSent(id, step.Sequence, covered)
|
|
return
|
|
}
|
|
}
|
|
if lease != nil {
|
|
diagnostic := "stage notification publication failed"
|
|
if len(receipts) > 0 && receipts[0].Diagnostic != "" {
|
|
diagnostic = receipts[0].Diagnostic
|
|
}
|
|
if err := r.store.FinishRunNotification(r.ctx, *lease, time.Now().Unix(), "", fmt.Errorf("%s", diagnostic)); err != nil {
|
|
log.Printf("release stage notification run=%d: %v", id, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ntfyPublish is the run notification publisher; tests may replace it.
|
|
var ntfyPublish = stage.PublishNtfy
|
|
|
|
// stageNotification maps a stage hook event to the app notification event
|
|
// (the id suffix, shared with the run outbox for the same run event so the
|
|
// two never both appear) and kind.
|
|
func stageNotification(event, target string) (string, string) {
|
|
switch {
|
|
case event == protocol.NotifyWaiting:
|
|
return protocol.NotifyWaiting, protocol.NotificationRunWaiting
|
|
case event == "failure" && target == protocol.FailureEnd:
|
|
return "failed", protocol.NotificationRunFailed
|
|
case event == "failure":
|
|
return "failed", protocol.NotificationStageFailed
|
|
case target == protocol.SuccessEnd:
|
|
return "succeeded", protocol.NotificationRunSucceeded
|
|
}
|
|
return "success", protocol.NotificationStageSucceeded
|
|
}
|
|
|
|
// Terminal hooks run only after FinishRun commits. They arbitrate against the
|
|
// durable intent just like waiting hooks, never publishing a proposed success.
|
|
func (r *runner) notifyTerminalStep(id int64, status string) {
|
|
detail, err := r.store.GetRun(r.ctx, id)
|
|
if err != nil || len(detail.Steps) == 0 {
|
|
return
|
|
}
|
|
step := detail.Steps[len(detail.Steps)-1]
|
|
stepStatus, _, _, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
|
|
if !terminal || stepStatus != status {
|
|
return
|
|
}
|
|
for _, definition := range detail.Graph.Stages {
|
|
if definition.Key == step.Stage {
|
|
r.notifyStep(id, detail.Name, detail.Workspace, definition.Notify, step)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// runAttentionEvent maps a run status to the operator attention notification it
|
|
// deserves. Only a run parked on a person (`waiting`) or a run that failed needs
|
|
// the operator; every other status — queued, running, succeeded, canceled,
|
|
// interrupted — returns "" so the mapping is the single place that decides
|
|
// whether a run produces a notification.
|
|
func runAttentionEvent(status string) string {
|
|
switch strings.TrimSpace(strings.ToLower(status)) {
|
|
case protocol.RunStatusWaiting:
|
|
return protocol.NotifyWaiting
|
|
case "failed":
|
|
return "failed"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// stageAttentionEvent maps a stage's notification event to the run-attention
|
|
// event it covers, or "" when the stage event is not a run-attention event (a
|
|
// success digest, for example). It is the one place the two vocabularies meet.
|
|
func stageAttentionEvent(event string) string {
|
|
switch event {
|
|
case protocol.NotifyWaiting:
|
|
return protocol.NotifyWaiting
|
|
case "failure":
|
|
return "failed"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// markAttentionSent records that [event] has already been delivered for step
|
|
// [sequence] of run [id] (by a stage policy or by the run-level default), so the
|
|
// same genuine event is never notified twice. The step sequence is part of the
|
|
// key because one run can wait on a person more than once; each wait is its own
|
|
// event.
|
|
func (r *runner) markAttentionSent(id int64, sequence int, event string) {
|
|
if event == "" {
|
|
return
|
|
}
|
|
if r.store != nil {
|
|
if err := r.store.CoverRunNotification(r.ctx, id, sequence, event); err != nil {
|
|
log.Printf("record stage notification coverage run=%d: %v", id, err)
|
|
}
|
|
}
|
|
r.attentionMu.Lock()
|
|
defer r.attentionMu.Unlock()
|
|
if r.attentionSent == nil {
|
|
r.attentionSent = map[int64]map[string]bool{}
|
|
}
|
|
if r.attentionSent[id] == nil {
|
|
r.attentionSent[id] = map[string]bool{}
|
|
}
|
|
r.attentionSent[id][fmt.Sprintf("%d:%s", sequence, event)] = true
|
|
}
|
|
|
|
// attentionAlready reports whether [event] was already delivered for step
|
|
// [sequence] of run [id].
|
|
func (r *runner) attentionAlready(id int64, sequence int, event string) bool {
|
|
r.attentionMu.Lock()
|
|
defer r.attentionMu.Unlock()
|
|
return r.attentionSent[id][fmt.Sprintf("%d:%s", sequence, event)]
|
|
}
|
|
|
|
func (r *runner) setNotifyMode(id int64, mode string) {
|
|
mode = strings.ToLower(strings.TrimSpace(mode))
|
|
if mode != "on" && mode != "off" {
|
|
mode = "auto"
|
|
}
|
|
r.notifyMu.Lock()
|
|
if r.notifyModes == nil {
|
|
r.notifyModes = map[int64]string{}
|
|
}
|
|
r.notifyModes[id] = mode
|
|
r.notifyMu.Unlock()
|
|
}
|
|
|
|
func (r *runner) notifyMode(id int64) string {
|
|
r.notifyMu.Lock()
|
|
mode := r.notifyModes[id]
|
|
r.notifyMu.Unlock()
|
|
if mode != "" {
|
|
return mode
|
|
}
|
|
if r.store != nil {
|
|
if stored, err := r.store.RunNotifyMode(r.ctx, id); err == nil && stored != "" {
|
|
r.notifyMu.Lock()
|
|
if r.notifyModes == nil {
|
|
r.notifyModes = map[int64]string{}
|
|
}
|
|
r.notifyModes[id] = stored
|
|
r.notifyMu.Unlock()
|
|
return stored
|
|
}
|
|
}
|
|
return "auto"
|
|
}
|
|
|
|
func (r *runner) forgetNotifyMode(id int64) {
|
|
r.notifyMu.Lock()
|
|
delete(r.notifyModes, id)
|
|
r.notifyMu.Unlock()
|
|
}
|
|
|
|
// forgetAttention drops a run's delivered-event set once it is terminal, so the
|
|
// registry is bounded by the runs still in flight.
|
|
func (r *runner) forgetAttention(id int64) {
|
|
r.attentionMu.Lock()
|
|
defer r.attentionMu.Unlock()
|
|
delete(r.attentionSent, id)
|
|
}
|
|
|
|
// runAttentionClick is the notification tap target for a run: always the native
|
|
// app deep link, so a tap on the phone opens the Pipelines app.
|
|
func runAttentionClick(id int64) string {
|
|
return fmt.Sprintf("pipelines://run/%d", id)
|
|
}
|
|
|
|
// runAttentionViews is the optional "Open in browser" action for a run.
|
|
func runAttentionViews(id int64) []stage.NtfyView {
|
|
return webAttentionViews(fmt.Sprintf("/runs/%d", id))
|
|
}
|
|
|
|
// webAttentionViews returns an "Open in browser" ntfy action for path on the
|
|
// operator's authenticated web origin, or nothing when none is configured.
|
|
// Never infer a public address from a request Host, the unauthenticated API
|
|
// listener or the ntfy server URL.
|
|
func webAttentionViews(path string) []stage.NtfyView {
|
|
raw := os.Getenv("PIPELINES_WEB_PUBLIC_URL")
|
|
if raw == "" {
|
|
return nil
|
|
}
|
|
base := publicNtfyBase(raw)
|
|
if base == "" {
|
|
log.Printf("notification web link unavailable: PIPELINES_WEB_PUBLIC_URL must be an HTTP(S) origin without credentials, query or fragment")
|
|
return nil
|
|
}
|
|
return []stage.NtfyView{{Label: "Open in browser", URL: base + path}}
|
|
}
|
|
|
|
// notifyRun publishes one run-level attention notification to the operator's
|
|
// already-configured "system" destination. It is the run counterpart of
|
|
// notifyStep and reuses the same PublishNtfy machinery, destination and token
|
|
// handling, so no second publisher, topic or service is introduced. The run id
|
|
// and pipeline name identify the run and the click action opens it in the app.
|
|
// Attempts are persisted before publication. Publisher acknowledgement is not
|
|
// proof of device reception; a crash in that acknowledgement window can repeat.
|
|
func (r *runner) notifyRun(id int64, name, status string, sequence int, currentStage, diagnostic string) {
|
|
if r.notifyMode(id) == "off" {
|
|
return
|
|
}
|
|
event := runAttentionEvent(status)
|
|
if status == "succeeded" {
|
|
event = "succeeded"
|
|
}
|
|
if event == "" {
|
|
return
|
|
}
|
|
// A run's end notifies only runs marked "on", unless the operator opted
|
|
// into every run end (run_finished); waiting notifies any non-off run.
|
|
if event != protocol.NotifyWaiting && r.notifyMode(id) != "on" {
|
|
settings, err := r.store.NotificationSettings(r.ctx)
|
|
if err != nil || !settings.RunFinished {
|
|
return
|
|
}
|
|
}
|
|
if r.attentionAlready(id, sequence, event) {
|
|
return
|
|
}
|
|
title := "Run needs your decision"
|
|
summary := fmt.Sprintf("Run #%d · %s · waiting for a human decision", id, name)
|
|
if event == "failed" {
|
|
title = "Run failed"
|
|
summary = fmt.Sprintf("Run #%d · %s · run failed", id, name)
|
|
}
|
|
if event == "succeeded" {
|
|
title = "Run succeeded"
|
|
summary = fmt.Sprintf("Run #%d · %s · run succeeded", id, name)
|
|
}
|
|
if currentStage != "" {
|
|
summary += " at stage " + currentStage
|
|
}
|
|
if diagnostic != "" {
|
|
summary += ": " + diagnostic
|
|
}
|
|
summary = truncateUTF8(summary, 2000)
|
|
if err := r.store.EnqueueRunNotification(r.ctx, pipeline.RunNotification{RunID: id, Sequence: sequence, Event: event, Channel: "system", Title: title, Body: summary, Click: runAttentionClick(id)}); err != nil {
|
|
log.Printf("enqueue run notification run=%d: %v", id, err)
|
|
return
|
|
}
|
|
r.deliverRunNotificationBatch(time.Now(), 1)
|
|
}
|
|
|
|
// resumeRunNotifications also recovers pending/expired leases after restart.
|
|
func (r *runner) resumeRunNotifications() {
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-r.ctx.Done():
|
|
return
|
|
case <-r.store.Done():
|
|
return
|
|
case <-ticker.C:
|
|
r.deliverRunNotifications(time.Now())
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *runner) deliverRunNotifications(now time.Time) {
|
|
r.deliverRunNotificationBatch(now, 32)
|
|
}
|
|
|
|
func (r *runner) deliverRunNotificationBatch(now time.Time, limit int) {
|
|
// Bound one drain so a queue cannot monopolize the runner.
|
|
started := time.Now()
|
|
for i := 0; i < limit; i++ {
|
|
n, err := r.store.ClaimRunNotification(r.ctx, now.Add(time.Since(started)).Unix())
|
|
if err != nil {
|
|
log.Printf("claim run notification: %v", err)
|
|
return
|
|
}
|
|
if n == nil {
|
|
return
|
|
}
|
|
// Links are derived from the run id at delivery, not from the stored
|
|
// click: rows queued by older servers may hold a web URL (or nothing),
|
|
// and the web origin is server configuration that can change.
|
|
n.Click = runAttentionClick(n.RunID)
|
|
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
|
|
r.notes.record(ctx, runNotificationRecord(n.RunID, n.Sequence, n.Event, runNotificationKind(n.Event), n.Title, n.Body))
|
|
var result stage.Values
|
|
var publishErr error
|
|
if ntfyEnabled(ctx, r.settingsStore()) {
|
|
result, publishErr = ntfyPublish(ctx, n.Channel, n.Title, n.Body, "", nil, n.Click, runAttentionViews(n.RunID)...)
|
|
}
|
|
cancel()
|
|
messageID, _ := result["message_id"].(string)
|
|
save, done := context.WithTimeout(context.Background(), 5*time.Second)
|
|
err = r.store.FinishRunNotification(save, *n, now.Add(time.Since(started)).Unix(), messageID, publishErr)
|
|
done()
|
|
if err != nil {
|
|
log.Printf("save run notification run=%d: %v", n.RunID, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// truncateUTF8 bounds a notification string without splitting a multi-byte rune.
|
|
func truncateUTF8(text string, max int) string {
|
|
if len(text) <= max {
|
|
return text
|
|
}
|
|
text = text[:max]
|
|
for !utf8.ValidString(text) {
|
|
text = text[:len(text)-1]
|
|
}
|
|
return text
|
|
}
|
|
|
|
func notificationArtifact(workspace, name string) ([]byte, error) {
|
|
if name == "" || filepath.IsAbs(name) || filepath.Clean(name) != name || strings.ContainsAny(name, "\\\x00\r\n") {
|
|
return nil, fmt.Errorf("artifact must be a clean workspace-relative file path")
|
|
}
|
|
for _, part := range strings.Split(filepath.ToSlash(name), "/") {
|
|
lower := strings.ToLower(part)
|
|
if strings.HasPrefix(part, ".") || lower == "secrets" || lower == "node_modules" || strings.HasSuffix(lower, ".pem") || strings.HasSuffix(lower, ".key") || strings.HasSuffix(lower, ".db") || strings.HasSuffix(lower, ".db-wal") || strings.HasSuffix(lower, ".db-shm") || strings.HasSuffix(lower, ".sqlite") || strings.HasSuffix(lower, ".sqlite3") {
|
|
return nil, fmt.Errorf("artifact path is protected")
|
|
}
|
|
}
|
|
root, err := filepath.EvalSymlinks(workspace)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("artifact workspace unavailable")
|
|
}
|
|
// os.Root prevents symlink traversal outside the workspace; disallow symlinks
|
|
// altogether so aliases cannot bypass protected path names within it either.
|
|
current := root
|
|
for _, part := range strings.Split(name, string(os.PathSeparator)) {
|
|
current = filepath.Join(current, part)
|
|
info, err := os.Lstat(current)
|
|
if err != nil || info.Mode()&os.ModeSymlink != 0 {
|
|
return nil, fmt.Errorf("artifact is missing or uses a symlink")
|
|
}
|
|
}
|
|
scoped, err := os.OpenRoot(root)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("artifact workspace unavailable")
|
|
}
|
|
defer scoped.Close()
|
|
file, err := scoped.OpenFile(name, os.O_RDONLY|syscall.O_NONBLOCK|syscall.O_NOFOLLOW, 0)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot read artifact")
|
|
}
|
|
defer file.Close()
|
|
info, err := file.Stat()
|
|
if err != nil || !info.Mode().IsRegular() || info.Size() > 1<<20 {
|
|
return nil, fmt.Errorf("artifact must be a regular file at most 1 MiB")
|
|
}
|
|
data, err := io.ReadAll(io.LimitReader(file, (1<<20)+1))
|
|
if err != nil || len(data) > 1<<20 {
|
|
return nil, fmt.Errorf("cannot read artifact within size limit")
|
|
}
|
|
return data, nil
|
|
}
|
|
|
|
// stageNotificationSummary is a stage notification's one-line body. The
|
|
// outcome is shown only when the stage recorded one (a waiting human stage has
|
|
// none yet), and a waiting stage says what it waits for.
|
|
func stageNotificationSummary(name string, runID int64, stage, event, outcome string) string {
|
|
state := event
|
|
if event == "waiting" {
|
|
state = "waiting for your decision"
|
|
}
|
|
if outcome = strings.TrimSpace(outcome); outcome != "" {
|
|
state += " (" + outcome + ")"
|
|
}
|
|
return fmt.Sprintf("%s · run #%d · stage %s · %s", name, runID, stage, state)
|
|
}
|