a733a927f3
Preserve Settings as an independent navigation destination while landing the reviewed notification and skill-management updates. Keep the server and CLI skill contracts versioned and validated so the merged tree can be verified by the delivery gate.
552 lines
18 KiB
Go
552 lines
18 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.NotificationPolicy, step protocol.RunStep) {
|
|
if r.notifyMode(id) == "off" || policy == nil {
|
|
return
|
|
}
|
|
event := "success"
|
|
if step.Status == protocol.StepStatusWaiting || step.Outcome == protocol.StepStatusWaiting {
|
|
event = protocol.NotifyWaiting
|
|
} else if step.Status != "completed" || step.Outcome == "error" || step.Target == protocol.FailureEnd {
|
|
event = "failure"
|
|
}
|
|
enabled := false
|
|
for _, on := range policy.On {
|
|
if on == event {
|
|
enabled = true
|
|
}
|
|
}
|
|
if !enabled {
|
|
return
|
|
}
|
|
var lease *pipeline.RunNotification
|
|
runEvent := stageAttentionEvent(event)
|
|
if event == "success" && step.Target == protocol.SuccessEnd {
|
|
runEvent = "succeeded"
|
|
}
|
|
if event == protocol.NotifyWaiting || step.Target == protocol.SuccessEnd || step.Target == protocol.FailureEnd {
|
|
var allowed bool
|
|
var err error
|
|
lease, allowed, err = r.store.ClaimStageRunNotification(r.ctx, id, step.Sequence, runEvent, time.Now().Unix())
|
|
if err != nil {
|
|
log.Printf("claim stage notification run=%d: %v", id, err)
|
|
return
|
|
}
|
|
if !allowed {
|
|
return
|
|
}
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
|
|
defer cancel()
|
|
summary := stageNotificationSummary(name, id, step.Stage, event, step.Outcome)
|
|
summary = truncateUTF8(summary, 1800)
|
|
title := fmt.Sprintf("Pipeline stage %s", event)
|
|
receipts := []protocol.NotificationReceipt{}
|
|
persist := func() bool {
|
|
save, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
if err := r.store.SaveNotifications(save, id, step.Sequence, receipts); err != nil {
|
|
log.Printf("notification audit run=%d step=%d: %v", id, step.Sequence, err)
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
// The app stream gets one notification per step event, recorded before
|
|
// the first ntfy publication; artifacts stay ntfy attachments.
|
|
appEvent, appKind := stageNotification(event, step.Target)
|
|
recorded := false
|
|
ntfyOn := ntfyEnabled(ctx, r.settingsStore())
|
|
send := func(filename string, data []byte) {
|
|
receipts = append(receipts, protocol.NotificationReceipt{Destination: "system", Status: "sending", Filename: filename})
|
|
if !persist() {
|
|
return
|
|
}
|
|
receipt := &receipts[len(receipts)-1]
|
|
if !recorded {
|
|
recorded = true
|
|
r.notes.record(ctx, runNotificationRecord(id, step.Sequence, appEvent, appKind, title, summary))
|
|
}
|
|
if !ntfyOn {
|
|
receipt.Status = "sent"
|
|
receipt.Diagnostic = "ntfy disabled; delivered to app clients only"
|
|
persist()
|
|
return
|
|
}
|
|
result, err := ntfyPublish(ctx, "system", title, summary, filename, data, runAttentionClick(id), runAttentionViews(id)...)
|
|
if err != nil {
|
|
receipt.Status = "failed"
|
|
receipt.Diagnostic = err.Error()
|
|
} else {
|
|
receipt.Status = "sent"
|
|
receipt.MessageID, _ = result["message_id"].(string)
|
|
receipt.AttachmentURL, _ = result["attachment_url"].(string)
|
|
}
|
|
persist()
|
|
}
|
|
reject := func(filename, diagnostic string) {
|
|
receipts = append(receipts, protocol.NotificationReceipt{Destination: "system", Status: "failed", Filename: filename, Diagnostic: diagnostic})
|
|
persist()
|
|
}
|
|
sentArtifact := false
|
|
for _, kind := range policy.Artifacts {
|
|
switch kind {
|
|
case "markdown":
|
|
if markdown, ok := step.Output["markdown"].(string); ok && strings.TrimSpace(markdown) != "" {
|
|
sentArtifact = true
|
|
send("digest.md", []byte(markdown))
|
|
}
|
|
case "files":
|
|
raw, exists := step.Output["artifacts"]
|
|
if !exists {
|
|
continue
|
|
}
|
|
var names []string
|
|
switch files := raw.(type) {
|
|
case []string:
|
|
names = files
|
|
case []any:
|
|
for _, value := range files {
|
|
name, ok := value.(string)
|
|
if !ok {
|
|
reject("", "output.artifacts must be an array of relative file paths")
|
|
names = nil
|
|
break
|
|
}
|
|
names = append(names, name)
|
|
}
|
|
default:
|
|
reject("", "output.artifacts must be an array of relative file paths")
|
|
}
|
|
if len(names) > 10 {
|
|
reject("", "at most 10 file artifacts can be notified")
|
|
continue
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, name := range names {
|
|
if seen[name] {
|
|
continue
|
|
}
|
|
seen[name] = true
|
|
data, err := notificationArtifact(workspace, name)
|
|
if err != nil {
|
|
reject(name, err.Error())
|
|
continue
|
|
}
|
|
sentArtifact = true
|
|
send(filepath.Base(name), data)
|
|
}
|
|
}
|
|
}
|
|
// No produced artifacts still merits an event, especially on failures.
|
|
if !sentArtifact {
|
|
send("", nil)
|
|
}
|
|
// Only a successful publish covers the run alert. A failed stage publisher
|
|
// must not swallow the durable run-level fallback and its retries.
|
|
for _, receipt := range receipts {
|
|
if receipt.Status == "sent" {
|
|
covered := stageAttentionEvent(event)
|
|
if event == "success" && step.Target == protocol.SuccessEnd {
|
|
covered = "succeeded"
|
|
}
|
|
r.markAttentionSent(id, step.Sequence, covered)
|
|
return
|
|
}
|
|
}
|
|
if lease != nil {
|
|
diagnostic := "stage notification publication failed"
|
|
if len(receipts) > 0 && receipts[0].Diagnostic != "" {
|
|
diagnostic = receipts[0].Diagnostic
|
|
}
|
|
if err := r.store.FinishRunNotification(r.ctx, *lease, time.Now().Unix(), "", fmt.Errorf("%s", diagnostic)); err != nil {
|
|
log.Printf("release stage notification run=%d: %v", id, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ntfyPublish is the run notification publisher; tests may replace it.
|
|
var ntfyPublish = stage.PublishNtfy
|
|
|
|
// stageNotification maps a stage hook event to the app notification event
|
|
// (the id suffix, shared with the run outbox for the same run event so the
|
|
// two never both appear) and kind.
|
|
func stageNotification(event, target string) (string, string) {
|
|
switch {
|
|
case event == protocol.NotifyWaiting:
|
|
return protocol.NotifyWaiting, protocol.NotificationRunWaiting
|
|
case event == "failure" && target == protocol.FailureEnd:
|
|
return "failed", protocol.NotificationRunFailed
|
|
case event == "failure":
|
|
return "failed", protocol.NotificationStageFailed
|
|
case target == protocol.SuccessEnd:
|
|
return "succeeded", protocol.NotificationRunSucceeded
|
|
}
|
|
return "success", protocol.NotificationStageSucceeded
|
|
}
|
|
|
|
// Terminal hooks run only after FinishRun commits. They arbitrate against the
|
|
// durable intent just like waiting hooks, never publishing a proposed success.
|
|
func (r *runner) notifyTerminalStep(id int64, status string) {
|
|
detail, err := r.store.GetRun(r.ctx, id)
|
|
if err != nil || len(detail.Steps) == 0 {
|
|
return
|
|
}
|
|
if status == "failed" && detail.FailureKind == "expected" {
|
|
return
|
|
}
|
|
step := detail.Steps[len(detail.Steps)-1]
|
|
stepStatus, _, _, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
|
|
if !terminal || stepStatus != status {
|
|
return
|
|
}
|
|
for _, definition := range detail.Graph.Stages {
|
|
if definition.Key == step.Stage {
|
|
r.notifyStep(id, detail.Name, detail.Workspace, definition.Notify, step)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// runAttentionEvent maps a run status to the operator attention notification it
|
|
// deserves. Only a run parked on a person (`waiting`) or a run that failed needs
|
|
// the operator; every other status — queued, running, succeeded, canceled,
|
|
// interrupted — returns "" so the mapping is the single place that decides
|
|
// whether a run produces a notification.
|
|
func runAttentionEvent(status string) string {
|
|
switch strings.TrimSpace(strings.ToLower(status)) {
|
|
case protocol.RunStatusWaiting:
|
|
return protocol.NotifyWaiting
|
|
case "failed", "interrupted":
|
|
return strings.TrimSpace(strings.ToLower(status))
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// stageAttentionEvent maps a stage's notification event to the run-attention
|
|
// event it covers, or "" when the stage event is not a run-attention event (a
|
|
// success digest, for example). It is the one place the two vocabularies meet.
|
|
func stageAttentionEvent(event string) string {
|
|
switch event {
|
|
case protocol.NotifyWaiting:
|
|
return protocol.NotifyWaiting
|
|
case "failure":
|
|
return "failed"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// markAttentionSent records that [event] has already been delivered for step
|
|
// [sequence] of run [id] (by a stage policy or by the run-level default), so the
|
|
// same genuine event is never notified twice. The step sequence is part of the
|
|
// key because one run can wait on a person more than once; each wait is its own
|
|
// event.
|
|
func (r *runner) markAttentionSent(id int64, sequence int, event string) {
|
|
if event == "" {
|
|
return
|
|
}
|
|
if r.store != nil {
|
|
if err := r.store.CoverRunNotification(r.ctx, id, sequence, event); err != nil {
|
|
log.Printf("record stage notification coverage run=%d: %v", id, err)
|
|
}
|
|
}
|
|
r.attentionMu.Lock()
|
|
defer r.attentionMu.Unlock()
|
|
if r.attentionSent == nil {
|
|
r.attentionSent = map[int64]map[string]bool{}
|
|
}
|
|
if r.attentionSent[id] == nil {
|
|
r.attentionSent[id] = map[string]bool{}
|
|
}
|
|
r.attentionSent[id][fmt.Sprintf("%d:%s", sequence, event)] = true
|
|
}
|
|
|
|
// attentionAlready reports whether [event] was already delivered for step
|
|
// [sequence] of run [id].
|
|
func (r *runner) attentionAlready(id int64, sequence int, event string) bool {
|
|
r.attentionMu.Lock()
|
|
defer r.attentionMu.Unlock()
|
|
return r.attentionSent[id][fmt.Sprintf("%d:%s", sequence, event)]
|
|
}
|
|
|
|
func (r *runner) setNotifyMode(id int64, mode string) {
|
|
mode = strings.ToLower(strings.TrimSpace(mode))
|
|
if mode != "on" && mode != "off" {
|
|
mode = "auto"
|
|
}
|
|
r.notifyMu.Lock()
|
|
if r.notifyModes == nil {
|
|
r.notifyModes = map[int64]string{}
|
|
}
|
|
r.notifyModes[id] = mode
|
|
r.notifyMu.Unlock()
|
|
}
|
|
|
|
func (r *runner) notifyMode(id int64) string {
|
|
r.notifyMu.Lock()
|
|
mode := r.notifyModes[id]
|
|
r.notifyMu.Unlock()
|
|
if mode != "" {
|
|
return mode
|
|
}
|
|
if r.store != nil {
|
|
if stored, err := r.store.RunNotifyMode(r.ctx, id); err == nil && stored != "" {
|
|
r.notifyMu.Lock()
|
|
if r.notifyModes == nil {
|
|
r.notifyModes = map[int64]string{}
|
|
}
|
|
r.notifyModes[id] = stored
|
|
r.notifyMu.Unlock()
|
|
return stored
|
|
}
|
|
}
|
|
return "auto"
|
|
}
|
|
|
|
func (r *runner) forgetNotifyMode(id int64) {
|
|
r.notifyMu.Lock()
|
|
delete(r.notifyModes, id)
|
|
r.notifyMu.Unlock()
|
|
}
|
|
|
|
// forgetAttention drops a run's delivered-event set once it is terminal, so the
|
|
// registry is bounded by the runs still in flight.
|
|
func (r *runner) forgetAttention(id int64) {
|
|
r.attentionMu.Lock()
|
|
defer r.attentionMu.Unlock()
|
|
delete(r.attentionSent, id)
|
|
}
|
|
|
|
// runAttentionClick is the notification tap target for a run: always the native
|
|
// app deep link, so a tap on the phone opens the Pipelines app.
|
|
func runAttentionClick(id int64) string {
|
|
return fmt.Sprintf("pipelines://run/%d", id)
|
|
}
|
|
|
|
// runAttentionViews is the optional "Open in browser" action for a run.
|
|
func runAttentionViews(id int64) []stage.NtfyView {
|
|
return webAttentionViews(fmt.Sprintf("/runs/%d", id))
|
|
}
|
|
|
|
// webAttentionViews returns an "Open in browser" ntfy action for path on the
|
|
// operator's authenticated web origin, or nothing when none is configured.
|
|
// Never infer a public address from a request Host, the unauthenticated API
|
|
// listener or the ntfy server URL.
|
|
func webAttentionViews(path string) []stage.NtfyView {
|
|
raw := os.Getenv("PIPELINES_WEB_PUBLIC_URL")
|
|
if raw == "" {
|
|
return nil
|
|
}
|
|
base := publicNtfyBase(raw)
|
|
if base == "" {
|
|
log.Printf("notification web link unavailable: PIPELINES_WEB_PUBLIC_URL must be an HTTP(S) origin without credentials, query or fragment")
|
|
return nil
|
|
}
|
|
return []stage.NtfyView{{Label: "Open in browser", URL: base + path}}
|
|
}
|
|
|
|
// notifyRun publishes one run-level attention notification to the operator's
|
|
// already-configured "system" destination. It is the run counterpart of
|
|
// notifyStep and reuses the same PublishNtfy machinery, destination and token
|
|
// handling, so no second publisher, topic or service is introduced. The run id
|
|
// and pipeline name identify the run and the click action opens it in the app.
|
|
// Attempts are persisted before publication. Publisher acknowledgement is not
|
|
// proof of device reception; a crash in that acknowledgement window can repeat.
|
|
func (r *runner) notifyRun(id int64, name, status string, sequence int, currentStage, diagnostic string) {
|
|
if r.notifyMode(id) == "off" {
|
|
return
|
|
}
|
|
event := runAttentionEvent(status)
|
|
if status == "succeeded" {
|
|
event = "succeeded"
|
|
}
|
|
if event == "" {
|
|
return
|
|
}
|
|
if status == "failed" {
|
|
if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail.FailureKind == "expected" {
|
|
settings, settingsErr := r.store.NotificationSettings(r.ctx)
|
|
if settingsErr != nil || (r.notifyMode(id) != "on" && !settings.RunFinished) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
// Failed or interrupted runs always notify; successful runs notify only when
|
|
// marked "on" or when the operator opted into every run end (run_finished).
|
|
if event == "succeeded" || (event != protocol.NotifyWaiting && event != "failed" && event != "interrupted") {
|
|
settings, err := r.store.NotificationSettings(r.ctx)
|
|
if err != nil || !settings.RunFinished {
|
|
return
|
|
}
|
|
}
|
|
if r.attentionAlready(id, sequence, event) {
|
|
return
|
|
}
|
|
title := "Run needs your decision"
|
|
summary := fmt.Sprintf("Run #%d · %s · waiting for a human decision", id, name)
|
|
if event == "failed" {
|
|
title = "Run failed"
|
|
summary = fmt.Sprintf("Run #%d · %s · run failed", id, name)
|
|
}
|
|
if event == "interrupted" {
|
|
title = "Run interrupted"
|
|
summary = fmt.Sprintf("Run #%d · %s · run interrupted", id, name)
|
|
}
|
|
if event == "succeeded" {
|
|
title = "Run succeeded"
|
|
summary = fmt.Sprintf("Run #%d · %s · run succeeded", id, name)
|
|
}
|
|
if currentStage != "" {
|
|
summary += " at stage " + currentStage
|
|
}
|
|
if diagnostic != "" {
|
|
summary += ": " + diagnostic
|
|
}
|
|
summary = truncateUTF8(summary, 2000)
|
|
if err := r.store.EnqueueRunNotification(r.ctx, pipeline.RunNotification{RunID: id, Sequence: sequence, Event: event, Channel: "system", Title: title, Body: summary, Click: runAttentionClick(id)}); err != nil {
|
|
log.Printf("enqueue run notification run=%d: %v", id, err)
|
|
return
|
|
}
|
|
r.deliverRunNotificationBatch(time.Now(), 1)
|
|
}
|
|
|
|
// resumeRunNotifications also recovers pending/expired leases after restart.
|
|
func (r *runner) resumeRunNotifications() {
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-r.ctx.Done():
|
|
return
|
|
case <-r.store.Done():
|
|
return
|
|
case <-ticker.C:
|
|
r.deliverRunNotifications(time.Now())
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *runner) deliverRunNotifications(now time.Time) {
|
|
r.deliverRunNotificationBatch(now, 32)
|
|
}
|
|
|
|
func (r *runner) deliverRunNotificationBatch(now time.Time, limit int) {
|
|
// Bound one drain so a queue cannot monopolize the runner.
|
|
started := time.Now()
|
|
for i := 0; i < limit; i++ {
|
|
n, err := r.store.ClaimRunNotification(r.ctx, now.Add(time.Since(started)).Unix())
|
|
if err != nil {
|
|
log.Printf("claim run notification: %v", err)
|
|
return
|
|
}
|
|
if n == nil {
|
|
return
|
|
}
|
|
// Links are derived from the run id at delivery, not from the stored
|
|
// click: rows queued by older servers may hold a web URL (or nothing),
|
|
// and the web origin is server configuration that can change.
|
|
n.Click = runAttentionClick(n.RunID)
|
|
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
|
|
r.notes.record(ctx, runNotificationRecord(n.RunID, n.Sequence, n.Event, runNotificationKind(n.Event), n.Title, n.Body))
|
|
var result stage.Values
|
|
var publishErr error
|
|
if ntfyEnabled(ctx, r.settingsStore()) {
|
|
result, publishErr = ntfyPublish(ctx, n.Channel, n.Title, n.Body, "", nil, n.Click, runAttentionViews(n.RunID)...)
|
|
}
|
|
cancel()
|
|
messageID, _ := result["message_id"].(string)
|
|
save, done := context.WithTimeout(context.Background(), 5*time.Second)
|
|
err = r.store.FinishRunNotification(save, *n, now.Add(time.Since(started)).Unix(), messageID, publishErr)
|
|
done()
|
|
if err != nil {
|
|
log.Printf("save run notification run=%d: %v", n.RunID, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// truncateUTF8 bounds a notification string without splitting a multi-byte rune.
|
|
func truncateUTF8(text string, max int) string {
|
|
if len(text) <= max {
|
|
return text
|
|
}
|
|
text = text[:max]
|
|
for !utf8.ValidString(text) {
|
|
text = text[:len(text)-1]
|
|
}
|
|
return text
|
|
}
|
|
|
|
func notificationArtifact(workspace, name string) ([]byte, error) {
|
|
if name == "" || filepath.IsAbs(name) || filepath.Clean(name) != name || strings.ContainsAny(name, "\\\x00\r\n") {
|
|
return nil, fmt.Errorf("artifact must be a clean workspace-relative file path")
|
|
}
|
|
for _, part := range strings.Split(filepath.ToSlash(name), "/") {
|
|
lower := strings.ToLower(part)
|
|
if strings.HasPrefix(part, ".") || lower == "secrets" || lower == "node_modules" || strings.HasSuffix(lower, ".pem") || strings.HasSuffix(lower, ".key") || strings.HasSuffix(lower, ".db") || strings.HasSuffix(lower, ".db-wal") || strings.HasSuffix(lower, ".db-shm") || strings.HasSuffix(lower, ".sqlite") || strings.HasSuffix(lower, ".sqlite3") {
|
|
return nil, fmt.Errorf("artifact path is protected")
|
|
}
|
|
}
|
|
root, err := filepath.EvalSymlinks(workspace)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("artifact workspace unavailable")
|
|
}
|
|
// os.Root prevents symlink traversal outside the workspace; disallow symlinks
|
|
// altogether so aliases cannot bypass protected path names within it either.
|
|
current := root
|
|
for _, part := range strings.Split(name, string(os.PathSeparator)) {
|
|
current = filepath.Join(current, part)
|
|
info, err := os.Lstat(current)
|
|
if err != nil || info.Mode()&os.ModeSymlink != 0 {
|
|
return nil, fmt.Errorf("artifact is missing or uses a symlink")
|
|
}
|
|
}
|
|
scoped, err := os.OpenRoot(root)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("artifact workspace unavailable")
|
|
}
|
|
defer scoped.Close()
|
|
file, err := scoped.OpenFile(name, os.O_RDONLY|syscall.O_NONBLOCK|syscall.O_NOFOLLOW, 0)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot read artifact")
|
|
}
|
|
defer file.Close()
|
|
info, err := file.Stat()
|
|
if err != nil || !info.Mode().IsRegular() || info.Size() > 1<<20 {
|
|
return nil, fmt.Errorf("artifact must be a regular file at most 1 MiB")
|
|
}
|
|
data, err := io.ReadAll(io.LimitReader(file, (1<<20)+1))
|
|
if err != nil || len(data) > 1<<20 {
|
|
return nil, fmt.Errorf("cannot read artifact within size limit")
|
|
}
|
|
return data, nil
|
|
}
|
|
|
|
// stageNotificationSummary is a stage notification's one-line body. The
|
|
// outcome is shown only when the stage recorded one (a waiting human stage has
|
|
// none yet), and a waiting stage says what it waits for.
|
|
func stageNotificationSummary(name string, runID int64, stage, event, outcome string) string {
|
|
state := event
|
|
if event == "waiting" {
|
|
state = "waiting for your decision"
|
|
}
|
|
if outcome = strings.TrimSpace(outcome); outcome != "" {
|
|
state += " (" + outcome + ")"
|
|
}
|
|
return fmt.Sprintf("%s · run #%d · stage %s · %s", name, runID, stage, state)
|
|
}
|