Files
pipelines/internal/server/notify.go
T

402 lines
13 KiB
Go

package server
import (
"context"
"fmt"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"io"
"log"
"os"
"path/filepath"
"strings"
"syscall"
"time"
"unicode/utf8"
)
func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.NotificationPolicy, step protocol.RunStep) {
if r.notifyMode(id) == "off" || policy == nil {
return
}
event := "success"
if step.Status == protocol.StepStatusWaiting || step.Outcome == protocol.StepStatusWaiting {
event = protocol.NotifyWaiting
} else if step.Status != "completed" || step.Outcome == "error" || step.Target == protocol.FailureEnd {
event = "failure"
}
enabled := false
for _, on := range policy.On {
if on == event {
enabled = true
}
}
if !enabled {
return
}
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
defer cancel()
summary := fmt.Sprintf("%s · run #%d · stage %s · %s (%s)", name, id, step.Stage, event, step.Outcome)
summary = truncateUTF8(summary, 1800)
title := fmt.Sprintf("Pipeline stage %s", event)
receipts := []protocol.NotificationReceipt{}
persist := func() bool {
save, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := r.store.SaveNotifications(save, id, step.Sequence, receipts); err != nil {
log.Printf("notification audit run=%d step=%d: %v", id, step.Sequence, err)
return false
}
return true
}
send := func(filename string, data []byte) {
receipts = append(receipts, protocol.NotificationReceipt{Destination: "system", Status: "sending", Filename: filename})
if !persist() {
return
}
receipt := &receipts[len(receipts)-1]
result, err := stage.PublishNtfy(ctx, "system", title, summary, filename, data, "")
if err != nil {
receipt.Status = "failed"
receipt.Diagnostic = err.Error()
} else {
receipt.Status = "sent"
receipt.MessageID, _ = result["message_id"].(string)
receipt.AttachmentURL, _ = result["attachment_url"].(string)
}
persist()
}
reject := func(filename, diagnostic string) {
receipts = append(receipts, protocol.NotificationReceipt{Destination: "system", Status: "failed", Filename: filename, Diagnostic: diagnostic})
persist()
}
sentArtifact := false
for _, kind := range policy.Artifacts {
switch kind {
case "markdown":
if markdown, ok := step.Output["markdown"].(string); ok && strings.TrimSpace(markdown) != "" {
sentArtifact = true
send("digest.md", []byte(markdown))
}
case "files":
raw, exists := step.Output["artifacts"]
if !exists {
continue
}
var names []string
switch files := raw.(type) {
case []string:
names = files
case []any:
for _, value := range files {
name, ok := value.(string)
if !ok {
reject("", "output.artifacts must be an array of relative file paths")
names = nil
break
}
names = append(names, name)
}
default:
reject("", "output.artifacts must be an array of relative file paths")
}
if len(names) > 10 {
reject("", "at most 10 file artifacts can be notified")
continue
}
seen := map[string]bool{}
for _, name := range names {
if seen[name] {
continue
}
seen[name] = true
data, err := notificationArtifact(workspace, name)
if err != nil {
reject(name, err.Error())
continue
}
sentArtifact = true
send(filepath.Base(name), data)
}
}
}
// No produced artifacts still merits an event, especially on failures.
if !sentArtifact {
send("", nil)
}
// Only a successful publish covers the run alert. A failed stage publisher
// must not swallow the durable run-level fallback and its retries.
for _, receipt := range receipts {
if receipt.Status == "sent" {
covered := stageAttentionEvent(event)
if event == "success" && step.Target == protocol.SuccessEnd {
covered = "succeeded"
}
r.markAttentionSent(id, step.Sequence, covered)
break
}
}
}
// runAttentionEvent maps a run status to the operator attention notification it
// deserves. Only a run parked on a person (`waiting`) or a run that failed needs
// the operator; every other status — queued, running, succeeded, canceled,
// interrupted — returns "" so the mapping is the single place that decides
// whether a run produces a notification.
func runAttentionEvent(status string) string {
switch strings.TrimSpace(strings.ToLower(status)) {
case protocol.RunStatusWaiting:
return protocol.NotifyWaiting
case "failed":
return "failed"
}
return ""
}
// stageAttentionEvent maps a stage's notification event to the run-attention
// event it covers, or "" when the stage event is not a run-attention event (a
// success digest, for example). It is the one place the two vocabularies meet.
func stageAttentionEvent(event string) string {
switch event {
case protocol.NotifyWaiting:
return protocol.NotifyWaiting
case "failure":
return "failed"
}
return ""
}
// markAttentionSent records that [event] has already been delivered for step
// [sequence] of run [id] (by a stage policy or by the run-level default), so the
// same genuine event is never notified twice. The step sequence is part of the
// key because one run can wait on a person more than once; each wait is its own
// event.
func (r *runner) markAttentionSent(id int64, sequence int, event string) {
if event == "" {
return
}
if r.store != nil {
if err := r.store.CoverRunNotification(r.ctx, id, sequence, event); err != nil {
log.Printf("record stage notification coverage run=%d: %v", id, err)
}
}
r.attentionMu.Lock()
defer r.attentionMu.Unlock()
if r.attentionSent == nil {
r.attentionSent = map[int64]map[string]bool{}
}
if r.attentionSent[id] == nil {
r.attentionSent[id] = map[string]bool{}
}
r.attentionSent[id][fmt.Sprintf("%d:%s", sequence, event)] = true
}
// attentionAlready reports whether [event] was already delivered for step
// [sequence] of run [id].
func (r *runner) attentionAlready(id int64, sequence int, event string) bool {
r.attentionMu.Lock()
defer r.attentionMu.Unlock()
return r.attentionSent[id][fmt.Sprintf("%d:%s", sequence, event)]
}
func (r *runner) setNotifyMode(id int64, mode string) {
mode = strings.ToLower(strings.TrimSpace(mode))
if mode != "on" && mode != "off" {
mode = "auto"
}
r.notifyMu.Lock()
if r.notifyModes == nil {
r.notifyModes = map[int64]string{}
}
r.notifyModes[id] = mode
r.notifyMu.Unlock()
}
func (r *runner) notifyMode(id int64) string {
r.notifyMu.Lock()
mode := r.notifyModes[id]
r.notifyMu.Unlock()
if mode != "" {
return mode
}
if r.store != nil {
if stored, err := r.store.RunNotifyMode(r.ctx, id); err == nil && stored != "" {
r.notifyMu.Lock()
if r.notifyModes == nil {
r.notifyModes = map[int64]string{}
}
r.notifyModes[id] = stored
r.notifyMu.Unlock()
return stored
}
}
return "auto"
}
func (r *runner) forgetNotifyMode(id int64) {
r.notifyMu.Lock()
delete(r.notifyModes, id)
r.notifyMu.Unlock()
}
// forgetAttention drops a run's delivered-event set once it is terminal, so the
// registry is bounded by the runs still in flight.
func (r *runner) forgetAttention(id int64) {
r.attentionMu.Lock()
defer r.attentionMu.Unlock()
delete(r.attentionSent, id)
}
// runAttentionClick is the deep link a run notification carries so tapping it
// opens the run in the Android app, whose intent filter registers this scheme
// and host.
func runAttentionClick(id int64) string {
return fmt.Sprintf("pipelines://run/%d", id)
}
// notifyRun publishes one run-level attention notification to the operator's
// already-configured "system" destination. It is the run counterpart of
// notifyStep and reuses the same PublishNtfy machinery, destination and token
// handling, so no second publisher, topic or service is introduced. The run id
// and pipeline name identify the run and the click action opens it in the app.
// Attempts are persisted before publication. Publisher acknowledgement is not
// proof of device reception; a crash in that acknowledgement window can repeat.
func (r *runner) notifyRun(id int64, name, status string, sequence int, currentStage, diagnostic string) {
if r.notifyMode(id) == "off" {
return
}
event := runAttentionEvent(status)
if status == "succeeded" && r.notifyMode(id) == "on" {
event = "succeeded"
}
if event == "" {
return
}
if r.attentionAlready(id, sequence, event) {
return
}
title := "Run needs your decision"
summary := fmt.Sprintf("Run #%d · %s · waiting for a human decision", id, name)
if event == "failed" {
title = "Run failed"
summary = fmt.Sprintf("Run #%d · %s · run failed", id, name)
}
if event == "succeeded" {
title = "Run succeeded"
summary = fmt.Sprintf("Run #%d · %s · run succeeded", id, name)
}
if currentStage != "" {
summary += " at stage " + currentStage
}
if diagnostic != "" {
summary += ": " + diagnostic
}
summary = truncateUTF8(summary, 2000)
if err := r.store.EnqueueRunNotification(r.ctx, pipeline.RunNotification{RunID: id, Sequence: sequence, Event: event, Channel: "system", Title: title, Body: summary, Click: runAttentionClick(id)}); err != nil {
log.Printf("enqueue run notification run=%d: %v", id, err)
return
}
r.deliverRunNotificationBatch(time.Now(), 1)
}
// resumeRunNotifications also recovers pending/expired leases after restart.
func (r *runner) resumeRunNotifications() {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-r.ctx.Done():
return
case <-ticker.C:
r.deliverRunNotifications(time.Now())
}
}
}
func (r *runner) deliverRunNotifications(now time.Time) {
r.deliverRunNotificationBatch(now, 32)
}
func (r *runner) deliverRunNotificationBatch(now time.Time, limit int) {
// Bound one drain so a queue cannot monopolize the runner.
started := time.Now()
for i := 0; i < limit; i++ {
n, err := r.store.ClaimRunNotification(r.ctx, now.Add(time.Since(started)).Unix())
if err != nil {
log.Printf("claim run notification: %v", err)
return
}
if n == nil {
return
}
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
result, publishErr := stage.PublishNtfy(ctx, n.Channel, n.Title, n.Body, "", nil, n.Click)
cancel()
messageID, _ := result["message_id"].(string)
save, done := context.WithTimeout(context.Background(), 5*time.Second)
err = r.store.FinishRunNotification(save, *n, now.Add(time.Since(started)).Unix(), messageID, publishErr)
done()
if err != nil {
log.Printf("save run notification run=%d: %v", n.RunID, err)
}
}
}
// truncateUTF8 bounds a notification string without splitting a multi-byte rune.
func truncateUTF8(text string, max int) string {
if len(text) <= max {
return text
}
text = text[:max]
for !utf8.ValidString(text) {
text = text[:len(text)-1]
}
return text
}
func notificationArtifact(workspace, name string) ([]byte, error) {
if name == "" || filepath.IsAbs(name) || filepath.Clean(name) != name || strings.ContainsAny(name, "\\\x00\r\n") {
return nil, fmt.Errorf("artifact must be a clean workspace-relative file path")
}
for _, part := range strings.Split(filepath.ToSlash(name), "/") {
lower := strings.ToLower(part)
if strings.HasPrefix(part, ".") || lower == "secrets" || lower == "node_modules" || strings.HasSuffix(lower, ".pem") || strings.HasSuffix(lower, ".key") || strings.HasSuffix(lower, ".db") || strings.HasSuffix(lower, ".db-wal") || strings.HasSuffix(lower, ".db-shm") || strings.HasSuffix(lower, ".sqlite") || strings.HasSuffix(lower, ".sqlite3") {
return nil, fmt.Errorf("artifact path is protected")
}
}
root, err := filepath.EvalSymlinks(workspace)
if err != nil {
return nil, fmt.Errorf("artifact workspace unavailable")
}
// os.Root prevents symlink traversal outside the workspace; disallow symlinks
// altogether so aliases cannot bypass protected path names within it either.
current := root
for _, part := range strings.Split(name, string(os.PathSeparator)) {
current = filepath.Join(current, part)
info, err := os.Lstat(current)
if err != nil || info.Mode()&os.ModeSymlink != 0 {
return nil, fmt.Errorf("artifact is missing or uses a symlink")
}
}
scoped, err := os.OpenRoot(root)
if err != nil {
return nil, fmt.Errorf("artifact workspace unavailable")
}
defer scoped.Close()
file, err := scoped.OpenFile(name, os.O_RDONLY|syscall.O_NONBLOCK|syscall.O_NOFOLLOW, 0)
if err != nil {
return nil, fmt.Errorf("cannot read artifact")
}
defer file.Close()
info, err := file.Stat()
if err != nil || !info.Mode().IsRegular() || info.Size() > 1<<20 {
return nil, fmt.Errorf("artifact must be a regular file at most 1 MiB")
}
data, err := io.ReadAll(io.LimitReader(file, (1<<20)+1))
if err != nil || len(data) > 1<<20 {
return nil, fmt.Errorf("cannot read artifact within size limit")
}
return data, nil
}