fix: persist and retry run alerts and honor notification modes
This commit is contained in:
+15
@@ -574,6 +574,21 @@ Everything else is free-form and preserved under the step's `markdown` output.
|
||||
the run. Artifacts are served from `/artifacts/<run>/<step>/<name>`; shared files
|
||||
appear with a `public/` prefix.
|
||||
|
||||
### Run notification modes and receipts
|
||||
|
||||
`--notify auto|on|off` records `notify_mode` on the run (also returned in
|
||||
summaries and `run_event` frames). Auto retains stage policies and run-level
|
||||
waiting/failure defaults. On also alerts terminal success without requiring a
|
||||
stage hook. Off suppresses stage, run-level system and browser tray alerts.
|
||||
|
||||
`inspect --json` / `get_run` includes top-level `notifications` receipts with
|
||||
`sequence`, `event`, `channel`, `status`, `attempts`, and optional `diagnostic`
|
||||
and `message_id`. Status is pending, sending, sent, failed, or covered (already
|
||||
published by the stage). Run-level system alerts persist and retry at most three
|
||||
times, recovering pending work after restart. A publisher acknowledgement is not
|
||||
proof of reception; a crash before saving it can duplicate a message. See
|
||||
[`notes/audit-notifications.md`](notes/audit-notifications.md) for limits.
|
||||
|
||||
## Investigating runs with a configured harness
|
||||
|
||||
On the runs list or inspector, press `D` or use `:debug`; CLI equivalent:
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# Run notification audit fixes
|
||||
|
||||
`notify_mode=auto` retains existing behavior: stage policies plus run-level
|
||||
waiting/failure defaults. `on` also alerts terminal success, even with no stage
|
||||
notification hook. `off` suppresses stage, run-level system and browser tray
|
||||
notifications. Needs you remains a status/action surface, not a delivery channel.
|
||||
Summaries and pushed events carry the mode; the browser seeds and folds it.
|
||||
|
||||
`run_notifications` is an additive, idempotent SQLite migration. Each
|
||||
run/sequence/event/channel has one durable outbox row. Its status is `pending`,
|
||||
`sending`, `sent`, `failed`, or `covered` (a stage already published this event).
|
||||
The worker polls once per second, uses an atomic 60-second sending lease and
|
||||
allows three total attempts with 5/10-second retry delays. Pending and expired
|
||||
leases survive restart. A final expired sending lease becomes failed with an
|
||||
unknown-acknowledgement diagnostic. Existing retention deletes associated rows.
|
||||
`get_run` exposes run-level `notifications` receipts (sequence, event, channel,
|
||||
status, attempts, diagnostic, message_id), separate from step receipts.
|
||||
|
||||
Successful stage publications cover the corresponding waiting/failure alert;
|
||||
a successful stage targeting `#success` also covers final success. Failed stage
|
||||
publication does not swallow the run-level fallback/retry. Stage artifact
|
||||
publications themselves are not made retryable by this change.
|
||||
|
||||
Publisher acknowledgement is **not device reception**. A crash after publish
|
||||
but before the acknowledgement is saved can duplicate a notification. The
|
||||
transition and enqueue are separate writes: a crash between them can miss an
|
||||
alert; this is bounded retry of enqueued alerts, not guaranteed delivery.
|
||||
Browser delivery remains page-session best effort and requires permission plus
|
||||
HTTPS/localhost; successful runs do not become Needs you entries. The older L5
|
||||
parity description is attention-only for auto; on now adds success.
|
||||
|
||||
Tests use temporary databases and fake HTTP publishers only, covering modes,
|
||||
no-hook success, stage dedupe, failed publisher retry/exhaustion, restart/lease
|
||||
recovery, audit persistence, and browser off/on. No live notifications, deployed
|
||||
database/configuration, service restart, binary installation or OS reception
|
||||
verification was part of this work. No new UI captures were taken.
|
||||
@@ -621,6 +621,7 @@ func (s *Store) removeRetentionRuns(ctx context.Context, policy RetentionPolicy,
|
||||
func deleteRetentionRows(ctx context.Context, tx *sql.Tx, ids []int64) error {
|
||||
for _, table := range []struct{ name, column string }{
|
||||
{"step_notifications", "run_id"},
|
||||
{"run_notifications", "run_id"},
|
||||
{"step_file_changes", "run_id"},
|
||||
{"step_control", "run_id"},
|
||||
{"step_artifacts", "run_id"},
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
)
|
||||
|
||||
// RunNotification is durable publisher evidence, not proof of device reception.
|
||||
// A sending lease permits restart recovery; a crash after publish but before
|
||||
// acknowledgement can repeat a message. Attempts are bounded, not guaranteed.
|
||||
type RunNotification struct {
|
||||
RunID int64
|
||||
Sequence int
|
||||
Event, Channel, Title, Body, Click, Status, Diagnostic, MessageID string
|
||||
Attempts int
|
||||
}
|
||||
|
||||
func (s *Store) initRunNotifications() error {
|
||||
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS run_notifications (
|
||||
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, event TEXT NOT NULL, channel TEXT NOT NULL,
|
||||
title TEXT NOT NULL, body TEXT NOT NULL, click TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending', attempts INTEGER NOT NULL DEFAULT 0,
|
||||
next_attempt INTEGER NOT NULL DEFAULT 0, diagnostic TEXT NOT NULL DEFAULT '', message_id TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY(run_id,sequence,event,channel));
|
||||
CREATE INDEX IF NOT EXISTS run_notifications_due ON run_notifications(status,next_attempt);`)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) EnqueueRunNotification(ctx context.Context, n RunNotification) error {
|
||||
_, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO run_notifications
|
||||
(run_id,sequence,event,channel,title,body,click) VALUES (?,?,?,?,?,?,?)`, n.RunID, n.Sequence, n.Event, n.Channel, n.Title, n.Body, n.Click)
|
||||
return err
|
||||
}
|
||||
|
||||
// ClaimRunNotification atomically leases one due attempt for 60 seconds. Three
|
||||
// attempts total bound both transient failures and crash/restart loops.
|
||||
func (s *Store) ClaimRunNotification(ctx context.Context, now int64) (*RunNotification, error) {
|
||||
if _, err := s.db.ExecContext(ctx, `UPDATE run_notifications SET status='failed',diagnostic='publisher acknowledgement unknown after final lease expired' WHERE status='sending' AND attempts>=3 AND next_attempt<=?`, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n := new(RunNotification)
|
||||
err := s.db.QueryRowContext(ctx, `UPDATE run_notifications SET status='sending', attempts=attempts+1, next_attempt=?
|
||||
WHERE rowid=(SELECT rowid FROM run_notifications WHERE status IN ('pending','sending') AND attempts<3 AND next_attempt<=? ORDER BY next_attempt,rowid LIMIT 1)
|
||||
RETURNING run_id,sequence,event,channel,title,body,click,status,attempts`, now+60, now).Scan(&n.RunID, &n.Sequence, &n.Event, &n.Channel, &n.Title, &n.Body, &n.Click, &n.Status, &n.Attempts)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// CoverRunNotification records a successful stage publication covering the same
|
||||
// run event, without publishing a second final alert.
|
||||
func (s *Store) CoverRunNotification(ctx context.Context, id int64, sequence int, event string) error {
|
||||
_, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO run_notifications(run_id,sequence,event,channel,title,body,click,status) VALUES (?,?,?,'system','','','','covered')`, id, sequence, event)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) FinishRunNotification(ctx context.Context, n RunNotification, now int64, messageID string, failure error) error {
|
||||
status, diagnostic := "sent", ""
|
||||
if failure != nil {
|
||||
status, diagnostic = "pending", failure.Error()
|
||||
if n.Attempts >= 3 {
|
||||
status = "failed"
|
||||
}
|
||||
}
|
||||
_, err := s.db.ExecContext(ctx, `UPDATE run_notifications SET status=?,diagnostic=?,message_id=?,next_attempt=?
|
||||
WHERE run_id=? AND sequence=? AND event=? AND channel=? AND status='sending' AND attempts=?`, status, diagnostic, messageID, now+int64(n.Attempts*5), n.RunID, n.Sequence, n.Event, n.Channel, n.Attempts)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) RunNotifications(ctx context.Context, id int64) ([]RunNotification, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT run_id,sequence,event,channel,status,attempts,diagnostic,message_id FROM run_notifications WHERE run_id=? ORDER BY sequence,event,channel`, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
result := []RunNotification{}
|
||||
for rows.Next() {
|
||||
var n RunNotification
|
||||
if err := rows.Scan(&n.RunID, &n.Sequence, &n.Event, &n.Channel, &n.Status, &n.Attempts, &n.Diagnostic, &n.MessageID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, n)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunNotificationPersistenceLeasesAndBound(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "outbox.db")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n := RunNotification{RunID: 7, Sequence: 2, Event: "failed", Channel: "system", Title: "Failed", Body: "Run #7", Click: "pipelines://run/7"}
|
||||
if err := s.EnqueueRunNotification(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.EnqueueRunNotification(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := s.ClaimRunNotification(ctx, 100)
|
||||
if err != nil || first == nil || first.Attempts != 1 {
|
||||
t.Fatalf("first claim: %+v %v", first, err)
|
||||
}
|
||||
if other, err := s.ClaimRunNotification(ctx, 100); err != nil || other != nil {
|
||||
t.Fatalf("lease double claim: %+v %v", other, err)
|
||||
}
|
||||
// Reopen an already-existing schema, preserving an in-flight lease.
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err = Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
second, err := s.ClaimRunNotification(ctx, 161)
|
||||
if err != nil || second == nil || second.Attempts != 2 || second.Body != n.Body {
|
||||
t.Fatalf("recovery: %+v %v", second, err)
|
||||
}
|
||||
if err := s.FinishRunNotification(ctx, *second, 161, "", errors.New("publisher unavailable")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if other, err := s.ClaimRunNotification(ctx, 170); err != nil || other != nil {
|
||||
t.Fatalf("backoff ignored: %+v %v", other, err)
|
||||
}
|
||||
third, err := s.ClaimRunNotification(ctx, 171)
|
||||
if err != nil || third == nil || third.Attempts != 3 {
|
||||
t.Fatalf("third claim: %+v %v", third, err)
|
||||
}
|
||||
// Final attempt crashes: its expired lease must become failed, not retry forever.
|
||||
if other, err := s.ClaimRunNotification(ctx, 232); err != nil || other != nil {
|
||||
t.Fatalf("retry bound ignored: %+v %v", other, err)
|
||||
}
|
||||
got, err := s.RunNotifications(ctx, 7)
|
||||
if err != nil || len(got) != 1 || got[0].Status != "failed" || got[0].Attempts != 3 {
|
||||
t.Fatalf("audit: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNotificationStageCoverageSurvivesReopen(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "coverage.db")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.CoverRunNotification(ctx, 7, 1, "succeeded"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err = Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.EnqueueRunNotification(ctx, RunNotification{RunID: 7, Sequence: 1, Event: "succeeded", Channel: "system"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := s.ClaimRunNotification(ctx, 100); err != nil || got != nil {
|
||||
t.Fatalf("covered stage repeated: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNotificationMigrationPreservesExistingRun(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "existing.db")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Submit(ctx, "existing", Template([]string{"one"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := s.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := s.Get(ctx, items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, _, err := s.CreateRun(ctx, d, "existing-key", "workspace", false, nil, map[string]any{"prompt": "preserve me"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate the immediately preceding schema, which has no run outbox.
|
||||
if _, err := s.db.ExecContext(ctx, "DROP TABLE run_notifications"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err = Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
got, err := s.GetRun(ctx, id)
|
||||
if err != nil || got.Input["prompt"] != "preserve me" || got.Status != "queued" || got.NotifyMode != "auto" || len(got.Notifications) != 0 {
|
||||
t.Fatalf("migration changed run: %+v %v", got, err)
|
||||
}
|
||||
if err := s.EnqueueRunNotification(ctx, RunNotification{RunID: id, Sequence: 1, Event: "failed", Channel: "system"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -21,12 +21,12 @@ const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime
|
||||
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
|
||||
` + runStamps + `,r.diagnostic,
|
||||
CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END,
|
||||
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models
|
||||
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode
|
||||
` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version`
|
||||
|
||||
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
|
||||
var output, stageModels string
|
||||
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels); err != nil {
|
||||
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode); err != nil {
|
||||
return err
|
||||
}
|
||||
if stageModels != "" && stageModels != "{}" {
|
||||
@@ -201,6 +201,23 @@ strftime('%Y-%m-%dT%H:%M:%SZ',started_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ'
|
||||
// The run total is one pass over the steps already loaded above: it adds no
|
||||
// query and no transcript parse.
|
||||
r.Usage = protocol.TotalUsage(r.Steps)
|
||||
alerts, err := tx.QueryContext(ctx, `SELECT sequence,event,channel,status,attempts,diagnostic,message_id FROM run_notifications WHERE run_id=? ORDER BY sequence,event,channel`, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for alerts.Next() {
|
||||
var receipt protocol.RunNotificationReceipt
|
||||
if err := alerts.Scan(&receipt.Sequence, &receipt.Event, &receipt.Channel, &receipt.Status, &receipt.Attempts, &receipt.Diagnostic, &receipt.MessageID); err != nil {
|
||||
alerts.Close()
|
||||
return nil, err
|
||||
}
|
||||
r.Notifications = append(r.Notifications, receipt)
|
||||
}
|
||||
err = alerts.Err()
|
||||
alerts.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -54,6 +54,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := s.initRunNotifications(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := s.initSchedules(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
|
||||
@@ -153,7 +153,7 @@ type Message struct {
|
||||
// them; the same fields are echoed on the run record.
|
||||
Model string `json:"model,omitempty"`
|
||||
StageModels map[string]string `json:"stage_models,omitempty"`
|
||||
// NotifyMode is auto (definition), on (ensure run-level attention) or off
|
||||
// NotifyMode is auto (definition), on (terminal success/failure and attention) or off
|
||||
// (silence this run). It is optional so older clients keep auto.
|
||||
NotifyMode string `json:"notify_mode,omitempty"`
|
||||
// Empty (or "resume") retries the failed visit; "restart" starts at the
|
||||
|
||||
@@ -24,8 +24,9 @@ const (
|
||||
// from the frame instead of re-reading the whole list. They repeat on every
|
||||
// frame (pipeline identity never changes mid-run).
|
||||
type RunEventFrame struct {
|
||||
RunID int64 `json:"run_id"`
|
||||
Workspace string `json:"workspace,omitempty"`
|
||||
NotifyMode string `json:"notify_mode,omitempty"`
|
||||
RunID int64 `json:"run_id"`
|
||||
Workspace string `json:"workspace,omitempty"`
|
||||
// PipelineID, Name, Title and Version identify the run's pipeline and its
|
||||
// operator-facing title, so a new run's row can be built without a fetch.
|
||||
PipelineID int64 `json:"pipeline_id,omitempty"`
|
||||
|
||||
+25
-12
@@ -119,6 +119,7 @@ type SessionExport struct {
|
||||
}
|
||||
|
||||
type RunSummary struct {
|
||||
NotifyMode string `json:"notify_mode,omitempty"`
|
||||
ID int64 `json:"id"`
|
||||
PipelineID int64 `json:"pipeline_id"`
|
||||
Name string `json:"name"`
|
||||
@@ -349,18 +350,19 @@ type NotificationReceipt struct {
|
||||
|
||||
type RunDetail struct {
|
||||
RunSummary
|
||||
Graph Graph `json:"graph"`
|
||||
RetryOf int64 `json:"retry_of,omitempty"`
|
||||
RetryMode string `json:"retry_mode,omitempty"`
|
||||
RetrySource int64 `json:"retry_source,omitempty"`
|
||||
RetrySourceVersion int `json:"retry_source_version,omitempty"`
|
||||
RetrySourceHash string `json:"retry_source_hash,omitempty"`
|
||||
Workspace string `json:"workspace"`
|
||||
AllowTools bool `json:"allow_tools"`
|
||||
WritePaths []string `json:"write_paths"`
|
||||
Input map[string]any `json:"input"`
|
||||
Output map[string]any `json:"output"`
|
||||
Steps []RunStep `json:"steps"`
|
||||
Notifications []RunNotificationReceipt `json:"notifications,omitempty"`
|
||||
Graph Graph `json:"graph"`
|
||||
RetryOf int64 `json:"retry_of,omitempty"`
|
||||
RetryMode string `json:"retry_mode,omitempty"`
|
||||
RetrySource int64 `json:"retry_source,omitempty"`
|
||||
RetrySourceVersion int `json:"retry_source_version,omitempty"`
|
||||
RetrySourceHash string `json:"retry_source_hash,omitempty"`
|
||||
Workspace string `json:"workspace"`
|
||||
AllowTools bool `json:"allow_tools"`
|
||||
WritePaths []string `json:"write_paths"`
|
||||
Input map[string]any `json:"input"`
|
||||
Output map[string]any `json:"output"`
|
||||
Steps []RunStep `json:"steps"`
|
||||
// Usage is the totalled reported usage of the run's model stages, summed in
|
||||
// one pass over Steps. It is absent for a run with no model usage recorded
|
||||
// (including historical runs).
|
||||
@@ -372,3 +374,14 @@ type RunDetail struct {
|
||||
TitleSource string `json:"title_source,omitempty"`
|
||||
TitleModel string `json:"title_model,omitempty"`
|
||||
}
|
||||
|
||||
// RunNotificationReceipt records publisher attempts, not device reception.
|
||||
type RunNotificationReceipt struct {
|
||||
Sequence int `json:"sequence"`
|
||||
Event string `json:"event"`
|
||||
Channel string `json:"channel"`
|
||||
Status string `json:"status"`
|
||||
Attempts int `json:"attempts"`
|
||||
Diagnostic string `json:"diagnostic,omitempty"`
|
||||
MessageID string `json:"message_id,omitempty"`
|
||||
}
|
||||
|
||||
+80
-14
@@ -3,6 +3,7 @@ package server
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
"io"
|
||||
@@ -34,10 +35,6 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
|
||||
if !enabled {
|
||||
return
|
||||
}
|
||||
// The stage's own policy already covers this attention event, so the
|
||||
// run-level default must not repeat it. Recorded before delivery so a failed
|
||||
// stage notification is reported by its own receipt, not silently duplicated.
|
||||
r.markAttentionSent(id, step.Sequence, stageAttentionEvent(event))
|
||||
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
summary := fmt.Sprintf("%s · run #%d · stage %s · %s (%s)", name, id, step.Stage, event, step.Outcome)
|
||||
@@ -128,6 +125,18 @@ func (r *runner) notifyStep(id int64, name, workspace string, policy *protocol.N
|
||||
if !sentArtifact {
|
||||
send("", nil)
|
||||
}
|
||||
// Only a successful publish covers the run alert. A failed stage publisher
|
||||
// must not swallow the durable run-level fallback and its retries.
|
||||
for _, receipt := range receipts {
|
||||
if receipt.Status == "sent" {
|
||||
covered := stageAttentionEvent(event)
|
||||
if event == "success" && step.Target == protocol.SuccessEnd {
|
||||
covered = "succeeded"
|
||||
}
|
||||
r.markAttentionSent(id, step.Sequence, covered)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runAttentionEvent maps a run status to the operator attention notification it
|
||||
@@ -167,8 +176,16 @@ func (r *runner) markAttentionSent(id int64, sequence int, event string) {
|
||||
if event == "" {
|
||||
return
|
||||
}
|
||||
if r.store != nil {
|
||||
if err := r.store.CoverRunNotification(r.ctx, id, sequence, event); err != nil {
|
||||
log.Printf("record stage notification coverage run=%d: %v", id, err)
|
||||
}
|
||||
}
|
||||
r.attentionMu.Lock()
|
||||
defer r.attentionMu.Unlock()
|
||||
if r.attentionSent == nil {
|
||||
r.attentionSent = map[int64]map[string]bool{}
|
||||
}
|
||||
if r.attentionSent[id] == nil {
|
||||
r.attentionSent[id] = map[string]bool{}
|
||||
}
|
||||
@@ -206,6 +223,9 @@ func (r *runner) notifyMode(id int64) string {
|
||||
if r.store != nil {
|
||||
if stored, err := r.store.RunNotifyMode(r.ctx, id); err == nil && stored != "" {
|
||||
r.notifyMu.Lock()
|
||||
if r.notifyModes == nil {
|
||||
r.notifyModes = map[int64]string{}
|
||||
}
|
||||
r.notifyModes[id] = stored
|
||||
r.notifyMu.Unlock()
|
||||
return stored
|
||||
@@ -240,29 +260,32 @@ func runAttentionClick(id int64) string {
|
||||
// notifyStep and reuses the same PublishNtfy machinery, destination and token
|
||||
// handling, so no second publisher, topic or service is introduced. The run id
|
||||
// and pipeline name identify the run and the click action opens it in the app.
|
||||
// A run that needs nobody produces nothing, and a delivery failure is logged and
|
||||
// never fails the run: this is a best-effort alert, not persisted run evidence.
|
||||
// Attempts are persisted before publication. Publisher acknowledgement is not
|
||||
// proof of device reception; a crash in that acknowledgement window can repeat.
|
||||
func (r *runner) notifyRun(id int64, name, status string, sequence int, currentStage, diagnostic string) {
|
||||
if r.notifyMode(id) == "off" {
|
||||
return
|
||||
}
|
||||
event := runAttentionEvent(status)
|
||||
if status == "succeeded" && r.notifyMode(id) == "on" {
|
||||
event = "succeeded"
|
||||
}
|
||||
if event == "" {
|
||||
return
|
||||
}
|
||||
if r.attentionAlready(id, sequence, event) {
|
||||
return
|
||||
}
|
||||
// Record the delivery attempt before publishing so a repeated transition for
|
||||
// the same run and event (a re-request, a restart-free re-entry) never
|
||||
// republishes; delivery itself is never retried.
|
||||
r.markAttentionSent(id, sequence, event)
|
||||
title := "Run needs your decision"
|
||||
summary := fmt.Sprintf("Run #%d · %s · waiting for a human decision", id, name)
|
||||
if event == "failed" {
|
||||
title = "Run failed"
|
||||
summary = fmt.Sprintf("Run #%d · %s · run failed", id, name)
|
||||
}
|
||||
if event == "succeeded" {
|
||||
title = "Run succeeded"
|
||||
summary = fmt.Sprintf("Run #%d · %s · run succeeded", id, name)
|
||||
}
|
||||
if currentStage != "" {
|
||||
summary += " at stage " + currentStage
|
||||
}
|
||||
@@ -270,10 +293,53 @@ func (r *runner) notifyRun(id int64, name, status string, sequence int, currentS
|
||||
summary += ": " + diagnostic
|
||||
}
|
||||
summary = truncateUTF8(summary, 2000)
|
||||
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := stage.PublishNtfy(ctx, "system", title, summary, "", nil, runAttentionClick(id)); err != nil {
|
||||
log.Printf("run attention notification run=%d: %v", id, err)
|
||||
if err := r.store.EnqueueRunNotification(r.ctx, pipeline.RunNotification{RunID: id, Sequence: sequence, Event: event, Channel: "system", Title: title, Body: summary, Click: runAttentionClick(id)}); err != nil {
|
||||
log.Printf("enqueue run notification run=%d: %v", id, err)
|
||||
return
|
||||
}
|
||||
r.deliverRunNotificationBatch(time.Now(), 1)
|
||||
}
|
||||
|
||||
// resumeRunNotifications also recovers pending/expired leases after restart.
|
||||
func (r *runner) resumeRunNotifications() {
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-r.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
r.deliverRunNotifications(time.Now())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *runner) deliverRunNotifications(now time.Time) {
|
||||
r.deliverRunNotificationBatch(now, 32)
|
||||
}
|
||||
|
||||
func (r *runner) deliverRunNotificationBatch(now time.Time, limit int) {
|
||||
// Bound one drain so a queue cannot monopolize the runner.
|
||||
started := time.Now()
|
||||
for i := 0; i < limit; i++ {
|
||||
n, err := r.store.ClaimRunNotification(r.ctx, now.Add(time.Since(started)).Unix())
|
||||
if err != nil {
|
||||
log.Printf("claim run notification: %v", err)
|
||||
return
|
||||
}
|
||||
if n == nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Second)
|
||||
result, publishErr := stage.PublishNtfy(ctx, n.Channel, n.Title, n.Body, "", nil, n.Click)
|
||||
cancel()
|
||||
messageID, _ := result["message_id"].(string)
|
||||
save, done := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
err = r.store.FinishRunNotification(save, *n, now.Add(time.Since(started)).Unix(), messageID, publishErr)
|
||||
done()
|
||||
if err != nil {
|
||||
log.Printf("save run notification run=%d: %v", n.RunID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -121,7 +121,7 @@ func TestRunEventWebSocketPushesTransitions(t *testing.T) {
|
||||
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "sub" {
|
||||
t.Fatalf("subscribe ack = %+v", ack)
|
||||
}
|
||||
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
|
||||
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, NotifyMode: "off", Input: map[string]any{"prompt": "test"}})
|
||||
|
||||
var runID int64
|
||||
var created, stepStarted, stepFinished, terminal *protocol.RunEventFrame
|
||||
@@ -175,6 +175,11 @@ func TestRunEventWebSocketPushesTransitions(t *testing.T) {
|
||||
if terminal.RunID != runID || terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.FinishedAt == "" {
|
||||
t.Fatalf("terminal frame = %+v", terminal)
|
||||
}
|
||||
for _, frame := range []*protocol.RunEventFrame{created, stepStarted, stepFinished, terminal} {
|
||||
if frame.NotifyMode != "off" {
|
||||
t.Fatalf("silent mode lost in event: %+v", frame)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunEventSubscribeSendsSnapshot proves a client that subscribes to an
|
||||
@@ -188,7 +193,7 @@ func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
workspace := eventWorkspace(t)
|
||||
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
|
||||
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, NotifyMode: "off", Input: map[string]any{"prompt": "test"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -203,6 +208,9 @@ func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
|
||||
if first.RunEvent.RunID != run.ID || first.RunEvent.Status != "succeeded" || first.RunEvent.CompletedSteps != 1 || first.RunEvent.FinishedAt == "" {
|
||||
t.Fatalf("snapshot frame = %+v", first.RunEvent)
|
||||
}
|
||||
if first.RunEvent.NotifyMode != "off" {
|
||||
t.Fatalf("silent mode lost in snapshot: %+v", first.RunEvent)
|
||||
}
|
||||
// A snapshot for a run the client never saw also carries the identity its row
|
||||
// needs, so a fresh subscribe renders it without a second read.
|
||||
if first.RunEvent.PipelineID != ids[0] || first.RunEvent.Name == "" || first.RunEvent.Version != 1 {
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func TestRunNotificationPublisherFailureRetriesAfterReopen(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "retry.db")
|
||||
s, err := pipeline.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if calls.Add(1) == 1 {
|
||||
http.Error(w, "transient", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
|
||||
}))
|
||||
defer server.Close()
|
||||
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
||||
r := &runner{ctx: ctx, store: s}
|
||||
r.notifyRun(7, "test", "failed", 1, "stage", "")
|
||||
got, err := s.RunNotifications(ctx, 7)
|
||||
if err != nil || len(got) != 1 || got[0].Status != "pending" || got[0].Attempts != 1 || got[0].Diagnostic == "" {
|
||||
t.Fatalf("failed attempt not audited: %+v %v", got, err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err = pipeline.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
r = &runner{ctx: ctx, store: s}
|
||||
r.deliverRunNotifications(time.Now().Add(time.Minute))
|
||||
r.notifyRun(7, "test", "failed", 1, "stage", "")
|
||||
got, err = s.RunNotifications(ctx, 7)
|
||||
if err != nil || len(got) != 1 || got[0].Status != "sent" || got[0].Attempts != 2 || got[0].MessageID != "ack" || calls.Load() != 2 {
|
||||
t.Fatalf("retry/dedupe: %+v calls=%d %v", got, calls.Load(), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNotificationModesAndFinalStageDedupe(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"id": "ack", "event": "message"})
|
||||
}))
|
||||
defer server.Close()
|
||||
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
||||
for _, tc := range []struct {
|
||||
mode string
|
||||
hook bool
|
||||
want int32
|
||||
}{
|
||||
{"on", false, 1}, {"auto", false, 0}, {"off", false, 0}, {"on", true, 1}, {"auto", true, 1}, {"off", true, 0},
|
||||
} {
|
||||
t.Run(tc.mode+map[bool]string{true: "-hook", false: "-no-hook"}[tc.hook], func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
s, err := pipeline.Open(filepath.Join(t.TempDir(), "modes.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
source := pipeline.Template([]string{"one"}, true)
|
||||
if tc.hook {
|
||||
source = strings.Replace(source, "# stage one", "# stage one\n```notify\non=success\n```", 1)
|
||||
}
|
||||
if _, err := s.Submit(ctx, "mode-test", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := s.List(ctx)
|
||||
r, err := newRunner(ctx, s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{Outcome: "success"}, nil
|
||||
}
|
||||
before := calls.Load()
|
||||
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mode", NotifyMode: tc.mode, Input: map[string]any{"prompt": "task"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
detail, err := s.GetRun(ctx, run.ID)
|
||||
if err != nil || detail.Status != "succeeded" || detail.NotifyMode != tc.mode || calls.Load()-before != tc.want {
|
||||
t.Fatalf("mode contract: %+v calls=%d %v", detail, calls.Load()-before, err)
|
||||
}
|
||||
if tc.mode == "on" && (len(detail.Notifications) != 1 || (detail.Notifications[0].Status != "sent" && detail.Notifications[0].Status != "covered")) {
|
||||
t.Fatalf("missing final audit: %+v", detail.Notifications)
|
||||
}
|
||||
cancel()
|
||||
r.schedulerWG.Wait()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNotificationAttentionModesAndRetryBound(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := pipeline.Open(filepath.Join(t.TempDir(), "attention.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
var calls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
http.Error(w, "unavailable", http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer server.Close()
|
||||
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
||||
r := &runner{ctx: ctx, store: s}
|
||||
for i, mode := range []string{"auto", "on", "off"} {
|
||||
id := int64(i + 1)
|
||||
r.setNotifyMode(id, mode)
|
||||
for sequence, status := range []string{"waiting", "failed"} {
|
||||
r.notifyRun(id, "test", status, sequence, "stage", "")
|
||||
}
|
||||
}
|
||||
if calls.Load() != 4 {
|
||||
t.Fatalf("attention modes: %d calls, want 4", calls.Load())
|
||||
}
|
||||
r.deliverRunNotifications(time.Now().Add(time.Minute))
|
||||
r.deliverRunNotifications(time.Now().Add(2 * time.Minute))
|
||||
r.deliverRunNotifications(time.Now().Add(3 * time.Minute))
|
||||
if calls.Load() != 12 {
|
||||
t.Fatalf("retry bound: %d calls, want 12", calls.Load())
|
||||
}
|
||||
for _, id := range []int64{1, 2, 3} {
|
||||
got, err := s.RunNotifications(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id == 3 && len(got) != 0 {
|
||||
t.Fatalf("off created attempts: %+v", got)
|
||||
}
|
||||
for _, n := range got {
|
||||
if n.Status != "failed" || n.Attempts != 3 {
|
||||
t.Fatalf("missing exhausted audit: %+v", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNotificationWorkerResumesPendingAfterRestart(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
path := filepath.Join(t.TempDir(), "worker.db")
|
||||
s, err := pipeline.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.EnqueueRunNotification(ctx, pipeline.RunNotification{RunID: 7, Sequence: 1, Event: "failed", Channel: "system", Title: "Run failed", Body: "pending"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err = pipeline.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
delivered := make(chan struct{}, 2)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"id": "recovered", "event": "message"})
|
||||
delivered <- struct{}{}
|
||||
}))
|
||||
defer server.Close()
|
||||
t.Setenv("PIPELINES_NTFY_URL", server.URL+"/fake")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
||||
r, err := newRunner(ctx, s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { cancel(); r.schedulerWG.Wait() }()
|
||||
select {
|
||||
case <-delivered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("pending outbox was not recovered")
|
||||
}
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
got, err := s.RunNotifications(ctx, 7)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) == 1 && got[0].Status == "sent" && got[0].MessageID == "recovered" {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("recovery audit: %+v", got)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -424,6 +424,9 @@ func (s *runSubscription) close() {
|
||||
}
|
||||
|
||||
func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
|
||||
if in.NotifyMode != "" {
|
||||
into.NotifyMode = in.NotifyMode
|
||||
}
|
||||
if in.Workspace != "" {
|
||||
into.Workspace = in.Workspace
|
||||
}
|
||||
@@ -500,6 +503,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
|
||||
continue
|
||||
}
|
||||
frame := protocol.RunEventFrame{
|
||||
NotifyMode: summary.NotifyMode,
|
||||
RunID: summary.ID,
|
||||
Workspace: summary.Workspace,
|
||||
PipelineID: summary.PipelineID,
|
||||
|
||||
@@ -101,7 +101,10 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
||||
return nil, err
|
||||
}
|
||||
root := filepath.Join(s.Directory(), "artifacts")
|
||||
return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}, nil
|
||||
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}
|
||||
r.schedulerWG.Add(1)
|
||||
go func() { defer r.schedulerWG.Done(); r.resumeRunNotifications() }()
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
|
||||
@@ -1257,6 +1260,7 @@ func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) {
|
||||
return
|
||||
}
|
||||
r.stampRunIdentity(&frame)
|
||||
frame.NotifyMode = r.notifyMode(frame.RunID)
|
||||
r.events.emit(frame, terminal)
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -166,7 +166,7 @@ to it, but the server never lists it and a reload drops it.
|
||||
| B3 | Equal phone dock cells | — | done — the narrow dock is a four-cell grid (one cell per destination plus Search) with a subgrid for the tablist, so every control has the same width and height. At 390×844 the measured widths are 92/92/92/92 and the row is 48px high. The phone shell uses `100svh` so a browser toolbar cannot hide the tab row while leaving only the Connected/Settings status row visible. |
|
||||
| B4 | Full-height pipeline Source | — | done — the Source block no longer caps itself at `60vh`; it is as tall as its content and the detail pane owns the scroll. Verified at 390×844: a 4386px definition sits in the full pane (656px viewport) and scrolls. |
|
||||
| B14 | Run a pipeline from the web | `run_pipeline` | done — the pipeline detail has a Run action that opens a dialog with the prompt, an absolute server working-directory field with recent-directory suggestions, and a per-run notification choice (pipeline default / notify / silent). The request carries `pipeline_id`, the pinned version, a fresh request key, `workspace`, `input.prompt` and `notify_mode` only when overriding. A live probe started a real run from the dialog with `notify_mode=off` and the server stored that mode; the run was canceled. |
|
||||
| B15 | Per-run notification override | `notify_mode` | done — `pipelines run --notify auto\|on\|off` (and the same option on retry) records the override with the run. Off suppresses both stage policies and run-level waiting/failed alerts; auto follows the definition; on keeps run-level attention. The mode is stored in `pipeline_runs`, reloaded after a restart, and inherited by a retry unless overridden. |
|
||||
| B15 | Per-run notification override | `notify_mode` | done — `pipelines run --notify auto\|on\|off` (and the same option on retry) records the override with the run. Off suppresses stage, run-level and browser tray notifications (Needs you still reflects run state). Auto retains existing definition plus waiting/failed defaults; on additionally alerts terminal success, including runs without notify hooks. Mode travels in summaries/events and is inherited by retries unless overridden. Run-level system publications use a durable bounded-retry outbox and publisher receipts; stage final success coverage avoids a second system alert. Browser mode behavior is unit-tested, not OS-observed; no new UI captures in this audit. |
|
||||
| B16 | One live status line for tools | — | done — the current tool call now shares the `N tools called` line; the separate bottom per-tool line is gone and the lower indicator appears only while no tool is running. The live reasoning label's duration ticks every second without waiting for a frame. Chat frames for a background session no longer repaint a Runs/Pipelines screen: verified a synthetic stream while opening run 12's detail left the run screen interactive with no per-frame replacements. |
|
||||
| B17 | Clear list chrome and chat recency | — | done — list pages no longer repeat the destination title/subtitle from the rail/dock, and the manual Refresh controls are gone; the visible list refreshes once when the page regains focus, preserving its scroll position and leaving open menus, dialogs and text fields alone. Chat rows replace the meaningless `Idle`/`Open` status with the server's latest-update recency (`just now`, `15 min ago`, `2 days ago`), keep `Streaming`/`New`/`Closed`, and give each directory a full-width elevated band with centred text that expands/collapses exactly like Closed (square corners, not a rounded chip), abbreviating the server home prefix to `~` (`~/repos/pipelines`) in both the band and the row meta. `New chat` is now a filled primary control beside the search field. Runs and Pipelines use the same title-free list header. Verified at 390×844 and 1440×900 in a real browser, with no console errors; the web unit suite is 633 tests. |
|
||||
| B18 | Chat attachments in the message and paste | `chat_prompt.attachments` | done — composer files are read into base64 and sent with the prompt (1 MiB total, under the 2 MiB websocket frame), the server writes them to private temporary files and passes `file://` references to OpenCode's prompt `files`, and both the live user turn and reloaded history render a named file chip inside the message. Pasting an image/file into the textarea attaches it with the same picker path; a pasted image without a filename gets a generated name. The optimistic user bubble shows the file immediately, and OpenCode history maps the top-level user `files` envelope (never its base64 data) into the same part. |
|
||||
|
||||
+20
-6
@@ -1,6 +1,5 @@
|
||||
// Pure, DOM-free notification model for failed/waiting runs (L5). It mirrors
|
||||
// android/.../notify/RunAlerts.kt: the same two qualifying kinds, the same
|
||||
// headlines, the same run line and the same one-notification-per-run+kind rule.
|
||||
// Pure, DOM-free notification model. Auto retains failed/waiting alerts;
|
||||
// per-run on also alerts terminal success and off suppresses every tray alert.
|
||||
//
|
||||
// Everything here is pure: no DOM, no clock, no storage access. The page owns
|
||||
// the Notification constructor, the permission request gesture and localStorage;
|
||||
@@ -13,14 +12,16 @@ import { AttentionKind, attentionKey, attentionKindOf } from "./attention.js";
|
||||
// Kinds, headlines and the run line
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** The two run events that raise a notification (RunAlerts.kt RunAlertKind). */
|
||||
/** Notification kinds; success does not create a Needs you entry. */
|
||||
export const RunAlertKind = Object.freeze({
|
||||
Decision: AttentionKind.Decision,
|
||||
Failed: AttentionKind.Failed,
|
||||
Succeeded: Object.freeze({ name: "Succeeded" }),
|
||||
});
|
||||
|
||||
/** `alertKindOf`: an attention kind is already exactly the notification kind. */
|
||||
export function alertKindOf(kind) {
|
||||
if (kind?.name === "Succeeded") return RunAlertKind.Succeeded;
|
||||
if (kind === AttentionKind.Decision || kind?.name === AttentionKind.Decision.name) {
|
||||
return AttentionKind.Decision;
|
||||
}
|
||||
@@ -37,6 +38,8 @@ export function alertTitle(kind) {
|
||||
return "A run needs your decision";
|
||||
case "Failed":
|
||||
return "A run failed";
|
||||
case "Succeeded":
|
||||
return "A run succeeded";
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
@@ -90,15 +93,18 @@ export function alertBody(run) {
|
||||
*/
|
||||
export function createAlertLedger() {
|
||||
const lastStatus = new Map();
|
||||
const modes = new Map();
|
||||
const notified = new Set();
|
||||
let seeding = false;
|
||||
|
||||
function seed(runs) {
|
||||
const next = new Map();
|
||||
modes.clear();
|
||||
for (const run of runs ?? []) {
|
||||
const runId = runIdOf(run?.id ?? run?.run_id);
|
||||
if (runId == null) continue;
|
||||
next.set(runId, normalizeStatus(run?.status));
|
||||
modes.set(runId, normalizeStatus(run?.notify_mode) || "auto");
|
||||
}
|
||||
lastStatus.clear();
|
||||
for (const [runId, status] of next) lastStatus.set(runId, status);
|
||||
@@ -113,16 +119,23 @@ export function createAlertLedger() {
|
||||
const previous = lastStatus.get(runId);
|
||||
lastStatus.set(runId, status);
|
||||
|
||||
const kind = attentionKindOf(status);
|
||||
if (frame?.notify_mode != null) modes.set(runId, normalizeStatus(frame.notify_mode));
|
||||
const mode = modes.get(runId) || "auto";
|
||||
|
||||
const kind = status === "succeeded" && mode === "on" ? RunAlertKind.Succeeded : attentionKindOf(status);
|
||||
if (!kind) {
|
||||
forget(runId, AttentionKind.Decision);
|
||||
forget(runId, AttentionKind.Failed);
|
||||
forget(runId, RunAlertKind.Succeeded);
|
||||
return null;
|
||||
}
|
||||
// A run can only be in one qualifying state; forget the other kind's key so
|
||||
// a later transition back into it is a new event.
|
||||
forget(runId, kind === AttentionKind.Decision ? AttentionKind.Failed : AttentionKind.Decision);
|
||||
for (const other of Object.values(RunAlertKind)) {
|
||||
if (other !== kind) forget(runId, other);
|
||||
}
|
||||
if (seeding) return null;
|
||||
if (mode === "off") return null;
|
||||
if (previous === undefined || previous === status) return null;
|
||||
|
||||
const key = alertKey(runId, kind);
|
||||
@@ -143,6 +156,7 @@ export function createAlertLedger() {
|
||||
|
||||
function reset() {
|
||||
lastStatus.clear();
|
||||
modes.clear();
|
||||
seeding = false;
|
||||
}
|
||||
|
||||
|
||||
@@ -172,6 +172,7 @@ function sameResourceWait(a, b) {
|
||||
/** The run-state fields a frame advances, shared by the summary row and the detail. */
|
||||
function runStateFields(current) {
|
||||
return {
|
||||
notify_mode: current.notify_mode ?? "auto",
|
||||
resource_wait: current.resource_wait ?? null,
|
||||
status: current.status ?? "",
|
||||
outcome: current.outcome ?? "",
|
||||
@@ -196,6 +197,7 @@ function applyRunStateFields(current, frame) {
|
||||
? "queued"
|
||||
: advanceRunStatus(before.status, frame.status);
|
||||
const merged = {
|
||||
notify_mode: blank(frame.notify_mode) ? before.notify_mode : frame.notify_mode,
|
||||
resource_wait: isTerminalRunStatus(status)
|
||||
? null
|
||||
: frame.resource_wait_set === true
|
||||
@@ -210,6 +212,7 @@ function applyRunStateFields(current, frame) {
|
||||
executing: executingFor(status, before.executing),
|
||||
};
|
||||
const changed =
|
||||
merged.notify_mode !== before.notify_mode ||
|
||||
!sameResourceWait(merged.resource_wait, before.resource_wait) ||
|
||||
merged.status !== before.status ||
|
||||
merged.outcome !== before.outcome ||
|
||||
@@ -236,6 +239,7 @@ export function newRunSummaryFromFrame(frame) {
|
||||
id: Number(frame.run_id),
|
||||
pipeline_id: pipelineId,
|
||||
name,
|
||||
notify_mode: frame.notify_mode ?? "auto",
|
||||
title: blank(frame.title) ? null : frame.title,
|
||||
version: Number(frame.version) || 0,
|
||||
status: frame.status ?? "",
|
||||
|
||||
@@ -31,6 +31,26 @@ const frame = (runId, status, extra = {}) => ({
|
||||
...extra,
|
||||
});
|
||||
|
||||
test("off suppresses every browser alert from summaries and pushed modes", () => {
|
||||
for (const status of ["waiting", "failed", "succeeded"]) {
|
||||
const ledger = createAlertLedger();
|
||||
ledger.seed([{ id: 1, status: "running", notify_mode: "off" }]);
|
||||
assert.equal(ledger.observe(frame(1, status)), null);
|
||||
ledger.seed([{ id: 1, status: "running", notify_mode: "on" }]);
|
||||
assert.equal(ledger.observe(frame(1, status, { notify_mode: "off" })), null);
|
||||
}
|
||||
});
|
||||
|
||||
test("on adds final success once while auto retains attention-only semantics", () => {
|
||||
const ledger = createAlertLedger();
|
||||
ledger.seed([{ id: 1, status: "running", notify_mode: "on" }, { id: 2, status: "running", notify_mode: "auto" }]);
|
||||
const alert = ledger.observe(frame(1, "succeeded"));
|
||||
assert.equal(alert.title, "A run succeeded");
|
||||
assert.equal(alert.key, "1:Succeeded");
|
||||
assert.equal(ledger.observe(frame(1, "succeeded")), null);
|
||||
assert.equal(ledger.observe(frame(2, "succeeded")), null);
|
||||
});
|
||||
|
||||
test("headlines are exactly the app's two, and unknown kinds are blank", () => {
|
||||
assert.equal(alertTitle(AttentionKind.Failed), "A run failed");
|
||||
assert.equal(alertTitle(AttentionKind.Decision), "A run needs your decision");
|
||||
|
||||
@@ -87,6 +87,16 @@ function frame({
|
||||
|
||||
// --- lifecycle vocabulary ----------------------------------------------------
|
||||
|
||||
test("run summaries and detail preserve notification mode from live frames", () => {
|
||||
const incoming = { ...frame({ pipelineId: 4, name: "test", status: "queued" }), notify_mode: "off" };
|
||||
assert.equal(newRunSummaryFromFrame(incoming).notify_mode, "off");
|
||||
const runs = [summary(1)];
|
||||
const folded = foldRunEvent(runs, { ...incoming, status: "running" });
|
||||
assert.equal(folded.runs[0].notify_mode, "off");
|
||||
const result = foldRunEventDetail(detail({status:"running"}), { ...incoming, status: "failed" });
|
||||
assert.equal(result.notify_mode, "off");
|
||||
});
|
||||
|
||||
test("the lifecycle ladders mirror RunEvents.kt", () => {
|
||||
assert.deepEqual(TERMINAL_RUN_STATUSES, ["succeeded", "failed", "canceled", "interrupted"]);
|
||||
assert.equal(runLifecycleRank("queued"), 0);
|
||||
|
||||
Reference in New Issue
Block a user