a733a927f3
Preserve Settings as an independent navigation destination while landing the reviewed notification and skill-management updates. Keep the server and CLI skill contracts versioned and validated so the merged tree can be verified by the delivery gate.
251 lines
9.6 KiB
Go
251 lines
9.6 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
func notificationTransitionRun(t *testing.T, path, mode string, sources ...string) (*Store, int64) {
|
|
t.Helper()
|
|
ctx := context.Background()
|
|
s, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
source := humanSource
|
|
if len(sources) > 0 {
|
|
source = sources[0]
|
|
}
|
|
if _, err := s.Submit(ctx, "stored pipeline", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := s.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d, err := s.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
id, _, err := s.CreateRun(ctx, d, "transition", "workspace", false, nil, map[string]any{"prompt": "task"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.db.ExecContext(ctx, `UPDATE pipeline_runs SET notify_mode=? WHERE id=?`, mode, id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.BeginStep(ctx, id, 1, "gate", map[string]any{"prompt": "task"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return s, id
|
|
}
|
|
|
|
func TestWaitingHookIntentRecoversWithoutPostCommitCallback(t *testing.T) {
|
|
ctx := context.Background()
|
|
path := filepath.Join(t.TempDir(), "lost-hook.db")
|
|
source := strings.Replace(humanSource, "# stage gate", "# stage gate\n```notify\non=waiting\n```", 1)
|
|
s, id := notificationTransitionRun(t, path, "on", source)
|
|
if err := notificationTransition(ctx, s, id, "waiting"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n, err := s.ClaimRunNotification(ctx, time.Now().Unix()); err != nil || n != nil {
|
|
t.Fatalf("worker raced hook opportunity: %+v %v", n, err)
|
|
}
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
n, err := s.ClaimRunNotification(ctx, time.Now().Add(time.Minute+time.Second).Unix())
|
|
if err != nil || n == nil || n.RunID != id || n.Event != "waiting" {
|
|
t.Fatalf("lost hook orphaned waiting alert: %+v %v", n, err)
|
|
}
|
|
}
|
|
|
|
func notificationTransition(ctx context.Context, s *Store, id int64, status string) error {
|
|
if status == "waiting" {
|
|
return s.MarkWaiting(ctx, id, 1, "gate", protocol.NewWaitingEvidence("Choose", []string{"deploy"}))
|
|
}
|
|
return s.FinishRun(ctx, id, status, "success", "stored diagnostic", map[string]any{"markdown": "result"})
|
|
}
|
|
|
|
func TestRunNotificationTransitionIntentAtomicRollbackAndRecovery(t *testing.T) {
|
|
for _, status := range []string{"succeeded", "failed", "waiting"} {
|
|
t.Run(status, func(t *testing.T) {
|
|
ctx := context.Background()
|
|
path := filepath.Join(t.TempDir(), "atomic.db")
|
|
s, id := notificationTransitionRun(t, path, "on")
|
|
// Force enqueue to fail after both state writes would have happened.
|
|
if _, err := s.db.ExecContext(ctx, `CREATE TRIGGER reject_notification BEFORE INSERT ON run_notifications BEGIN SELECT RAISE(ABORT,'forced enqueue failure'); END`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := notificationTransition(ctx, s, id, status); err == nil || !strings.Contains(err.Error(), "forced enqueue failure") {
|
|
t.Fatalf("enqueue failure missing: %v", err)
|
|
}
|
|
got, err := s.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Status != "running" || got.FinishedAt != "" || got.Diagnostic != "" || got.Outcome != "" || len(got.Output) != 0 || got.Steps[0].Status != "running" || len(got.Steps[0].Output) != 0 || len(got.Notifications) != 0 {
|
|
t.Fatalf("partial transition committed: %+v", got)
|
|
}
|
|
if _, err := s.db.ExecContext(ctx, `DROP TRIGGER reject_notification`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := notificationTransition(ctx, s, id, status); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Simulate a crash immediately after commit: no runner enqueue/delivery.
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err = Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
got, err = s.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Status != status || len(got.Notifications) != 1 || got.Notifications[0].Status != "pending" {
|
|
t.Fatalf("orphaned state after reopen: %+v", got)
|
|
}
|
|
n, err := s.ClaimRunNotification(ctx, 100)
|
|
if err != nil || n == nil {
|
|
t.Fatalf("intent not recoverable: %+v %v", n, err)
|
|
}
|
|
if n.RunID != id || n.Sequence != 1 || n.Event != status || n.Channel != "system" || !strings.Contains(n.Body, "stored pipeline") || !strings.Contains(n.Body, "gate") {
|
|
t.Fatalf("intent not derived from stored run: %+v", n)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunNotificationTransitionModesAndCoverage(t *testing.T) {
|
|
for _, mode := range []string{"auto", "on", "off"} {
|
|
for _, status := range []string{"succeeded", "failed", "waiting"} {
|
|
t.Run(mode+"/"+status, func(t *testing.T) {
|
|
ctx := context.Background()
|
|
s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "mode.db"), mode)
|
|
defer s.Close()
|
|
if err := notificationTransition(ctx, s, id, status); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ns, err := s.RunNotifications(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Failed runs always notify in auto mode; successful runs require
|
|
// "on" or run_finished. Waiting notifies auto and on.
|
|
want := 1
|
|
if mode == "off" || (mode == "auto" && status == "succeeded") {
|
|
want = 0
|
|
}
|
|
if len(ns) != want {
|
|
t.Fatalf("mode %s status %s: %+v", mode, status, ns)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
for _, mode := range []string{"on", "auto"} {
|
|
ctx := context.Background()
|
|
s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), mode+"-covered.db"), mode)
|
|
defer s.Close()
|
|
if err := s.CoverRunNotification(ctx, id, 1, "succeeded"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := notificationTransition(ctx, s, id, "succeeded"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ns, err := s.RunNotifications(ctx, id)
|
|
if err != nil || len(ns) != 1 || ns[0].Status != "covered" {
|
|
t.Fatalf("stage coverage lost: %+v %v", ns, err)
|
|
}
|
|
if n, err := s.ClaimRunNotification(ctx, 100); err != nil || n != nil {
|
|
t.Fatalf("covered stage duplicated: %+v %v", n, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestFailedRunNotificationClassifiesExpectedAndUnexpectedCauses(t *testing.T) {
|
|
tests := []struct {
|
|
name, status, outcome, diagnostic string
|
|
expectedKind string
|
|
notify bool
|
|
completeStep bool
|
|
control *protocol.ControlEvidence
|
|
}{
|
|
{name: "blocked with reason", status: "failed", outcome: "blocked", diagnostic: "Blocked: approval is required", expectedKind: "expected", notify: false, completeStep: true},
|
|
{name: "harness error", status: "failed", outcome: "error", diagnostic: "agent exception: process exited", expectedKind: "unexpected", notify: true},
|
|
{name: "budget exceeded", status: "failed", outcome: "error", diagnostic: "token budget exceeded", expectedKind: "unexpected", notify: true},
|
|
{name: "timeout", status: "failed", outcome: "error", diagnostic: "stage timed out", expectedKind: "unexpected", notify: true},
|
|
{name: "interrupted", status: "interrupted", outcome: "", diagnostic: "Server stopped; run was not resumed", expectedKind: "unexpected", notify: true},
|
|
{name: "visit limit", status: "failed", outcome: "error", diagnostic: "stopped stage after 10 visits", expectedKind: "unexpected", notify: true},
|
|
{name: "blocked reason mentions stopping", status: "failed", outcome: "blocked", diagnostic: "Blocked: stopped until approval", expectedKind: "expected", notify: false, completeStep: true},
|
|
{name: "visit limit from control evidence", status: "failed", outcome: "rejected", diagnostic: "Pipelines stopped review after 3 visits on outcome rejected", expectedKind: "unexpected", notify: true, completeStep: true, control: &protocol.ControlEvidence{Visit: 3, MaxVisits: 3, LimitReached: true}},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
ctx := context.Background()
|
|
s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "classification.db"), "auto")
|
|
defer s.Close()
|
|
if test.completeStep {
|
|
if err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "gate", Status: "completed", Outcome: test.outcome, Target: protocol.FailureEnd, Diagnostic: test.diagnostic, Control: test.control}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := s.FinishRun(ctx, id, test.status, test.outcome, test.diagnostic, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, err := s.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.FailureKind != test.expectedKind || got.FailureReason != test.diagnostic {
|
|
t.Fatalf("classification: kind=%q reason=%q, want %q/%q", got.FailureKind, got.FailureReason, test.expectedKind, test.diagnostic)
|
|
}
|
|
ns, err := s.RunNotifications(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if (len(ns) != 0) != test.notify {
|
|
t.Fatalf("notifications=%+v, want notify=%v", ns, test.notify)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestWaitingNotificationStageAndWorkerShareLease(t *testing.T) {
|
|
ctx := context.Background()
|
|
s, id := notificationTransitionRun(t, filepath.Join(t.TempDir(), "lease.db"), "on")
|
|
defer s.Close()
|
|
if err := notificationTransition(ctx, s, id, "waiting"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lease, allowed, err := s.ClaimStageRunNotification(ctx, id, 1, "waiting", 100)
|
|
if err != nil || !allowed || lease == nil {
|
|
t.Fatalf("stage claim: %+v %v %v", lease, allowed, err)
|
|
}
|
|
if n, err := s.ClaimRunNotification(ctx, 101); err != nil || n != nil {
|
|
t.Fatalf("worker duplicated stage: %+v %v", n, err)
|
|
}
|
|
if err := s.CoverRunNotification(ctx, id, 1, "waiting"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n, err := s.ClaimRunNotification(ctx, 161); err != nil || n != nil {
|
|
t.Fatalf("covered stage retried: %+v %v", n, err)
|
|
}
|
|
if _, allowed, err := s.ClaimStageRunNotification(ctx, id, 1, "waiting", 161); err != nil || allowed {
|
|
t.Fatalf("stage repeated: %v %v", allowed, err)
|
|
}
|
|
}
|