package server import ( "context" "database/sql" "errors" "fmt" "path/filepath" "strings" "sync/atomic" "testing" "time" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) func resourceRunner(t *testing.T) (*runner, <-chan protocol.RunEventFrame, func(string, string, string) *protocol.RunDetail) { t.Helper() ctx, cancel := context.WithCancel(context.Background()) s, err := pipeline.Open(filepath.Join(t.TempDir(), "resources.db")) if err != nil { t.Fatal(err) } r, err := newRunner(ctx, s) if err != nil { t.Fatal(err) } r.events = newRunEventHub() r.events.interval = time.Millisecond events := make(chan protocol.RunEventFrame, 128) sub := r.events.subscribe("resource-test", "", nil, func(m protocol.Message) error { if m.RunEvent != nil { events <- *m.RunEvent } return nil }) sub.ready() t.Cleanup(func() { cancel(); r.wg.Wait(); sub.close(); s.Close() }) count := 0 return r, events, func(resource, timeout, prompt string) *protocol.RunDetail { count++ source := strings.Replace(pipeline.Template([]string{"work"}, true), "timeout=1h", "timeout="+timeout, 1) if resource != "" { source += "```resources\n{\"exclusive\":\"" + resource + "\"}\n```\n" } name := fmt.Sprint("resource-", count) if _, err := s.Submit(ctx, name, source); err != nil { t.Fatal(err) } items, _ := s.List(ctx) var pid int64 for _, p := range items { if p.Name == name { pid = p.ID } } d, err := r.start(protocol.Message{PipelineID: pid, Version: 1, RequestKey: name, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}}) if err != nil { t.Fatal(err) } return d } } func awaitResourceEvent(t *testing.T, events <-chan protocol.RunEventFrame, id int64, predicate func(protocol.RunEventFrame) bool) protocol.RunEventFrame { t.Helper() timer := time.NewTimer(3 * time.Second) defer timer.Stop() for { select { case event := <-events: if event.RunID == id && predicate(event) { return event } case <-timer.C: t.Fatalf("resource event missing for run %d", id) return protocol.RunEventFrame{} } } } func TestResourcePauseBeyondTimeoutAndAutomaticResume(t *testing.T) { r, events, start := resourceRunner(t) aStarted, releaseA := make(chan struct{}), make(chan struct{}) bStarted := make(chan struct{}, 1) var bCalls atomic.Int32 r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { switch input["prompt"] { case "A": close(aStarted) select { case <-releaseA: case <-ctx.Done(): return stage.Result{}, ctx.Err() } case "B": bCalls.Add(1) deadline, ok := ctx.Deadline() if !ok || time.Until(deadline) < 15*time.Millisecond { return stage.Result{}, errors.New("execution timeout was consumed in queue") } bStarted <- struct{}{} } return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } a := start("X", "2s", "A") <-aStarted b := start("X", "30ms", "B") awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) <-time.After(70 * time.Millisecond) // Deliberately longer than B's whole execution timeout. d, err := r.store.GetRun(context.Background(), b.ID) if err != nil || d.Status != protocol.RunStatusQueued || d.ResourceWait == nil || len(d.Steps) != 0 || bCalls.Load() != 0 { t.Fatal("queued execution consumed accounting", d, err, bCalls.Load()) } if _, err := r.decide(protocol.Message{RunID: b.ID, Option: "success"}); err == nil { t.Fatal("resource wait exposed a human decision") } c := start("Y", "1s", "C") awaitResourceEvent(t, events, c.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) select { case <-bStarted: t.Fatal("B ran while A held X") default: } owner, err := r.store.GetRun(context.Background(), a.ID) if err != nil || owner.Status != "running" { t.Fatal("A did not remain owner", err) } close(releaseA) awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) r.wg.Wait() d, err = r.store.GetRun(context.Background(), b.ID) if err != nil || bCalls.Load() != 1 || d.ResourceWait != nil || len(d.Steps) != 1 || d.Steps[0].Control.Visit != 1 || d.Steps[0].Sequence != 1 { t.Fatal("resume did not execute exactly one fresh visit", d, err) } } func TestResourcePoolRunnerHandoffUsesConcreteTargetAfterQueue(t *testing.T) { r, events, _ := resourceRunner(t) ctx := context.Background() source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"pool\":\"targets\"}\n```\n" if _, err := r.store.Submit(ctx, "pool-runner", source); err != nil { t.Fatal(err) } items, err := r.store.List(ctx) if err != nil || len(items) != 1 { t.Fatal("pool definition missing", err) } definition, err := r.store.Get(ctx, items[0].ID, 1) if err != nil { t.Fatal(err) } if err := r.store.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{ {Member: "phone", Enabled: true, Metadata: map[string]string{"serial": "phone-serial", "type": "phone"}}, {Member: "emulator", Enabled: true, Metadata: map[string]string{"serial": "emu-serial", "type": "emulator"}}, }}); err != nil { t.Fatal(err) } ownerRun, _, err := r.store.CreateRun(ctx, definition, "phone-owner", t.TempDir(), false, nil, nil) if err != nil { t.Fatal(err) } phone, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "phone", RunID: ownerRun, Stage: "work", Sequence: 1}) if err != nil || ch != nil { t.Fatalf("phone owner acquire: channel=%v err=%v", ch, err) } defer r.releaseResource(phone, true) aStarted, releaseA := make(chan struct{}), make(chan struct{}) bRan := make(chan *protocol.ResourceTarget, 1) r.execute = func(stageCtx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { target, ok := stage.ResourceTarget(stageCtx) if !ok || target == nil { return stage.Result{}, errors.New("missing resource target context") } if input["prompt"] == "A" { if target.Member != "emulator" || target.Metadata["serial"] != "emu-serial" || target.Metadata["type"] != "emulator" { return stage.Result{}, fmt.Errorf("wrong first target: %#v", target) } close(aStarted) <-releaseA } else { if deadline, ok := stageCtx.Deadline(); !ok || time.Until(deadline) < 15*time.Millisecond { return stage.Result{}, errors.New("execution deadline was consumed in pool queue") } bRan <- target } return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } start := func(key, prompt string) *protocol.RunDetail { run, err := r.start(protocol.Message{PipelineID: definition.ID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}}) if err != nil { t.Fatal(err) } return run } a := start("pool-a", "A") <-aStarted b := start("pool-b", "B") awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) time.Sleep(80 * time.Millisecond) select { case <-bRan: t.Fatal("pool waiter ran before either member was released") default: } if err := r.releaseResource(phone, true); err != nil { t.Fatal(err) } var target *protocol.ResourceTarget select { case target = <-bRan: case <-time.After(3 * time.Second): t.Fatal("pool waiter did not resume") } if target.Member != "phone" || target.Metadata["serial"] != "phone-serial" { t.Fatalf("wrong queued target: %#v", target) } awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) close(releaseA) awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) r.wg.Wait() detail, err := r.store.GetRun(ctx, b.ID) if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Control.ResourceTarget == nil { t.Fatalf("pool target evidence missing: detail=%#v err=%v", detail, err) } } func TestResourceCancellationWaitsForOwnerCleanup(t *testing.T) { r, events, start := resourceRunner(t) started, cleaning, cleanupDone := make(chan struct{}), make(chan struct{}), make(chan struct{}) var waitingCalls atomic.Int32 r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { if input["prompt"] == "owner" { close(started) <-ctx.Done() close(cleaning) <-cleanupDone return stage.Result{}, ctx.Err() } waitingCalls.Add(1) return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } a := start("X", "2s", "owner") <-started b := start("X", "30ms", "waiter") awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) if err := r.cancel(a.ID); err != nil { t.Fatal(err) } <-cleaning select { case <-time.After(50 * time.Millisecond): if waitingCalls.Load() != 0 { t.Fatal("handoff before cleanup") } } close(cleanupDone) awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) if waitingCalls.Load() != 1 { t.Fatal("waiter did not resume exactly once") } } func TestResourceQuarantineAndQueuedCancellation(t *testing.T) { for _, failure := range []string{"cleanup", "panic", "restoration"} { t.Run(failure, func(t *testing.T) { r, events, start := resourceRunner(t) started, release := make(chan struct{}), make(chan struct{}) var calls atomic.Int32 r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { if input["prompt"] == "owner" { close(started) select { case <-release: case <-ctx.Done(): } if failure == "panic" { panic("unknown cleanup") } if failure == "restoration" { return stage.Result{Outcome: "success", Output: stage.Values{protocol.ResourceCleanupConfirmedKey: false}}, nil } return stage.Result{}, stage.ErrStageCleanup } calls.Add(1) return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } a := start("X", "1s", "owner") <-started b := start("X", "30ms", "waiter") awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) close(release) awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" }) if err := r.cancel(b.ID); err != nil { t.Fatal(err) } awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "canceled" && e.ResourceWaitSet && e.ResourceWait == nil }) r.wg.Wait() if calls.Load() != 0 { t.Fatal("quarantined resource handed off") } exists, err := r.store.HasResourceOwner(context.Background(), a.ID) if err != nil || !exists { t.Fatal("quarantine evidence lost", err) } d, err := r.store.GetRun(context.Background(), b.ID) if err != nil || d.ResourceWait != nil || len(d.Steps) != 0 { t.Fatal("canceled queue consumed step", err) } }) } } func TestResourceEventClearCoalescing(t *testing.T) { frame := protocol.RunEventFrame{ResourceWaitSet: true, ResourceWait: &protocol.ResourceWait{Resource: "X"}} mergeRunEventFrame(&frame, &protocol.RunEventFrame{ResourceWaitSet: true}) if !frame.ResourceWaitSet || frame.ResourceWait != nil { t.Fatal("coalescer retained stale resource wait") } } func TestResourceReleaseOnBeginAndPersistenceFailure(t *testing.T) { for _, failure := range []string{"begin", "persist"} { t.Run(failure, func(t *testing.T) { r, events, start := resourceRunner(t) db, err := sql.Open("sqlite", filepath.Join(r.store.Directory(), "resources.db")) if err != nil { t.Fatal(err) } defer db.Close() trigger := `CREATE TRIGGER resource_fail BEFORE INSERT ON run_steps WHEN json_extract(NEW.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test BeginStep failure'); END` if failure == "persist" { trigger = `CREATE TRIGGER resource_fail BEFORE UPDATE OF output_json ON run_steps WHEN json_extract(OLD.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test persistence failure'); END` } if _, err := db.Exec(trigger); err != nil { t.Fatal(err) } var calls atomic.Int32 r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { calls.Add(1) return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } a := start("X", "1s", "fail") awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" }) r.wg.Wait() if exists, err := r.store.HasResourceOwner(context.Background(), a.ID); err != nil || exists { t.Fatal("safe failure stranded resource", err) } if failure == "begin" && calls.Load() != 0 { t.Fatal("BeginStep failure executed work") } b := start("X", "1s", "pass") awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) }) } } func TestResourceCleanupQueryFailureQuarantines(t *testing.T) { r, events, start := resourceRunner(t) r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { if err := r.store.Close(); err != nil { return stage.Result{}, err } return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } a := start("X", "1s", "query-failure") awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" }) r.wg.Wait() reopened, err := pipeline.Open(filepath.Join(r.store.Directory(), "resources.db")) if err != nil { t.Fatal(err) } defer reopened.Close() if err := reopened.RecoverRuns(context.Background()); err != nil { t.Fatal(err) } if exists, err := reopened.HasResourceOwner(context.Background(), a.ID); err != nil || !exists { t.Fatal("query failure released uncertain owner", err) } } func TestResourceSkippedVisitDoesNotAcquire(t *testing.T) { r, events, _ := resourceRunner(t) ctx := context.Background() source := strings.Replace(pipeline.Template([]string{"review"}, true), "success -> #success", "success -> #success\nskipped -> #success", 1) + "```resources\n{\"exclusive\":\"X\"}\n```\n" if _, err := r.store.Submit(ctx, "skip", source); err != nil { t.Fatal(err) } items, _ := r.store.List(ctx) d, _ := r.store.Get(ctx, items[0].ID, 1) owner, _, err := r.store.CreateRun(ctx, d, "held", t.TempDir(), false, nil, nil) if err != nil { t.Fatal(err) } lease, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "X", RunID: owner, Stage: "review", Sequence: 1}) if err != nil || ch != nil { t.Fatal("owner fixture failed", err) } defer r.releaseResource(lease, true) r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { t.Error("skipped stage executed") return stage.Result{}, nil } run, err := r.start(protocol.Message{PipelineID: d.ID, Version: 1, RequestKey: "skip", Workspace: t.TempDir(), Input: map[string]any{"prompt": "no UI"}}) if err != nil { t.Fatal(err) } awaitResourceEvent(t, events, run.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) detail, err := r.store.GetRun(ctx, run.ID) if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Outcome != "skipped" { t.Fatal("skip not preserved", detail, err) } }