419 lines
15 KiB
Go
419 lines
15 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
func resourceRunner(t *testing.T) (*runner, <-chan protocol.RunEventFrame, func(string, string, string) *protocol.RunDetail) {
|
|
t.Helper()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "resources.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, err := newRunner(ctx, s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.events = newRunEventHub()
|
|
r.events.interval = time.Millisecond
|
|
events := make(chan protocol.RunEventFrame, 128)
|
|
sub := r.events.subscribe("resource-test", "", nil, func(m protocol.Message) error {
|
|
if m.RunEvent != nil {
|
|
events <- *m.RunEvent
|
|
}
|
|
return nil
|
|
})
|
|
sub.ready()
|
|
t.Cleanup(func() { cancel(); r.wg.Wait(); sub.close(); s.Close() })
|
|
count := 0
|
|
return r, events, func(resource, timeout, prompt string) *protocol.RunDetail {
|
|
count++
|
|
source := strings.Replace(pipeline.Template([]string{"work"}, true), "timeout=1h", "timeout="+timeout, 1)
|
|
if resource != "" {
|
|
source += "```resources\n{\"exclusive\":\"" + resource + "\"}\n```\n"
|
|
}
|
|
name := fmt.Sprint("resource-", count)
|
|
if _, err := s.Submit(ctx, name, source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := s.List(ctx)
|
|
var pid int64
|
|
for _, p := range items {
|
|
if p.Name == name {
|
|
pid = p.ID
|
|
}
|
|
}
|
|
d, err := r.start(protocol.Message{PipelineID: pid, Version: 1, RequestKey: name, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return d
|
|
}
|
|
}
|
|
|
|
func awaitResourceEvent(t *testing.T, events <-chan protocol.RunEventFrame, id int64, predicate func(protocol.RunEventFrame) bool) protocol.RunEventFrame {
|
|
t.Helper()
|
|
timer := time.NewTimer(3 * time.Second)
|
|
defer timer.Stop()
|
|
for {
|
|
select {
|
|
case event := <-events:
|
|
if event.RunID == id && predicate(event) {
|
|
return event
|
|
}
|
|
case <-timer.C:
|
|
t.Fatalf("resource event missing for run %d", id)
|
|
return protocol.RunEventFrame{}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestResourcePauseBeyondTimeoutAndAutomaticResume(t *testing.T) {
|
|
r, events, start := resourceRunner(t)
|
|
aStarted, releaseA := make(chan struct{}), make(chan struct{})
|
|
bStarted := make(chan struct{}, 1)
|
|
var bCalls atomic.Int32
|
|
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
|
|
switch input["prompt"] {
|
|
case "A":
|
|
close(aStarted)
|
|
select {
|
|
case <-releaseA:
|
|
case <-ctx.Done():
|
|
return stage.Result{}, ctx.Err()
|
|
}
|
|
case "B":
|
|
bCalls.Add(1)
|
|
deadline, ok := ctx.Deadline()
|
|
if !ok || time.Until(deadline) < 15*time.Millisecond {
|
|
return stage.Result{}, errors.New("execution timeout was consumed in queue")
|
|
}
|
|
bStarted <- struct{}{}
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
a := start("X", "2s", "A")
|
|
<-aStarted
|
|
b := start("X", "30ms", "B")
|
|
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
|
|
<-time.After(70 * time.Millisecond) // Deliberately longer than B's whole execution timeout.
|
|
d, err := r.store.GetRun(context.Background(), b.ID)
|
|
if err != nil || d.Status != protocol.RunStatusQueued || d.ResourceWait == nil || len(d.Steps) != 0 || bCalls.Load() != 0 {
|
|
t.Fatal("queued execution consumed accounting", d, err, bCalls.Load())
|
|
}
|
|
if _, err := r.decide(protocol.Message{RunID: b.ID, Option: "success"}); err == nil {
|
|
t.Fatal("resource wait exposed a human decision")
|
|
}
|
|
c := start("Y", "1s", "C")
|
|
awaitResourceEvent(t, events, c.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
|
select {
|
|
case <-bStarted:
|
|
t.Fatal("B ran while A held X")
|
|
default:
|
|
}
|
|
owner, err := r.store.GetRun(context.Background(), a.ID)
|
|
if err != nil || owner.Status != "running" {
|
|
t.Fatal("A did not remain owner", err)
|
|
}
|
|
close(releaseA)
|
|
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
|
r.wg.Wait()
|
|
d, err = r.store.GetRun(context.Background(), b.ID)
|
|
if err != nil || bCalls.Load() != 1 || d.ResourceWait != nil || len(d.Steps) != 1 || d.Steps[0].Control.Visit != 1 || d.Steps[0].Sequence != 1 {
|
|
t.Fatal("resume did not execute exactly one fresh visit", d, err)
|
|
}
|
|
}
|
|
|
|
func TestResourcePoolRunnerHandoffUsesConcreteTargetAfterQueue(t *testing.T) {
|
|
r, events, _ := resourceRunner(t)
|
|
ctx := context.Background()
|
|
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"pool\":\"targets\"}\n```\n"
|
|
if _, err := r.store.Submit(ctx, "pool-runner", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := r.store.List(ctx)
|
|
if err != nil || len(items) != 1 {
|
|
t.Fatal("pool definition missing", err)
|
|
}
|
|
definition, err := r.store.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := r.store.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{
|
|
{Member: "phone", Enabled: true, Metadata: map[string]string{"serial": "phone-serial", "type": "phone"}},
|
|
{Member: "emulator", Enabled: true, Metadata: map[string]string{"serial": "emu-serial", "type": "emulator"}},
|
|
}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ownerRun, _, err := r.store.CreateRun(ctx, definition, "phone-owner", t.TempDir(), false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
phone, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "phone", RunID: ownerRun, Stage: "work", Sequence: 1})
|
|
if err != nil || ch != nil {
|
|
t.Fatalf("phone owner acquire: channel=%v err=%v", ch, err)
|
|
}
|
|
defer r.releaseResource(phone, true)
|
|
|
|
aStarted, releaseA := make(chan struct{}), make(chan struct{})
|
|
bRan := make(chan *protocol.ResourceTarget, 1)
|
|
r.execute = func(stageCtx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
|
|
target, ok := stage.ResourceTarget(stageCtx)
|
|
if !ok || target == nil {
|
|
return stage.Result{}, errors.New("missing resource target context")
|
|
}
|
|
if input["prompt"] == "A" {
|
|
if target.Member != "emulator" || target.Metadata["serial"] != "emu-serial" || target.Metadata["type"] != "emulator" {
|
|
return stage.Result{}, fmt.Errorf("wrong first target: %#v", target)
|
|
}
|
|
close(aStarted)
|
|
<-releaseA
|
|
} else {
|
|
if deadline, ok := stageCtx.Deadline(); !ok || time.Until(deadline) < 15*time.Millisecond {
|
|
return stage.Result{}, errors.New("execution deadline was consumed in pool queue")
|
|
}
|
|
bRan <- target
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
start := func(key, prompt string) *protocol.RunDetail {
|
|
run, err := r.start(protocol.Message{PipelineID: definition.ID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return run
|
|
}
|
|
a := start("pool-a", "A")
|
|
<-aStarted
|
|
b := start("pool-b", "B")
|
|
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
|
|
time.Sleep(80 * time.Millisecond)
|
|
select {
|
|
case <-bRan:
|
|
t.Fatal("pool waiter ran before either member was released")
|
|
default:
|
|
}
|
|
if err := r.releaseResource(phone, true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var target *protocol.ResourceTarget
|
|
select {
|
|
case target = <-bRan:
|
|
case <-time.After(3 * time.Second):
|
|
t.Fatal("pool waiter did not resume")
|
|
}
|
|
if target.Member != "phone" || target.Metadata["serial"] != "phone-serial" {
|
|
t.Fatalf("wrong queued target: %#v", target)
|
|
}
|
|
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
|
close(releaseA)
|
|
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
|
r.wg.Wait()
|
|
detail, err := r.store.GetRun(ctx, b.ID)
|
|
if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Control.ResourceTarget == nil {
|
|
t.Fatalf("pool target evidence missing: detail=%#v err=%v", detail, err)
|
|
}
|
|
}
|
|
|
|
func TestResourceCancellationWaitsForOwnerCleanup(t *testing.T) {
|
|
r, events, start := resourceRunner(t)
|
|
started, cleaning, cleanupDone := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
|
var waitingCalls atomic.Int32
|
|
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
|
|
if input["prompt"] == "owner" {
|
|
close(started)
|
|
<-ctx.Done()
|
|
close(cleaning)
|
|
<-cleanupDone
|
|
return stage.Result{}, ctx.Err()
|
|
}
|
|
waitingCalls.Add(1)
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
a := start("X", "2s", "owner")
|
|
<-started
|
|
b := start("X", "30ms", "waiter")
|
|
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
|
|
if err := r.cancel(a.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
<-cleaning
|
|
select {
|
|
case <-time.After(50 * time.Millisecond):
|
|
if waitingCalls.Load() != 0 {
|
|
t.Fatal("handoff before cleanup")
|
|
}
|
|
}
|
|
close(cleanupDone)
|
|
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
|
if waitingCalls.Load() != 1 {
|
|
t.Fatal("waiter did not resume exactly once")
|
|
}
|
|
}
|
|
|
|
func TestResourceQuarantineAndQueuedCancellation(t *testing.T) {
|
|
for _, failure := range []string{"cleanup", "panic", "restoration"} {
|
|
t.Run(failure, func(t *testing.T) {
|
|
r, events, start := resourceRunner(t)
|
|
started, release := make(chan struct{}), make(chan struct{})
|
|
var calls atomic.Int32
|
|
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
|
|
if input["prompt"] == "owner" {
|
|
close(started)
|
|
select {
|
|
case <-release:
|
|
case <-ctx.Done():
|
|
}
|
|
if failure == "panic" {
|
|
panic("unknown cleanup")
|
|
}
|
|
if failure == "restoration" {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{protocol.ResourceCleanupConfirmedKey: false}}, nil
|
|
}
|
|
return stage.Result{}, stage.ErrStageCleanup
|
|
}
|
|
calls.Add(1)
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
a := start("X", "1s", "owner")
|
|
<-started
|
|
b := start("X", "30ms", "waiter")
|
|
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
|
|
close(release)
|
|
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
|
|
if err := r.cancel(b.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool {
|
|
return e.Status == "canceled" && e.ResourceWaitSet && e.ResourceWait == nil
|
|
})
|
|
r.wg.Wait()
|
|
if calls.Load() != 0 {
|
|
t.Fatal("quarantined resource handed off")
|
|
}
|
|
exists, err := r.store.HasResourceOwner(context.Background(), a.ID)
|
|
if err != nil || !exists {
|
|
t.Fatal("quarantine evidence lost", err)
|
|
}
|
|
d, err := r.store.GetRun(context.Background(), b.ID)
|
|
if err != nil || d.ResourceWait != nil || len(d.Steps) != 0 {
|
|
t.Fatal("canceled queue consumed step", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResourceEventClearCoalescing(t *testing.T) {
|
|
frame := protocol.RunEventFrame{ResourceWaitSet: true, ResourceWait: &protocol.ResourceWait{Resource: "X"}}
|
|
mergeRunEventFrame(&frame, &protocol.RunEventFrame{ResourceWaitSet: true})
|
|
if !frame.ResourceWaitSet || frame.ResourceWait != nil {
|
|
t.Fatal("coalescer retained stale resource wait")
|
|
}
|
|
}
|
|
|
|
func TestResourceReleaseOnBeginAndPersistenceFailure(t *testing.T) {
|
|
for _, failure := range []string{"begin", "persist"} {
|
|
t.Run(failure, func(t *testing.T) {
|
|
r, events, start := resourceRunner(t)
|
|
db, err := sql.Open("sqlite", filepath.Join(r.store.Directory(), "resources.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
trigger := `CREATE TRIGGER resource_fail BEFORE INSERT ON run_steps WHEN json_extract(NEW.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test BeginStep failure'); END`
|
|
if failure == "persist" {
|
|
trigger = `CREATE TRIGGER resource_fail BEFORE UPDATE OF output_json ON run_steps WHEN json_extract(OLD.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test persistence failure'); END`
|
|
}
|
|
if _, err := db.Exec(trigger); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var calls atomic.Int32
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
calls.Add(1)
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
a := start("X", "1s", "fail")
|
|
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
|
|
r.wg.Wait()
|
|
if exists, err := r.store.HasResourceOwner(context.Background(), a.ID); err != nil || exists {
|
|
t.Fatal("safe failure stranded resource", err)
|
|
}
|
|
if failure == "begin" && calls.Load() != 0 {
|
|
t.Fatal("BeginStep failure executed work")
|
|
}
|
|
b := start("X", "1s", "pass")
|
|
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResourceCleanupQueryFailureQuarantines(t *testing.T) {
|
|
r, events, start := resourceRunner(t)
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
if err := r.store.Close(); err != nil {
|
|
return stage.Result{}, err
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
a := start("X", "1s", "query-failure")
|
|
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
|
|
r.wg.Wait()
|
|
reopened, err := pipeline.Open(filepath.Join(r.store.Directory(), "resources.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer reopened.Close()
|
|
if err := reopened.RecoverRuns(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if exists, err := reopened.HasResourceOwner(context.Background(), a.ID); err != nil || !exists {
|
|
t.Fatal("query failure released uncertain owner", err)
|
|
}
|
|
}
|
|
|
|
func TestResourceSkippedVisitDoesNotAcquire(t *testing.T) {
|
|
r, events, _ := resourceRunner(t)
|
|
ctx := context.Background()
|
|
source := strings.Replace(pipeline.Template([]string{"review"}, true), "success -> #success", "success -> #success\nskipped -> #success", 1) + "```resources\n{\"exclusive\":\"X\"}\n```\n"
|
|
if _, err := r.store.Submit(ctx, "skip", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := r.store.List(ctx)
|
|
d, _ := r.store.Get(ctx, items[0].ID, 1)
|
|
owner, _, err := r.store.CreateRun(ctx, d, "held", t.TempDir(), false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lease, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "X", RunID: owner, Stage: "review", Sequence: 1})
|
|
if err != nil || ch != nil {
|
|
t.Fatal("owner fixture failed", err)
|
|
}
|
|
defer r.releaseResource(lease, true)
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
t.Error("skipped stage executed")
|
|
return stage.Result{}, nil
|
|
}
|
|
run, err := r.start(protocol.Message{PipelineID: d.ID, Version: 1, RequestKey: "skip", Workspace: t.TempDir(), Input: map[string]any{"prompt": "no UI"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
awaitResourceEvent(t, events, run.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
|
detail, err := r.store.GetRun(ctx, run.ID)
|
|
if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Outcome != "skipped" {
|
|
t.Fatal("skip not preserved", detail, err)
|
|
}
|
|
}
|