293 lines
9.3 KiB
Go
293 lines
9.3 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"path/filepath"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"git.a-wan.net/alan/pipelines/internal/summarise"
|
|
)
|
|
|
|
func scheduleFixture(t *testing.T) (context.Context, *pipeline.Store, *runner, int64) {
|
|
t.Helper()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "schedule.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = s.Submit(ctx, "research", pipeline.Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := s.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, err := newRunner(ctx, s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { cancel(); r.schedulerWG.Wait(); r.wg.Wait(); s.Close() })
|
|
return ctx, s, r, items[0].ID
|
|
}
|
|
|
|
func saveTestSchedule(t *testing.T, ctx context.Context, s *pipeline.Store, pid int64, prompt, workspace, every string, now time.Time) *protocol.SavedSchedule {
|
|
t.Helper()
|
|
item, err := s.SaveSchedule(ctx, protocol.SavedSchedule{Name: prompt, PipelineID: pid, Version: 1, Enabled: true, Workspace: workspace, Input: map[string]any{"prompt": prompt, "custom": map[string]any{"nested": "kept"}}, Rule: protocol.Schedule{Every: every}, Model: "openai/default", StageModels: map[string]string{"one": "openai/override"}, NotifyMode: "off", Title: "Scheduled research", OutputFormat: "markdown"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return item
|
|
}
|
|
|
|
func dueOf(t *testing.T, item *protocol.SavedSchedule) time.Time {
|
|
t.Helper()
|
|
due, err := time.Parse(time.RFC3339Nano, item.NextDue)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return due
|
|
}
|
|
|
|
func TestIndependentScheduledRunsAndProvenance(t *testing.T) {
|
|
ctx, s, r, pid := scheduleFixture(t)
|
|
now := time.Now()
|
|
a := saveTestSchedule(t, ctx, s, pid, "First research", t.TempDir(), "1m", now)
|
|
b := saveTestSchedule(t, ctx, s, pid, "Second research", t.TempDir(), "90s", now)
|
|
r.events = newRunEventHub()
|
|
frames := make(chan protocol.RunEventFrame, 64)
|
|
sub := r.events.subscribe("schedules", "", nil, func(m protocol.Message) error { frames <- *m.RunEvent; return nil })
|
|
sub.ready()
|
|
defer sub.close()
|
|
var mu sync.Mutex
|
|
seen := map[string]bool{}
|
|
r.execute = func(ctx context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
|
mu.Lock()
|
|
seen[fmt.Sprint(input["prompt"])] = true
|
|
mu.Unlock()
|
|
if def.Configs["model"] != "override" || def.Configs["provider"] != "openai" {
|
|
t.Error("model override missing", def.Configs)
|
|
}
|
|
if input["custom"] == nil {
|
|
t.Error("arbitrary saved input lost")
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"done": true, "markdown": "# Research report"}}, nil
|
|
}
|
|
for _, item := range []*protocol.SavedSchedule{a, b} {
|
|
if err := r.scheduleTick(dueOf(t, item), false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
}
|
|
runs, err := s.ListRuns(ctx)
|
|
if err != nil || len(runs) != 2 {
|
|
t.Fatal(runs, err)
|
|
}
|
|
for _, run := range runs {
|
|
if !run.Scheduled || run.ScheduleID == 0 || run.ScheduleName == "" || run.Version != 1 || run.NotifyMode != "off" || run.Title != "Scheduled research" || run.Status != "succeeded" || run.OutputFormat != "markdown" {
|
|
t.Fatalf("missing provenance or spec: %+v", run)
|
|
}
|
|
d, err := s.GetRun(ctx, run.ID)
|
|
if err != nil || !d.Scheduled || d.Input["custom"] == nil {
|
|
t.Fatal(d, err)
|
|
}
|
|
}
|
|
mu.Lock()
|
|
if !seen[a.Name] || !seen[b.Name] {
|
|
t.Error("independent prompts missing", seen)
|
|
}
|
|
mu.Unlock()
|
|
terminals := 0
|
|
deadline := time.After(2 * time.Second)
|
|
for terminals < 2 {
|
|
select {
|
|
case frame := <-frames:
|
|
if !frame.Scheduled || frame.ScheduleID == 0 || frame.ScheduleName == "" {
|
|
t.Fatal("frame preceded provenance", frame)
|
|
}
|
|
if frame.Status == "succeeded" {
|
|
terminals++
|
|
}
|
|
case <-deadline:
|
|
t.Fatal("missing scheduled terminal frames")
|
|
}
|
|
}
|
|
manual, err := r.start(protocol.Message{ScheduleID: a.ID, PipelineID: pid, Version: 1, Workspace: a.Workspace, RequestKey: "manual", Input: map[string]any{"prompt": "manual", "custom": true}, Model: "openai/override", NotifyMode: "off"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
if manual.Scheduled || manual.ScheduleID != 0 {
|
|
t.Fatal("manual run forged schedule origin", manual)
|
|
}
|
|
manual, err = s.GetRun(ctx, manual.ID)
|
|
if err != nil || manual.Scheduled {
|
|
t.Fatal(manual, err)
|
|
}
|
|
if _, err = s.Submit(ctx, "research", pipeline.Template([]string{"one", "two"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
entries, _ := s.ListSchedules(ctx, pid)
|
|
if len(entries) != 2 || entries[0].Version != 1 {
|
|
t.Fatal(entries)
|
|
}
|
|
if err = s.DeleteSchedule(ctx, a.ID, a.Revision); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, run := range runs {
|
|
d, _ := s.GetRun(ctx, run.ID)
|
|
if !d.Scheduled || d.ScheduleName != run.ScheduleName {
|
|
t.Fatal("delete erased provenance", d)
|
|
}
|
|
}
|
|
// A reconnect snapshot uses the same durable origin after deletion, while
|
|
// the manually admitted run remains unscheduled.
|
|
var snapshots []protocol.RunEventFrame
|
|
if !handleRunEvents(protocol.Message{Type: protocol.Subscribe}, r.events, r, "snapshot", func(m protocol.Message) error {
|
|
if m.RunEvent != nil {
|
|
snapshots = append(snapshots, *m.RunEvent)
|
|
}
|
|
return nil
|
|
}) {
|
|
t.Fatal("subscribe not handled")
|
|
}
|
|
defer r.events.unsubscribeClient("snapshot")
|
|
if len(snapshots) != 3 {
|
|
t.Fatal("snapshot count", snapshots)
|
|
}
|
|
for _, frame := range snapshots {
|
|
if frame.RunID == manual.ID {
|
|
if frame.Scheduled {
|
|
t.Fatal("manual snapshot forged", frame)
|
|
}
|
|
} else if !frame.Scheduled || frame.ScheduleID == 0 || frame.ScheduleName == "" {
|
|
t.Fatal("snapshot lost origin", frame)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestScheduleDeletionDoesNotCancelAdmittedRun(t *testing.T) {
|
|
ctx, s, r, pid := scheduleFixture(t)
|
|
item := saveTestSchedule(t, ctx, s, pid, "active research", t.TempDir(), "1m", time.Now())
|
|
entered, release := make(chan struct{}), make(chan struct{})
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
close(entered)
|
|
<-release
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "# Finished"}}, nil
|
|
}
|
|
if err := r.scheduleTick(dueOf(t, item), false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
<-entered
|
|
if err := s.DeleteSchedule(ctx, item.ID, item.Revision); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := r.scheduleTick(dueOf(t, item).Add(time.Minute), false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
close(release)
|
|
r.wg.Wait()
|
|
runs, err := s.ListRuns(ctx)
|
|
if err != nil || len(runs) != 1 || runs[0].Status != "succeeded" || !runs[0].Scheduled || runs[0].ScheduleID != item.ID {
|
|
t.Fatal(runs, err)
|
|
}
|
|
}
|
|
|
|
func TestSavedScheduleRecoveryAndOverlap(t *testing.T) {
|
|
ctx, s, r, pid := scheduleFixture(t)
|
|
item := saveTestSchedule(t, ctx, s, pid, "task", t.TempDir(), "1m", time.Now())
|
|
due := dueOf(t, item)
|
|
entered, release := make(chan struct{}), make(chan struct{})
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
close(entered)
|
|
<-release
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
if err := r.scheduleTick(due, false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
<-entered
|
|
if err := r.scheduleTick(due.Add(time.Minute), false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
close(release)
|
|
r.wg.Wait()
|
|
if err := r.scheduleTick(due.Add(time.Minute), false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := r.scheduleTick(due.Add(time.Hour), true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := r.scheduleTick(due.Add(time.Hour), false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runs, err := s.ListRuns(ctx)
|
|
if err != nil || len(runs) != 1 {
|
|
t.Fatal("replayed or overlapping work", runs, err)
|
|
}
|
|
entries, _ := s.ListSchedules(ctx, pid)
|
|
item = &entries[0]
|
|
next := dueOf(t, item)
|
|
days := []string{"sun", "mon", "tue", "wed", "thu", "fri", "sat"}
|
|
item.Rule.Days = []string{days[(next.In(time.Local).Weekday()+1)%7]}
|
|
item, err = s.SaveSchedule(ctx, *item, next.Add(-time.Minute))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = r.scheduleTick(dueOf(t, item), false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runs, _ = s.ListRuns(ctx)
|
|
if len(runs) != 1 {
|
|
t.Fatal("excluded day dispatched", runs)
|
|
}
|
|
}
|
|
|
|
func TestScheduleEditBetweenClaimAndAdmission(t *testing.T) {
|
|
for _, operation := range []string{"disable", "edit", "delete"} {
|
|
t.Run(operation, func(t *testing.T) {
|
|
ctx, s, r, pid := scheduleFixture(t)
|
|
item := saveTestSchedule(t, ctx, s, pid, "task", t.TempDir(), "1m", time.Now())
|
|
item.Title = ""
|
|
var err error
|
|
item, err = s.SaveSchedule(ctx, *item, time.Now())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.title = func(context.Context, string) summarise.Title {
|
|
if operation == "delete" {
|
|
err = s.DeleteSchedule(ctx, item.ID, item.Revision)
|
|
} else {
|
|
if operation == "disable" {
|
|
item.Enabled = false
|
|
} else {
|
|
item.Input["prompt"] = "new prompt"
|
|
}
|
|
_, err = s.SaveSchedule(ctx, *item, time.Now())
|
|
}
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
return summarise.Title{Text: "title"}
|
|
}
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
t.Error("invalidated occurrence executed")
|
|
return stage.Result{}, nil
|
|
}
|
|
if err = r.scheduleTick(dueOf(t, item), false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
runs, _ := s.ListRuns(ctx)
|
|
if len(runs) != 0 {
|
|
t.Fatal("invalidated occurrence admitted", runs)
|
|
}
|
|
})
|
|
}
|
|
}
|