e6d00d4ee5
The per-workspace concurrency added in 2618f0d exposed three places that assumed a
single run, and the second concurrent run failed with 'agent quiescence not
confirmed' because of them.
- ActiveAgentSessionsForRun(runID) scopes the end-of-step quiescence check to the
run's own sessions, so another run's legitimately active agent no longer fails
this one. A run still fails when its own session cannot be confirmed idle.
- The startup guard now uses ActiveSessionOwners (which carries the owning run id)
and refuses only an ORPHANED session - one whose run is not active now - which is
what restart recovery is actually about. A session owned by a currently active
run no longer blocks new work.
- sessionActive examines only the target session: foreign entries in the
/api/session/active map are ignored, the target's absence is inactive, and the
target present with an unsupported state still fails closed.
Verified: go build, go vet and go test ./... pass, including ten concurrency tests
(two workspaces run concurrently, same-workspace refusal, symlink and default-cwd
equivalence, idempotent repeats, restart recovery, retry scoping, overlapping runs
not failing quiescence, and a run's own active session still failing it).
Operator verification, which a stage cannot perform because restarting the server
would abort the run doing it: the service was rebuilt and restarted on this
commit, then two runs in separate workspaces (/tmp/ws-c and /tmp/ws-d) executed
concurrently - both reported executing at the same time, a third run in ws-c was
refused naming run 46, and both finished succeeded with no diagnostic.
152 lines
4.5 KiB
Go
152 lines
4.5 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"path/filepath"
|
|
"reflect"
|
|
"testing"
|
|
)
|
|
|
|
func TestRunListIncludesHistoryBeyondFiftyForSearchAndPipelineScope(t *testing.T) {
|
|
ctx := context.Background()
|
|
store, err := Open(filepath.Join(t.TempDir(), "runs.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "history", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
definition, err := store.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 55; i++ {
|
|
if _, _, err := store.CreateRun(ctx, definition, fmt.Sprintf("history-%d", i), "workspace", false, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
runs, err := store.ListRuns(ctx)
|
|
if err != nil || len(runs) != 55 || runs[len(runs)-1].ID != 1 {
|
|
t.Fatal("older searchable runs missing", len(runs), err)
|
|
}
|
|
}
|
|
|
|
func TestRunRecoveryAndVersionPinning(t *testing.T) {
|
|
ctx := context.Background()
|
|
path := filepath.Join(t.TempDir(), "audit.db")
|
|
store, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
source := Template([]string{"one"}, true)
|
|
if _, err := store.Submit(ctx, "audit", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
definition, _ := store.Get(ctx, items[0].ID, 1)
|
|
id, _, err := store.CreateRun(ctx, definition, "recover", "workspace", false, nil, map[string]any{"n": 123})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.BeginStep(ctx, id, 1, "one", map[string]any{"n": 123}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.RunSession(ctx, id, 1, "ses_recover", true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err = Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
owners, err := store.ActiveSessionOwners(ctx)
|
|
if err != nil || len(owners) != 1 {
|
|
t.Fatal("lost active session ownership")
|
|
}
|
|
if err := store.RecoverRuns(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Submit(ctx, "audit", Template([]string{"replacement", "another"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
run, err := store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if run.Status != "interrupted" || run.Steps[0].Status != "interrupted" || run.Steps[0].AgentActive || run.ContentHash != definition.ContentHash || run.Version != 1 {
|
|
t.Fatal("recovery silently resumed or changed version", run)
|
|
}
|
|
if !reflect.DeepEqual(run.Graph, definition.Graph) {
|
|
t.Fatal("run graph did not retain its pinned definition", run.Graph)
|
|
}
|
|
loaded, _ := store.Get(ctx, definition.ID, 1)
|
|
if loaded.Source != source {
|
|
t.Fatal("run modified immutable source")
|
|
}
|
|
}
|
|
|
|
// TestActiveAgentSessionsAreScopedToRun proves the run-scoped query returns only
|
|
// the row's own active session: a second run's active session is never returned,
|
|
// and clearing the session clears the result.
|
|
func TestActiveAgentSessionsAreScopedToRun(t *testing.T) {
|
|
ctx := context.Background()
|
|
store, err := Open(filepath.Join(t.TempDir(), "scoped.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "scoped", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
definition, err := store.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runA, _, err := store.CreateRun(ctx, definition, "a", "workspace-a", false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runB, _, err := store.CreateRun(ctx, definition, "b", "workspace-b", false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, run := range []struct {
|
|
id int64
|
|
session string
|
|
}{{runA, "ses_a"}, {runB, "ses_b"}} {
|
|
if err := store.BeginStep(ctx, run.id, 1, "one", map[string]any{"prompt": "task"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.RunSession(ctx, run.id, 1, run.session, true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
sessions, err := store.ActiveAgentSessionsForRun(ctx, runA)
|
|
if err != nil || !reflect.DeepEqual(sessions, []string{"ses_a"}) {
|
|
t.Fatalf("run-scoped query leaked another run's session: %v %v", sessions, err)
|
|
}
|
|
if err := store.RunSession(ctx, runA, 1, "ses_a", false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sessions, err = store.ActiveAgentSessionsForRun(ctx, runA)
|
|
if err != nil || len(sessions) != 0 {
|
|
t.Fatalf("inactive session still reported active: %v %v", sessions, err)
|
|
}
|
|
if sessions, err = store.ActiveAgentSessionsForRun(ctx, runB); err != nil || !reflect.DeepEqual(sessions, []string{"ses_b"}) {
|
|
t.Fatalf("second run's own session was affected: %v %v", sessions, err)
|
|
}
|
|
}
|