Scope agent session bookkeeping to the run for concurrent runs

The per-workspace concurrency added in 2618f0d exposed three places that assumed a
single run, and the second concurrent run failed with 'agent quiescence not
confirmed' because of them.

- ActiveAgentSessionsForRun(runID) scopes the end-of-step quiescence check to the
  run's own sessions, so another run's legitimately active agent no longer fails
  this one. A run still fails when its own session cannot be confirmed idle.
- The startup guard now uses ActiveSessionOwners (which carries the owning run id)
  and refuses only an ORPHANED session - one whose run is not active now - which is
  what restart recovery is actually about. A session owned by a currently active
  run no longer blocks new work.
- sessionActive examines only the target session: foreign entries in the
  /api/session/active map are ignored, the target's absence is inactive, and the
  target present with an unsupported state still fails closed.

Verified: go build, go vet and go test ./... pass, including ten concurrency tests
(two workspaces run concurrently, same-workspace refusal, symlink and default-cwd
equivalence, idempotent repeats, restart recovery, retry scoping, overlapping runs
not failing quiescence, and a run's own active session still failing it).

Operator verification, which a stage cannot perform because restarting the server
would abort the run doing it: the service was rebuilt and restarted on this
commit, then two runs in separate workspaces (/tmp/ws-c and /tmp/ws-d) executed
concurrently - both reported executing at the same time, a third run in ws-c was
refused naming run 46, and both finished succeeded with no diagnostic.
This commit is contained in:
2026-10-15 15:38:50 +01:00
parent 2618f0d244
commit e6d00d4ee5
7 changed files with 330 additions and 23 deletions
@@ -25,7 +25,11 @@ parallel/distributed task engine or a guarantee that model reports are correct.
resolved workspaces execute concurrently; only a second run in a workspace an
active run is already changing is refused, naming the conflicting run and
advising you to wait or use a different workspace (for example another
checkout). Reusing that active run's request key reattaches idempotently.
checkout). Agent-session quiescence is scoped to the run: a run fails only when
one of its own sessions cannot be confirmed idle, so one run's active agent never
fails another's concurrent run. An active session left over by a restart with no
owning active run is an orphan and still refuses new work until interrupted.
Reusing that active run's request key reattaches idempotently.
There are no parallel branches or built-in human approval stages. Optional
worktrees, full backup/restore, automatic database sync and independent
verifier stage types are future ideas, not existing capabilities. Do not
+17 -5
View File
@@ -313,7 +313,15 @@ func (s *Store) RunSession(ctx context.Context, id int64, sequence int, session
return tx.Commit()
}
type SessionOwner struct{ ID, Backend, Directory, Reference string }
type SessionOwner struct {
ID string
Backend string
Directory string
Reference string
// RunID is the owning run, used to tell an orphaned session (no active run)
// from one owned by a run that is active right now.
RunID int64
}
func (s *Store) RunSessionOwner(ctx context.Context, id int64, sequence int) (SessionOwner, error) {
var owner SessionOwner
@@ -322,7 +330,7 @@ func (s *Store) RunSessionOwner(ctx context.Context, id int64, sequence int) (Se
}
func (s *Store) ActiveSessionOwners(ctx context.Context) ([]SessionOwner, error) {
rows, err := s.db.QueryContext(ctx, `SELECT s.session_id,COALESCE(o.backend,'v2'),r.workspace,COALESCE(o.reference,'')
rows, err := s.db.QueryContext(ctx, `SELECT s.session_id,COALESCE(o.backend,'v2'),r.workspace,COALESCE(o.reference,''),s.run_id
FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id LEFT JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence WHERE s.agent_active=1 AND s.session_id<>''`)
if err != nil {
return nil, err
@@ -331,7 +339,7 @@ FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id LEFT JOIN run_session_own
var owners []SessionOwner
for rows.Next() {
var owner SessionOwner
if err = rows.Scan(&owner.ID, &owner.Backend, &owner.Directory, &owner.Reference); err != nil {
if err = rows.Scan(&owner.ID, &owner.Backend, &owner.Directory, &owner.Reference, &owner.RunID); err != nil {
return nil, err
}
owners = append(owners, owner)
@@ -386,8 +394,12 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno
return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id))
}
func (s *Store) ActiveAgentSessions(ctx context.Context) ([]string, error) {
rows, err := s.db.QueryContext(ctx, "SELECT DISTINCT session_id FROM run_steps WHERE agent_active=1 AND session_id<>''")
// ActiveAgentSessionsForRun returns the distinct active agent session ids that
// belong to one run. It is scoped to run_steps.run_id so concurrent runs never
// observe each other's sessions: run_steps is keyed by (run_id, sequence), so
// this is one indexed lookup, not a scan.
func (s *Store) ActiveAgentSessionsForRun(ctx context.Context, runID int64) ([]string, error) {
rows, err := s.db.QueryContext(ctx, "SELECT DISTINCT session_id FROM run_steps WHERE run_id=? AND agent_active=1 AND session_id<>''", runID)
if err != nil {
return nil, err
}
+58 -2
View File
@@ -68,8 +68,8 @@ func TestRunRecoveryAndVersionPinning(t *testing.T) {
t.Fatal(err)
}
defer store.Close()
sessions, err := store.ActiveAgentSessions(ctx)
if err != nil || len(sessions) != 1 {
owners, err := store.ActiveSessionOwners(ctx)
if err != nil || len(owners) != 1 {
t.Fatal("lost active session ownership")
}
if err := store.RecoverRuns(ctx); err != nil {
@@ -93,3 +93,59 @@ func TestRunRecoveryAndVersionPinning(t *testing.T) {
t.Fatal("run modified immutable source")
}
}
// TestActiveAgentSessionsAreScopedToRun proves the run-scoped query returns only
// the row's own active session: a second run's active session is never returned,
// and clearing the session clears the result.
func TestActiveAgentSessionsAreScopedToRun(t *testing.T) {
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), "scoped.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "scoped", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
definition, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runA, _, err := store.CreateRun(ctx, definition, "a", "workspace-a", false, nil, nil)
if err != nil {
t.Fatal(err)
}
runB, _, err := store.CreateRun(ctx, definition, "b", "workspace-b", false, nil, nil)
if err != nil {
t.Fatal(err)
}
for _, run := range []struct {
id int64
session string
}{{runA, "ses_a"}, {runB, "ses_b"}} {
if err := store.BeginStep(ctx, run.id, 1, "one", map[string]any{"prompt": "task"}); err != nil {
t.Fatal(err)
}
if err := store.RunSession(ctx, run.id, 1, run.session, true); err != nil {
t.Fatal(err)
}
}
sessions, err := store.ActiveAgentSessionsForRun(ctx, runA)
if err != nil || !reflect.DeepEqual(sessions, []string{"ses_a"}) {
t.Fatalf("run-scoped query leaked another run's session: %v %v", sessions, err)
}
if err := store.RunSession(ctx, runA, 1, "ses_a", false); err != nil {
t.Fatal(err)
}
sessions, err = store.ActiveAgentSessionsForRun(ctx, runA)
if err != nil || len(sessions) != 0 {
t.Fatalf("inactive session still reported active: %v %v", sessions, err)
}
if sessions, err = store.ActiveAgentSessionsForRun(ctx, runB); err != nil || !reflect.DeepEqual(sessions, []string{"ses_b"}) {
t.Fatalf("second run's own session was affected: %v %v", sessions, err)
}
}
+174 -1
View File
@@ -16,6 +16,16 @@ import (
)
func concurrentStore(t *testing.T) (context.Context, *runner, int64) {
t.Helper()
return newConcurrentRunner(t, pipeline.Template([]string{"one"}, true))
}
// newConcurrentRunner builds the shared test store/runner around a submitted
// pipeline source. Tests that need an agent stage (useAgent true) must pass a
// source with a non-empty tools block: the default template's empty tools block
// explicitly denies tools, so useAgent is false and the step-completion
// quiescence check is never exercised.
func newConcurrentRunner(t *testing.T, source string) (context.Context, *runner, int64) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
store, err := pipeline.Open(filepath.Join(t.TempDir(), "concurrent.db"))
@@ -23,7 +33,7 @@ func concurrentStore(t *testing.T) (context.Context, *runner, int64) {
cancel()
t.Fatal(err)
}
if _, err := store.Submit(ctx, "test", pipeline.Template([]string{"one"}, true)); err != nil {
if _, err := store.Submit(ctx, "test", source); err != nil {
cancel()
store.Close()
t.Fatal(err)
@@ -48,6 +58,14 @@ func concurrentStore(t *testing.T) (context.Context, *runner, int64) {
return ctx, r, items[0].ID
}
// agentStore returns a runner whose pipeline has an agent stage (a non-empty,
// explicitly declared tools block), so the step-completion quiescence check runs.
func agentStore(t *testing.T) (context.Context, *runner, int64) {
t.Helper()
source := strings.Replace(pipeline.Template([]string{"one"}, true), "```tools\n```", "```tools\nread\nshell\n```", 1)
return newConcurrentRunner(t, source)
}
// waitEntered fails fast instead of hanging when a stage never starts.
func waitEntered(t *testing.T, entered <-chan struct{}) {
t.Helper()
@@ -344,3 +362,158 @@ func TestRetryConflictsOnlyWithSameWorkspaceActiveRun(t *testing.T) {
close(release)
r.wg.Wait()
}
// waitTerminal polls one run until it leaves queued/running, failing fast instead
// of hanging. It is scoped to the run so a blocked sibling does not stall it.
func waitTerminal(t *testing.T, r *runner, id int64) *protocol.RunDetail {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for {
detail, err := r.getRun(id)
if err != nil {
t.Fatal(err)
}
switch detail.Status {
case "succeeded", "failed", "canceled":
return detail
}
if time.Now().After(deadline) {
t.Fatalf("run %d did not reach a terminal state: %+v", id, detail)
}
time.Sleep(5 * time.Millisecond)
}
}
// TestOverlappingRunsDoNotFailQuiescence proves the step-completion check is
// scoped to the run: with two agent runs in different workspaces each recording
// its OWN session active, releasing run A lets A reach succeeded while run B's
// session is still active, and vice versa. A server-wide check would fail A
// deterministically against B's active session.
func TestOverlappingRunsDoNotFailQuiescence(t *testing.T) {
_, r, pipelineID := agentStore(t)
a, b := t.TempDir(), t.TempDir()
type agentRun struct {
id int64
session string
}
var mu sync.Mutex
byWorkspace := map[string]agentRun{}
releases := map[string]chan struct{}{}
ready := make(chan struct{})
entered := make(chan string, 2)
resolve := func(dir string) string {
t.Helper()
resolved, err := filepath.EvalSymlinks(dir)
if err != nil {
t.Fatal(err)
}
return resolved
}
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
<-ready
workspace, _ := input["pwd"].(string)
mu.Lock()
agent, ok := byWorkspace[workspace]
release := releases[workspace]
mu.Unlock()
if !ok {
return stage.Result{}, fmt.Errorf("execute for unknown workspace %q", workspace)
}
if err := r.store.RunSession(context.Background(), agent.id, 1, agent.session, true, "v2", ""); err != nil {
return stage.Result{}, err
}
entered <- workspace
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
if err := r.store.RunSession(context.Background(), agent.id, 1, agent.session, false, "v2", ""); err != nil {
return stage.Result{}, err
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "a", Workspace: a, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
second, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "b", Workspace: b, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatalf("second workspace was blocked by the first: %v", err)
}
if first.Workspace != resolve(a) || second.Workspace != resolve(b) {
t.Fatalf("unexpected resolved workspaces: %q, %q", first.Workspace, second.Workspace)
}
mu.Lock()
byWorkspace[first.Workspace] = agentRun{id: first.ID, session: "ses_a"}
byWorkspace[second.Workspace] = agentRun{id: second.ID, session: "ses_b"}
releases[first.Workspace] = make(chan struct{})
releases[second.Workspace] = make(chan struct{})
mu.Unlock()
close(ready)
waitEnteredString(t, entered)
waitEnteredString(t, entered)
for _, id := range []int64{first.ID, second.ID} {
detail, err := r.getRun(id)
if err != nil || !detail.Executing {
t.Fatalf("run %d was not reported executing: %v", id, err)
}
}
// Release A only; B's own session stays active while A finishes.
close(releases[first.Workspace])
done := waitTerminal(t, r, first.ID)
if done.Status != "succeeded" || strings.Contains(done.Diagnostic, "agent quiescence not confirmed") {
t.Fatalf("run A failed despite its own session being idle: %+v", done)
}
if detail, err := r.getRun(second.ID); err != nil || !detail.Executing {
t.Fatalf("run B stopped while A finished: %+v %v", detail, err)
}
close(releases[second.Workspace])
done = waitTerminal(t, r, second.ID)
if done.Status != "succeeded" || strings.Contains(done.Diagnostic, "agent quiescence not confirmed") {
t.Fatalf("run B failed despite its own session being idle: %+v", done)
}
r.wg.Wait()
}
// waitEnteredString fails fast instead of hanging when a stage never starts; it
// carries the entering workspace for tests that key on it.
func waitEnteredString(t *testing.T, entered <-chan string) {
t.Helper()
select {
case <-entered:
case <-time.After(10 * time.Second):
t.Fatal("stage execution did not start")
}
}
// TestOwnActiveSessionStillFailsRun proves a run still fails when one of its OWN
// agent sessions is left active after the stage completes.
func TestOwnActiveSessionStillFailsRun(t *testing.T) {
_, r, pipelineID := agentStore(t)
workspace := t.TempDir()
ready := make(chan struct{})
var runID int64
r.execute = func(_ context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
<-ready
if err := r.store.RunSession(context.Background(), runID, 1, "ses_own", true, "v2", ""); err != nil {
return stage.Result{}, err
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
run, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "own", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
runID = run.ID
close(ready)
r.wg.Wait()
detail, err := r.store.GetRun(context.Background(), run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "agent quiescence not confirmed") {
t.Fatalf("own active session did not fail the run: %+v", detail)
}
}
+9 -4
View File
@@ -160,12 +160,17 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
if r.ctx.Err() != nil {
return nil, r.ctx.Err()
}
sessions, err := r.store.ActiveAgentSessions(r.ctx)
owners, err := r.store.ActiveSessionOwners(r.ctx)
if err != nil {
return nil, err
}
if len(sessions) > 0 && len(r.active) == 0 {
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
// Refuse only an orphaned session: one whose owning run is not active now.
// A session owned by a currently active run (another run in another
// workspace) must not block new work.
for _, owner := range owners {
if _, ok := r.active[owner.RunID]; !ok {
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
}
}
// Idempotent repeats are accepted even while the original run is active.
if len(r.active) > 0 {
@@ -489,7 +494,7 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail,
}
input = stage.Values(step.Output)
if useAgent || errors.Is(err, stage.ErrStageCleanup) {
sessions, checkErr := r.store.ActiveAgentSessions(context.Background())
sessions, checkErr := r.store.ActiveAgentSessionsForRun(context.Background(), id)
if checkErr != nil || len(sessions) > 0 {
finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work")
return
+13 -9
View File
@@ -201,9 +201,12 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string
}
}
// sessionActive reports whether sessionID is currently active. Missing, null or
// non-object data, and any state other than the documented "running" type, are
// errors so a bad response can never be mistaken for a settled session.
// sessionActive reports whether sessionID is currently active. Only the target
// session is examined; entries for other sessions are ignored entirely, so a
// foreign session's state never affects this wait. The target's absence is
// inactive. Missing, null or non-object data, and the target present with any
// state other than the documented "running" type, are errors so a bad response
// can never be mistaken for a settled session.
func (c OpenCodeClient) sessionActive(ctx context.Context, sessionID string) (bool, error) {
data, err := c.call(ctx, "get", "/api/session/active", nil)
if err != nil {
@@ -215,13 +218,14 @@ func (c OpenCodeClient) sessionActive(ctx context.Context, sessionID string) (bo
if err := json.Unmarshal(data, &response); err != nil || response.Data == nil {
return false, fmt.Errorf("%w: OpenCode returned an invalid active-session response", ErrHarnessTransport)
}
for _, state := range response.Data {
if state.Type != "running" {
return false, fmt.Errorf("%w: OpenCode reported an unsupported active-session state", ErrHarnessTransport)
}
state, ok := response.Data[sessionID]
if !ok {
return false, nil
}
_, active := response.Data[sessionID]
return active, nil
if state.Type != "running" {
return false, fmt.Errorf("%w: OpenCode reported an unsupported active-session state", ErrHarnessTransport)
}
return true, nil
}
func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) {
+54 -1
View File
@@ -40,7 +40,6 @@ func TestWaitForSessionIdleFailsClosedOnInvalidActiveResponse(t *testing.T) {
``, `not json`, `{}`, `{"data":null}`, `{"data":[]}`, `{"data":"running"}`,
`{"data":{"ses_wait":"running"}}`, `{"data":{"ses_wait":{}}}`,
`{"data":{"ses_wait":{"type":""}}}`, `{"data":{"ses_wait":{"type":"idle"}}}`,
`{"data":{"ses_wait":{"type":"running"},"ses_other":{"type":"unknown"}}}`,
} {
client := OpenCodeClient{sessionPollInterval: time.Millisecond, command: func(_ context.Context, _ string, _ ...string) ([]byte, error) {
return []byte(payload), nil
@@ -58,6 +57,60 @@ func TestWaitForSessionIdleFailsClosedOnInvalidActiveResponse(t *testing.T) {
}
}
// TestWaitForSessionIdleScopesToTargetSession proves the poll examines only the
// admitted session: a foreign session in an unsupported state is ignored (the
// target's absence is inactive), the target running keeps polling while that
// foreign entry exists, while the target's own unsupported state still fails
// closed and malformed/missing/non-object responses still fail closed.
func TestWaitForSessionIdleScopesToTargetSession(t *testing.T) {
// (a) Foreign session in an unsupported state, target absent: inactive, one
// poll, no error.
polls := 0
foreign := OpenCodeClient{sessionPollInterval: time.Millisecond, command: func(_ context.Context, _ string, _ ...string) ([]byte, error) {
polls++
return []byte(`{"data":{"ses_other":{"type":"unknown"}}}`), nil
}}
if err := foreign.waitForSessionIdle(context.Background(), "ses_wait"); err != nil {
t.Fatalf("foreign unsupported state failed the target wait: %v", err)
}
if polls != 1 {
t.Fatalf("foreign-only response should settle on one poll, got %d", polls)
}
// (b) Target running alongside a foreign unsupported entry keeps polling
// until the target leaves; assert a bounded poll count.
polls = 0
transition := OpenCodeClient{sessionPollInterval: time.Millisecond, command: func(_ context.Context, _ string, _ ...string) ([]byte, error) {
polls++
if polls < 3 {
return []byte(`{"data":{"ses_wait":{"type":"running"},"ses_other":{"type":"unknown"}}}`), nil
}
return []byte(`{"data":{"ses_other":{"type":"unknown"}}}`), nil
}}
if err := transition.waitForSessionIdle(context.Background(), "ses_wait"); err != nil {
t.Fatalf("wait did not settle after the target left: %v", err)
}
if polls != 3 {
t.Fatalf("expected 3 bounded polls, got %d", polls)
}
// (c) Target present with an unsupported type still fails closed even while
// a foreign entry exists.
unsupported := OpenCodeClient{sessionPollInterval: time.Millisecond, command: func(_ context.Context, _ string, _ ...string) ([]byte, error) {
return []byte(`{"data":{"ses_wait":{"type":"idle"},"ses_other":{"type":"unknown"}}}`), nil
}}
if err := unsupported.waitForSessionIdle(context.Background(), "ses_wait"); !errors.Is(err, ErrHarnessTransport) {
t.Fatalf("target unsupported state did not fail closed: %v", err)
}
// (d) Empty/malformed/missing/non-object data still fail closed.
for _, payload := range []string{``, `not json`, `{}`, `{"data":null}`, `{"data":[]}`, `{"data":"running"}`} {
malformed := OpenCodeClient{sessionPollInterval: time.Millisecond, command: func(_ context.Context, _ string, _ ...string) ([]byte, error) {
return []byte(payload), nil
}}
if err := malformed.waitForSessionIdle(context.Background(), "ses_wait"); !errors.Is(err, ErrHarnessTransport) {
t.Fatalf("malformed active response accepted: %q: %v", payload, err)
}
}
}
// TestWaitForSessionIdlePropagatesRequestFailure ensures a transport/CLI error
// aborts promptly and is surfaced rather than swallowed.
func TestWaitForSessionIdlePropagatesRequestFailure(t *testing.T) {