From e6d00d4ee5e6c9d6a3a06683f4ab4648991e6f76 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Thu, 15 Oct 2026 15:38:50 +0100 Subject: [PATCH] Scope agent session bookkeeping to the run for concurrent runs The per-workspace concurrency added in 2618f0d exposed three places that assumed a single run, and the second concurrent run failed with 'agent quiescence not confirmed' because of them. - ActiveAgentSessionsForRun(runID) scopes the end-of-step quiescence check to the run's own sessions, so another run's legitimately active agent no longer fails this one. A run still fails when its own session cannot be confirmed idle. - The startup guard now uses ActiveSessionOwners (which carries the owning run id) and refuses only an ORPHANED session - one whose run is not active now - which is what restart recovery is actually about. A session owned by a currently active run no longer blocks new work. - sessionActive examines only the target session: foreign entries in the /api/session/active map are ignored, the target's absence is inactive, and the target present with an unsupported state still fails closed. Verified: go build, go vet and go test ./... pass, including ten concurrency tests (two workspaces run concurrently, same-workspace refusal, symlink and default-cwd equivalence, idempotent repeats, restart recovery, retry scoping, overlapping runs not failing quiescence, and a run's own active session still failing it). Operator verification, which a stage cannot perform because restarting the server would abort the run doing it: the service was rebuilt and restarted on this commit, then two runs in separate workspaces (/tmp/ws-c and /tmp/ws-d) executed concurrently - both reported executing at the same time, a third run in ws-c was refused naming run 46, and both finished succeeded with no diagnostic. --- .../author/skill/references/operations.md | 6 +- internal/pipeline/runs.go | 22 ++- internal/pipeline/runs_test.go | 60 +++++- internal/server/concurrency_test.go | 175 +++++++++++++++++- internal/server/runner.go | 13 +- internal/stage/opencode.go | 22 ++- internal/stage/wait_test.go | 55 +++++- 7 files changed, 330 insertions(+), 23 deletions(-) diff --git a/internal/author/skill/references/operations.md b/internal/author/skill/references/operations.md index 52dfc044..52c9901e 100644 --- a/internal/author/skill/references/operations.md +++ b/internal/author/skill/references/operations.md @@ -25,7 +25,11 @@ parallel/distributed task engine or a guarantee that model reports are correct. resolved workspaces execute concurrently; only a second run in a workspace an active run is already changing is refused, naming the conflicting run and advising you to wait or use a different workspace (for example another - checkout). Reusing that active run's request key reattaches idempotently. + checkout). Agent-session quiescence is scoped to the run: a run fails only when + one of its own sessions cannot be confirmed idle, so one run's active agent never + fails another's concurrent run. An active session left over by a restart with no + owning active run is an orphan and still refuses new work until interrupted. + Reusing that active run's request key reattaches idempotently. There are no parallel branches or built-in human approval stages. Optional worktrees, full backup/restore, automatic database sync and independent verifier stage types are future ideas, not existing capabilities. Do not diff --git a/internal/pipeline/runs.go b/internal/pipeline/runs.go index 94cf4b0f..df08f7ac 100644 --- a/internal/pipeline/runs.go +++ b/internal/pipeline/runs.go @@ -313,7 +313,15 @@ func (s *Store) RunSession(ctx context.Context, id int64, sequence int, session return tx.Commit() } -type SessionOwner struct{ ID, Backend, Directory, Reference string } +type SessionOwner struct { + ID string + Backend string + Directory string + Reference string + // RunID is the owning run, used to tell an orphaned session (no active run) + // from one owned by a run that is active right now. + RunID int64 +} func (s *Store) RunSessionOwner(ctx context.Context, id int64, sequence int) (SessionOwner, error) { var owner SessionOwner @@ -322,7 +330,7 @@ func (s *Store) RunSessionOwner(ctx context.Context, id int64, sequence int) (Se } func (s *Store) ActiveSessionOwners(ctx context.Context) ([]SessionOwner, error) { - rows, err := s.db.QueryContext(ctx, `SELECT s.session_id,COALESCE(o.backend,'v2'),r.workspace,COALESCE(o.reference,'') + rows, err := s.db.QueryContext(ctx, `SELECT s.session_id,COALESCE(o.backend,'v2'),r.workspace,COALESCE(o.reference,''),s.run_id FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id LEFT JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence WHERE s.agent_active=1 AND s.session_id<>''`) if err != nil { return nil, err @@ -331,7 +339,7 @@ FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id LEFT JOIN run_session_own var owners []SessionOwner for rows.Next() { var owner SessionOwner - if err = rows.Scan(&owner.ID, &owner.Backend, &owner.Directory, &owner.Reference); err != nil { + if err = rows.Scan(&owner.ID, &owner.Backend, &owner.Directory, &owner.Reference, &owner.RunID); err != nil { return nil, err } owners = append(owners, owner) @@ -386,8 +394,12 @@ func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagno return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id)) } -func (s *Store) ActiveAgentSessions(ctx context.Context) ([]string, error) { - rows, err := s.db.QueryContext(ctx, "SELECT DISTINCT session_id FROM run_steps WHERE agent_active=1 AND session_id<>''") +// ActiveAgentSessionsForRun returns the distinct active agent session ids that +// belong to one run. It is scoped to run_steps.run_id so concurrent runs never +// observe each other's sessions: run_steps is keyed by (run_id, sequence), so +// this is one indexed lookup, not a scan. +func (s *Store) ActiveAgentSessionsForRun(ctx context.Context, runID int64) ([]string, error) { + rows, err := s.db.QueryContext(ctx, "SELECT DISTINCT session_id FROM run_steps WHERE run_id=? AND agent_active=1 AND session_id<>''", runID) if err != nil { return nil, err } diff --git a/internal/pipeline/runs_test.go b/internal/pipeline/runs_test.go index c281c479..bd1a281c 100644 --- a/internal/pipeline/runs_test.go +++ b/internal/pipeline/runs_test.go @@ -68,8 +68,8 @@ func TestRunRecoveryAndVersionPinning(t *testing.T) { t.Fatal(err) } defer store.Close() - sessions, err := store.ActiveAgentSessions(ctx) - if err != nil || len(sessions) != 1 { + owners, err := store.ActiveSessionOwners(ctx) + if err != nil || len(owners) != 1 { t.Fatal("lost active session ownership") } if err := store.RecoverRuns(ctx); err != nil { @@ -93,3 +93,59 @@ func TestRunRecoveryAndVersionPinning(t *testing.T) { t.Fatal("run modified immutable source") } } + +// TestActiveAgentSessionsAreScopedToRun proves the run-scoped query returns only +// the row's own active session: a second run's active session is never returned, +// and clearing the session clears the result. +func TestActiveAgentSessionsAreScopedToRun(t *testing.T) { + ctx := context.Background() + store, err := Open(filepath.Join(t.TempDir(), "scoped.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, err := store.Submit(ctx, "scoped", Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + definition, err := store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + runA, _, err := store.CreateRun(ctx, definition, "a", "workspace-a", false, nil, nil) + if err != nil { + t.Fatal(err) + } + runB, _, err := store.CreateRun(ctx, definition, "b", "workspace-b", false, nil, nil) + if err != nil { + t.Fatal(err) + } + for _, run := range []struct { + id int64 + session string + }{{runA, "ses_a"}, {runB, "ses_b"}} { + if err := store.BeginStep(ctx, run.id, 1, "one", map[string]any{"prompt": "task"}); err != nil { + t.Fatal(err) + } + if err := store.RunSession(ctx, run.id, 1, run.session, true); err != nil { + t.Fatal(err) + } + } + sessions, err := store.ActiveAgentSessionsForRun(ctx, runA) + if err != nil || !reflect.DeepEqual(sessions, []string{"ses_a"}) { + t.Fatalf("run-scoped query leaked another run's session: %v %v", sessions, err) + } + if err := store.RunSession(ctx, runA, 1, "ses_a", false); err != nil { + t.Fatal(err) + } + sessions, err = store.ActiveAgentSessionsForRun(ctx, runA) + if err != nil || len(sessions) != 0 { + t.Fatalf("inactive session still reported active: %v %v", sessions, err) + } + if sessions, err = store.ActiveAgentSessionsForRun(ctx, runB); err != nil || !reflect.DeepEqual(sessions, []string{"ses_b"}) { + t.Fatalf("second run's own session was affected: %v %v", sessions, err) + } +} diff --git a/internal/server/concurrency_test.go b/internal/server/concurrency_test.go index d99b1dbc..354f13b2 100644 --- a/internal/server/concurrency_test.go +++ b/internal/server/concurrency_test.go @@ -16,6 +16,16 @@ import ( ) func concurrentStore(t *testing.T) (context.Context, *runner, int64) { + t.Helper() + return newConcurrentRunner(t, pipeline.Template([]string{"one"}, true)) +} + +// newConcurrentRunner builds the shared test store/runner around a submitted +// pipeline source. Tests that need an agent stage (useAgent true) must pass a +// source with a non-empty tools block: the default template's empty tools block +// explicitly denies tools, so useAgent is false and the step-completion +// quiescence check is never exercised. +func newConcurrentRunner(t *testing.T, source string) (context.Context, *runner, int64) { t.Helper() ctx, cancel := context.WithCancel(context.Background()) store, err := pipeline.Open(filepath.Join(t.TempDir(), "concurrent.db")) @@ -23,7 +33,7 @@ func concurrentStore(t *testing.T) (context.Context, *runner, int64) { cancel() t.Fatal(err) } - if _, err := store.Submit(ctx, "test", pipeline.Template([]string{"one"}, true)); err != nil { + if _, err := store.Submit(ctx, "test", source); err != nil { cancel() store.Close() t.Fatal(err) @@ -48,6 +58,14 @@ func concurrentStore(t *testing.T) (context.Context, *runner, int64) { return ctx, r, items[0].ID } +// agentStore returns a runner whose pipeline has an agent stage (a non-empty, +// explicitly declared tools block), so the step-completion quiescence check runs. +func agentStore(t *testing.T) (context.Context, *runner, int64) { + t.Helper() + source := strings.Replace(pipeline.Template([]string{"one"}, true), "```tools\n```", "```tools\nread\nshell\n```", 1) + return newConcurrentRunner(t, source) +} + // waitEntered fails fast instead of hanging when a stage never starts. func waitEntered(t *testing.T, entered <-chan struct{}) { t.Helper() @@ -344,3 +362,158 @@ func TestRetryConflictsOnlyWithSameWorkspaceActiveRun(t *testing.T) { close(release) r.wg.Wait() } + +// waitTerminal polls one run until it leaves queued/running, failing fast instead +// of hanging. It is scoped to the run so a blocked sibling does not stall it. +func waitTerminal(t *testing.T, r *runner, id int64) *protocol.RunDetail { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + detail, err := r.getRun(id) + if err != nil { + t.Fatal(err) + } + switch detail.Status { + case "succeeded", "failed", "canceled": + return detail + } + if time.Now().After(deadline) { + t.Fatalf("run %d did not reach a terminal state: %+v", id, detail) + } + time.Sleep(5 * time.Millisecond) + } +} + +// TestOverlappingRunsDoNotFailQuiescence proves the step-completion check is +// scoped to the run: with two agent runs in different workspaces each recording +// its OWN session active, releasing run A lets A reach succeeded while run B's +// session is still active, and vice versa. A server-wide check would fail A +// deterministically against B's active session. +func TestOverlappingRunsDoNotFailQuiescence(t *testing.T) { + _, r, pipelineID := agentStore(t) + a, b := t.TempDir(), t.TempDir() + + type agentRun struct { + id int64 + session string + } + var mu sync.Mutex + byWorkspace := map[string]agentRun{} + releases := map[string]chan struct{}{} + ready := make(chan struct{}) + entered := make(chan string, 2) + resolve := func(dir string) string { + t.Helper() + resolved, err := filepath.EvalSymlinks(dir) + if err != nil { + t.Fatal(err) + } + return resolved + } + r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + <-ready + workspace, _ := input["pwd"].(string) + mu.Lock() + agent, ok := byWorkspace[workspace] + release := releases[workspace] + mu.Unlock() + if !ok { + return stage.Result{}, fmt.Errorf("execute for unknown workspace %q", workspace) + } + if err := r.store.RunSession(context.Background(), agent.id, 1, agent.session, true, "v2", ""); err != nil { + return stage.Result{}, err + } + entered <- workspace + select { + case <-release: + case <-ctx.Done(): + return stage.Result{}, ctx.Err() + } + if err := r.store.RunSession(context.Background(), agent.id, 1, agent.session, false, "v2", ""); err != nil { + return stage.Result{}, err + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + first, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "a", Workspace: a, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + second, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "b", Workspace: b, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatalf("second workspace was blocked by the first: %v", err) + } + if first.Workspace != resolve(a) || second.Workspace != resolve(b) { + t.Fatalf("unexpected resolved workspaces: %q, %q", first.Workspace, second.Workspace) + } + mu.Lock() + byWorkspace[first.Workspace] = agentRun{id: first.ID, session: "ses_a"} + byWorkspace[second.Workspace] = agentRun{id: second.ID, session: "ses_b"} + releases[first.Workspace] = make(chan struct{}) + releases[second.Workspace] = make(chan struct{}) + mu.Unlock() + close(ready) + waitEnteredString(t, entered) + waitEnteredString(t, entered) + for _, id := range []int64{first.ID, second.ID} { + detail, err := r.getRun(id) + if err != nil || !detail.Executing { + t.Fatalf("run %d was not reported executing: %v", id, err) + } + } + // Release A only; B's own session stays active while A finishes. + close(releases[first.Workspace]) + done := waitTerminal(t, r, first.ID) + if done.Status != "succeeded" || strings.Contains(done.Diagnostic, "agent quiescence not confirmed") { + t.Fatalf("run A failed despite its own session being idle: %+v", done) + } + if detail, err := r.getRun(second.ID); err != nil || !detail.Executing { + t.Fatalf("run B stopped while A finished: %+v %v", detail, err) + } + close(releases[second.Workspace]) + done = waitTerminal(t, r, second.ID) + if done.Status != "succeeded" || strings.Contains(done.Diagnostic, "agent quiescence not confirmed") { + t.Fatalf("run B failed despite its own session being idle: %+v", done) + } + r.wg.Wait() +} + +// waitEnteredString fails fast instead of hanging when a stage never starts; it +// carries the entering workspace for tests that key on it. +func waitEnteredString(t *testing.T, entered <-chan string) { + t.Helper() + select { + case <-entered: + case <-time.After(10 * time.Second): + t.Fatal("stage execution did not start") + } +} + +// TestOwnActiveSessionStillFailsRun proves a run still fails when one of its OWN +// agent sessions is left active after the stage completes. +func TestOwnActiveSessionStillFailsRun(t *testing.T) { + _, r, pipelineID := agentStore(t) + workspace := t.TempDir() + ready := make(chan struct{}) + var runID int64 + r.execute = func(_ context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) { + <-ready + if err := r.store.RunSession(context.Background(), runID, 1, "ses_own", true, "v2", ""); err != nil { + return stage.Result{}, err + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + run, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: "own", Workspace: workspace, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + runID = run.ID + close(ready) + r.wg.Wait() + detail, err := r.store.GetRun(context.Background(), run.ID) + if err != nil { + t.Fatal(err) + } + if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "agent quiescence not confirmed") { + t.Fatalf("own active session did not fail the run: %+v", detail) + } +} diff --git a/internal/server/runner.go b/internal/server/runner.go index abb2f992..5b8058d7 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -160,12 +160,17 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru if r.ctx.Err() != nil { return nil, r.ctx.Err() } - sessions, err := r.store.ActiveAgentSessions(r.ctx) + owners, err := r.store.ActiveSessionOwners(r.ctx) if err != nil { return nil, err } - if len(sessions) > 0 && len(r.active) == 0 { - return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work") + // Refuse only an orphaned session: one whose owning run is not active now. + // A session owned by a currently active run (another run in another + // workspace) must not block new work. + for _, owner := range owners { + if _, ok := r.active[owner.RunID]; !ok { + return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work") + } } // Idempotent repeats are accepted even while the original run is active. if len(r.active) > 0 { @@ -489,7 +494,7 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, } input = stage.Values(step.Output) if useAgent || errors.Is(err, stage.ErrStageCleanup) { - sessions, checkErr := r.store.ActiveAgentSessions(context.Background()) + sessions, checkErr := r.store.ActiveAgentSessionsForRun(context.Background(), id) if checkErr != nil || len(sessions) > 0 { finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work") return diff --git a/internal/stage/opencode.go b/internal/stage/opencode.go index e1fbb7ae..0c6c3c37 100644 --- a/internal/stage/opencode.go +++ b/internal/stage/opencode.go @@ -201,9 +201,12 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string } } -// sessionActive reports whether sessionID is currently active. Missing, null or -// non-object data, and any state other than the documented "running" type, are -// errors so a bad response can never be mistaken for a settled session. +// sessionActive reports whether sessionID is currently active. Only the target +// session is examined; entries for other sessions are ignored entirely, so a +// foreign session's state never affects this wait. The target's absence is +// inactive. Missing, null or non-object data, and the target present with any +// state other than the documented "running" type, are errors so a bad response +// can never be mistaken for a settled session. func (c OpenCodeClient) sessionActive(ctx context.Context, sessionID string) (bool, error) { data, err := c.call(ctx, "get", "/api/session/active", nil) if err != nil { @@ -215,13 +218,14 @@ func (c OpenCodeClient) sessionActive(ctx context.Context, sessionID string) (bo if err := json.Unmarshal(data, &response); err != nil || response.Data == nil { return false, fmt.Errorf("%w: OpenCode returned an invalid active-session response", ErrHarnessTransport) } - for _, state := range response.Data { - if state.Type != "running" { - return false, fmt.Errorf("%w: OpenCode reported an unsupported active-session state", ErrHarnessTransport) - } + state, ok := response.Data[sessionID] + if !ok { + return false, nil } - _, active := response.Data[sessionID] - return active, nil + if state.Type != "running" { + return false, fmt.Errorf("%w: OpenCode reported an unsupported active-session state", ErrHarnessTransport) + } + return true, nil } func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) { diff --git a/internal/stage/wait_test.go b/internal/stage/wait_test.go index 6c1d3759..23ae3453 100644 --- a/internal/stage/wait_test.go +++ b/internal/stage/wait_test.go @@ -40,7 +40,6 @@ func TestWaitForSessionIdleFailsClosedOnInvalidActiveResponse(t *testing.T) { ``, `not json`, `{}`, `{"data":null}`, `{"data":[]}`, `{"data":"running"}`, `{"data":{"ses_wait":"running"}}`, `{"data":{"ses_wait":{}}}`, `{"data":{"ses_wait":{"type":""}}}`, `{"data":{"ses_wait":{"type":"idle"}}}`, - `{"data":{"ses_wait":{"type":"running"},"ses_other":{"type":"unknown"}}}`, } { client := OpenCodeClient{sessionPollInterval: time.Millisecond, command: func(_ context.Context, _ string, _ ...string) ([]byte, error) { return []byte(payload), nil @@ -58,6 +57,60 @@ func TestWaitForSessionIdleFailsClosedOnInvalidActiveResponse(t *testing.T) { } } +// TestWaitForSessionIdleScopesToTargetSession proves the poll examines only the +// admitted session: a foreign session in an unsupported state is ignored (the +// target's absence is inactive), the target running keeps polling while that +// foreign entry exists, while the target's own unsupported state still fails +// closed and malformed/missing/non-object responses still fail closed. +func TestWaitForSessionIdleScopesToTargetSession(t *testing.T) { + // (a) Foreign session in an unsupported state, target absent: inactive, one + // poll, no error. + polls := 0 + foreign := OpenCodeClient{sessionPollInterval: time.Millisecond, command: func(_ context.Context, _ string, _ ...string) ([]byte, error) { + polls++ + return []byte(`{"data":{"ses_other":{"type":"unknown"}}}`), nil + }} + if err := foreign.waitForSessionIdle(context.Background(), "ses_wait"); err != nil { + t.Fatalf("foreign unsupported state failed the target wait: %v", err) + } + if polls != 1 { + t.Fatalf("foreign-only response should settle on one poll, got %d", polls) + } + // (b) Target running alongside a foreign unsupported entry keeps polling + // until the target leaves; assert a bounded poll count. + polls = 0 + transition := OpenCodeClient{sessionPollInterval: time.Millisecond, command: func(_ context.Context, _ string, _ ...string) ([]byte, error) { + polls++ + if polls < 3 { + return []byte(`{"data":{"ses_wait":{"type":"running"},"ses_other":{"type":"unknown"}}}`), nil + } + return []byte(`{"data":{"ses_other":{"type":"unknown"}}}`), nil + }} + if err := transition.waitForSessionIdle(context.Background(), "ses_wait"); err != nil { + t.Fatalf("wait did not settle after the target left: %v", err) + } + if polls != 3 { + t.Fatalf("expected 3 bounded polls, got %d", polls) + } + // (c) Target present with an unsupported type still fails closed even while + // a foreign entry exists. + unsupported := OpenCodeClient{sessionPollInterval: time.Millisecond, command: func(_ context.Context, _ string, _ ...string) ([]byte, error) { + return []byte(`{"data":{"ses_wait":{"type":"idle"},"ses_other":{"type":"unknown"}}}`), nil + }} + if err := unsupported.waitForSessionIdle(context.Background(), "ses_wait"); !errors.Is(err, ErrHarnessTransport) { + t.Fatalf("target unsupported state did not fail closed: %v", err) + } + // (d) Empty/malformed/missing/non-object data still fail closed. + for _, payload := range []string{``, `not json`, `{}`, `{"data":null}`, `{"data":[]}`, `{"data":"running"}`} { + malformed := OpenCodeClient{sessionPollInterval: time.Millisecond, command: func(_ context.Context, _ string, _ ...string) ([]byte, error) { + return []byte(payload), nil + }} + if err := malformed.waitForSessionIdle(context.Background(), "ses_wait"); !errors.Is(err, ErrHarnessTransport) { + t.Fatalf("malformed active response accepted: %q: %v", payload, err) + } + } +} + // TestWaitForSessionIdlePropagatesRequestFailure ensures a transport/CLI error // aborts promptly and is surfaced rather than swallowed. func TestWaitForSessionIdlePropagatesRequestFailure(t *testing.T) {