Files
pipelines/internal/server/runner.go
T
alan 2618f0d244 Allow concurrent runs in different workspaces
Track each active run's resolved workspace alongside its cancel function
in a single runner.active struct map. A new request key is now refused
only when an already-active run is changing the same resolved workspace;
the refusal names the conflicting run id and advises waiting or using a
different workspace. Workspace resolution is unchanged (default cwd,
absolute request workspace, EvalSymlinks), so two spellings of one
directory still compare equal. The conflict check and registration stay
inside the existing r.mu critical section, so concurrent starts for one
workspace cannot both pass. Idempotent repeats of an active request key
and the restart-recovery guard are unchanged.

Add concurrency_test.go covering concurrent different-workspace runs,
same-workspace refusal, symlink/default-cwd equivalence, idempotent
repeats, restart recovery and retry workspace scoping; strengthen the
runner_test.go refusal assertion. Update docs/cli.md,
docs/first-version-testing.md and the packaged operations skill reference
to describe per-workspace concurrency.

Assisted-by: DeepSeek V4.1 Flash
2026-10-15 15:08:44 +01:00

518 lines
17 KiB
Go

package server
import (
"context"
"errors"
"fmt"
"log"
"maps"
"os"
"path/filepath"
"strings"
"sync"
"time"
"git.a-wan.net/alan/pipelines/internal/filediff"
"git.a-wan.net/alan/pipelines/internal/harness"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// activeRun records one in-flight run's cancellation seam and the resolved
// workspace it is changing. A single map is the only active-run state, so the
// workspace cannot drift from the cancel function.
type activeRun struct {
cancel context.CancelFunc
workspace string
}
type runner struct {
ctx context.Context
store *pipeline.Store
mu sync.Mutex
active map[int64]activeRun
execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error)
wg sync.WaitGroup
schedulerWG sync.WaitGroup
}
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
sessions, err := s.ActiveSessionOwners(ctx)
if err != nil {
return nil, err
}
for _, owner := range sessions {
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
var err error
if owner.Backend == "pi" {
err = stage.InterruptPi(stop, owner.Reference)
} else {
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID)
}
cancel()
if err != nil {
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
}
}
if err := s.RecoverRuns(ctx); err != nil {
return nil, err
}
return &runner{ctx: ctx, store: s, active: map[int64]activeRun{}}, nil
}
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 {
return nil, fmt.Errorf("an explicit version and request key are required")
}
workspace, err := os.Getwd()
if err != nil {
return nil, err
}
workspace, err = filepath.EvalSymlinks(workspace)
if err != nil {
return nil, err
}
if request.Workspace != "" {
if !filepath.IsAbs(request.Workspace) {
return nil, fmt.Errorf("run workspace must be an absolute directory")
}
selected, err := filepath.EvalSymlinks(request.Workspace)
if err != nil {
return nil, fmt.Errorf("run workspace is unavailable on the execution server: %q: %w; for Docker, mount this host directory at the same absolute path and use its owner's UID/GID", request.Workspace, err)
}
info, err := os.Stat(selected)
if err != nil || !info.IsDir() {
return nil, fmt.Errorf("run workspace %q must be an accessible directory on the execution server", request.Workspace)
}
workspace = selected
}
if len(request.WritePaths) > 20 {
return nil, fmt.Errorf("too many write paths")
}
for _, path := range request.WritePaths {
if filepath.IsAbs(path) || filepath.Clean(path) != path || path == ".." || strings.HasPrefix(path, ".."+string(os.PathSeparator)) || strings.ContainsAny(path, "*?\\") {
return nil, fmt.Errorf("optional write paths must be exact run-directory-relative paths")
}
parent := filepath.Join(workspace, path)
for {
_, err := os.Lstat(parent)
if err == nil {
break
}
if !os.IsNotExist(err) {
return nil, err
}
next := filepath.Dir(parent)
if next == parent {
return nil, fmt.Errorf("write path is unavailable")
}
parent = next
}
resolved, err := filepath.EvalSymlinks(parent)
if err != nil || (resolved != workspace && !strings.HasPrefix(resolved, workspace+string(os.PathSeparator))) {
return nil, fmt.Errorf("write path must stay within the run directory")
}
}
request.Input = maps.Clone(request.Input)
if request.Input == nil {
request.Input = map[string]any{}
}
request.Input["pwd"] = workspace
if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" {
return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt")
}
d, err := r.store.Get(r.ctx, request.PipelineID, request.Version)
if err != nil {
return nil, err
}
if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil {
return nil, err
}
for _, s := range d.Graph.Stages {
if stage.EffectiveType(s.Type) != stage.ModelType {
continue
}
if s.Configs["harness"] == "pi" && r.execute == nil {
configured, err := harness.Load()
if err != nil {
return nil, err
}
if _, ok := configured.Profiles["pi"]; !ok {
return nil, fmt.Errorf("Pi is not configured on this execution server; run pipelines harness setup pi there")
}
}
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s))
if err != nil {
return nil, err
}
if !explicit {
request.AllowTools = true
}
for _, tool := range tools {
if tool == "edit" || tool == "shell" {
request.AllowTools = true
}
}
}
r.mu.Lock()
defer r.mu.Unlock()
if r.ctx.Err() != nil {
return nil, r.ctx.Err()
}
sessions, err := r.store.ActiveAgentSessions(r.ctx)
if err != nil {
return nil, err
}
if len(sessions) > 0 && len(r.active) == 0 {
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
}
// Idempotent repeats are accepted even while the original run is active.
if len(r.active) > 0 {
existing, err := r.store.RunByKey(r.ctx, request.RequestKey)
if err != nil {
return nil, err
}
if existing == 0 {
// The workspace is the only scope a run reserves: refuse a second
// run only when an active run is already changing this exact
// resolved directory. One active run per workspace keeps this scan
// finding at most one match, so the message is deterministic.
for id, entry := range r.active {
if entry.workspace == workspace {
return nil, fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
}
}
}
}
id, created, err := r.store.CreateRun(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, retryOf...)
if err != nil {
return nil, err
}
if created {
ctx, cancel := context.WithCancel(r.ctx)
r.active[id] = activeRun{cancel: cancel, workspace: workspace}
r.wg.Add(1)
go func() {
defer r.wg.Done()
defer cancel()
defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }()
r.run(ctx, id, d, workspace, request)
}()
}
detail, err := r.store.GetRun(r.ctx, id)
if detail != nil {
_, detail.Executing = r.active[id]
}
return detail, err
}
func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
r.mu.Lock()
defer r.mu.Unlock()
d, err := r.store.GetRun(r.ctx, id)
if d != nil {
_, d.Executing = r.active[id]
for _, s := range d.Steps {
if s.AgentActive {
d.Executing = true
}
}
}
return d, err
}
func (r *runner) listRuns() ([]protocol.RunSummary, error) {
r.mu.Lock()
defer r.mu.Unlock()
items, err := r.store.ListRuns(r.ctx)
for i := range items {
_, items[i].Executing = r.active[items[i].ID]
if items[i].Status == "failed" && !items[i].Executing {
d, err := r.store.GetRun(r.ctx, items[i].ID)
if err != nil {
return nil, err
}
for _, step := range d.Steps {
if step.AgentActive {
items[i].Executing = true
}
}
}
}
return items, err
}
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
if request.RunID <= 0 {
return nil, fmt.Errorf("retry requires a positive run ID")
}
original, err := r.getRun(request.RunID)
if err != nil {
return nil, err
}
if original.Executing {
return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID)
}
if original.Status != "failed" || original.FinishedAt == "" {
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
}
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths}, original.ID)
}
func (r *runner) cancel(id int64) error {
r.mu.Lock()
defer r.mu.Unlock()
entry, ok := r.active[id]
if !ok {
return fmt.Errorf("run is not active")
}
entry.cancel()
return nil
}
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
input := stage.Values(request.Input)
key := d.Graph.Stages[0].Key
visits := map[string]int{}
budgets := map[string]*stage.Budget{}
finish := func(status, outcome, diagnostic string) {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
log.Printf("finish run %d: %v", id, err)
}
}
original, copyErr := copyValues(stage.Values(request.Input))
if copyErr != nil {
finish("failed", "error", copyErr.Error())
return
}
memory := map[string]memoryValue{"input": {output: original}}
defer func() {
if recovered := recover(); recovered != nil {
finish("failed", "error", "stage execution panicked")
}
}()
for sequence := 1; sequence <= 30; sequence++ {
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
visits[key]++
if visits[key] > 5 {
finish("failed", "error", "stage visit budget exceeded")
return
}
var def stage.Definition
var selected protocol.Stage
found := false
for _, s := range d.Graph.Stages {
if s.Key == key {
selected = s
def = pipeline.StageDefinition(s)
found = true
break
}
}
if !found {
finish("failed", "error", "missing stage")
return
}
maxVisits := 5
if selected.Limits != nil {
maxVisits = selected.Limits.MaxVisits
}
if visits[key] > maxVisits {
finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits))
return
}
var reads map[string]protocol.MemoryRead
input, reads, copyErr = memoryInput(input, selected.Memory, memory)
if copyErr != nil {
finish("failed", "error", copyErr.Error())
return
}
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
finish("failed", "error", err.Error())
return
}
var tools []string
explicitTools := false
var toolErr error
if stage.EffectiveType(def.Type) == stage.ModelType {
tools, explicitTools, toolErr = stage.ToolNames(def)
}
if toolErr != nil {
finish("failed", "error", toolErr.Error())
return
}
useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools)
operations := &stage.OperationLog{}
client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations}, WritePaths: request.WritePaths, Tools: tools}
client.OnSession = func(session string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference)
}
var result stage.Result
var piClient *stage.PiClient
var err error
stageCtx := ctx
var modelConfig stage.ModelConfig
if stage.EffectiveType(def.Type) == stage.ModelType {
spec, specErr := (stage.ModelStage{}).Spec(def)
if specErr != nil {
finish("failed", "error", specErr.Error())
return
}
modelConfig = spec.Config
if budgets[key] == nil {
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
}
stageCtx = stage.WithBudget(ctx, budgets[key])
}
snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second)
before := filediff.Capture(snapshotCtx, workspace)
snapshotCancel()
var stageSession string
panicked := false
func() {
defer func() {
if recover() != nil {
panicked = true
err = fmt.Errorf("stage execution panicked")
}
}()
if r.execute != nil {
timeout, _ := stage.StageTimeout(def)
executeCtx, cancel := context.WithTimeout(stageCtx, timeout)
defer cancel()
result, err = r.execute(executeCtx, def, input)
} else if modelConfig.Harness == "pi" {
configured, configErr := harness.Load()
if configErr != nil {
err = configErr
return
}
profile, ok := configured.Profiles["pi"]
if !ok {
err = fmt.Errorf("Pi is not configured on execution server")
return
}
piClient = &stage.PiClient{Executable: profile.Executable, Directory: workspace, Tools: tools, WritePaths: request.WritePaths}
piClient.OnOwnership = func(session, reference string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return r.store.RunSession(c, id, sequence, session, active, "pi", reference)
}
result, err = (stage.ModelStage{Client: piClient}).Run(stageCtx, def, input)
stageSession = piClient.SessionID
client.Transcript = piClient.Transcript
} else if useAgent {
result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input)
stageSession = client.SessionID
} else if stage.EffectiveType(def.Type) == stage.ModelType {
plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations}
plain.OnGenerationSession = func(session, backend, reference string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if active {
stageSession = session
}
return r.store.RunSession(c, id, sequence, session, active, backend, reference)
}
result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input)
client.Transcript = plain.FormatAudit
} else {
result, err = stage.DefaultRegistry().Run(ctx, def, input)
}
}()
// Capture after cleanup, even when the stage's context was canceled.
snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
after := filediff.Capture(snapshotCtx, workspace, before)
snapshotCancel()
changes := filediff.Compare(before, after)
if errors.Is(err, stage.ErrStageCleanup) {
changes.Status = "partial"
changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing."
}
step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes}
if budget := budgets[key]; budget != nil {
step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries)
}
if err != nil {
step.Status = "failed"
step.Diagnostic = err.Error()
step.Outcome = "error"
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
}
step.Target = def.Edges[step.Outcome]
step.Control = &protocol.ControlEvidence{Visit: visits[key], MaxVisits: maxVisits, Memory: reads}
if err == nil && selected.Limits != nil && visits[key] >= maxVisits {
for _, outcome := range selected.Limits.On {
if outcome == step.Outcome {
step.Control.LimitReached = true
step.Control.ModelTarget = step.Target
step.Target = protocol.FailureEnd
step.Diagnostic = fmt.Sprintf("Pipelines stopped %s after %d visits on outcome %s", key, maxVisits, step.Outcome)
}
}
}
if panicked {
step.Target = protocol.FailureEnd
}
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
persistErr := r.store.FinishStep(save, id, step)
cancel()
if persistErr != nil {
finish("failed", "error", persistErr.Error())
return
}
if step.Status == "completed" {
values, err := copyValues(stage.Values(step.Output))
if err != nil {
finish("failed", "error", err.Error())
return
}
memory[key] = memoryValue{output: values, sequence: sequence}
}
for _, s := range d.Graph.Stages {
if s.Key == key {
r.notifyStep(id, d, workspace, s.Notify, step)
break
}
}
input = stage.Values(step.Output)
if useAgent || errors.Is(err, stage.ErrStageCleanup) {
sessions, checkErr := r.store.ActiveAgentSessions(context.Background())
if checkErr != nil || len(sessions) > 0 {
finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work")
return
}
}
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
if step.Target == protocol.SuccessEnd {
finish("succeeded", "success", "")
return
}
if step.Target == protocol.FailureEnd {
finish("failed", "error", step.Diagnostic)
return
}
if step.Target == "" {
finish("failed", "error", "stage returned an undeclared outcome")
return
}
key = step.Target
}
finish("failed", "error", "total stage budget exceeded")
}