2618f0d244
Track each active run's resolved workspace alongside its cancel function in a single runner.active struct map. A new request key is now refused only when an already-active run is changing the same resolved workspace; the refusal names the conflicting run id and advises waiting or using a different workspace. Workspace resolution is unchanged (default cwd, absolute request workspace, EvalSymlinks), so two spellings of one directory still compare equal. The conflict check and registration stay inside the existing r.mu critical section, so concurrent starts for one workspace cannot both pass. Idempotent repeats of an active request key and the restart-recovery guard are unchanged. Add concurrency_test.go covering concurrent different-workspace runs, same-workspace refusal, symlink/default-cwd equivalence, idempotent repeats, restart recovery and retry workspace scoping; strengthen the runner_test.go refusal assertion. Update docs/cli.md, docs/first-version-testing.md and the packaged operations skill reference to describe per-workspace concurrency. Assisted-by: DeepSeek V4.1 Flash
518 lines
17 KiB
Go
518 lines
17 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"maps"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/filediff"
|
|
"git.a-wan.net/alan/pipelines/internal/harness"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
// activeRun records one in-flight run's cancellation seam and the resolved
|
|
// workspace it is changing. A single map is the only active-run state, so the
|
|
// workspace cannot drift from the cancel function.
|
|
type activeRun struct {
|
|
cancel context.CancelFunc
|
|
workspace string
|
|
}
|
|
|
|
type runner struct {
|
|
ctx context.Context
|
|
store *pipeline.Store
|
|
mu sync.Mutex
|
|
active map[int64]activeRun
|
|
execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error)
|
|
wg sync.WaitGroup
|
|
schedulerWG sync.WaitGroup
|
|
}
|
|
|
|
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
|
sessions, err := s.ActiveSessionOwners(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, owner := range sessions {
|
|
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
|
|
var err error
|
|
if owner.Backend == "pi" {
|
|
err = stage.InterruptPi(stop, owner.Reference)
|
|
} else {
|
|
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID)
|
|
}
|
|
cancel()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
|
|
}
|
|
}
|
|
if err := s.RecoverRuns(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
return &runner{ctx: ctx, store: s, active: map[int64]activeRun{}}, nil
|
|
}
|
|
|
|
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
|
|
if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 {
|
|
return nil, fmt.Errorf("an explicit version and request key are required")
|
|
}
|
|
workspace, err := os.Getwd()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
workspace, err = filepath.EvalSymlinks(workspace)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if request.Workspace != "" {
|
|
if !filepath.IsAbs(request.Workspace) {
|
|
return nil, fmt.Errorf("run workspace must be an absolute directory")
|
|
}
|
|
selected, err := filepath.EvalSymlinks(request.Workspace)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("run workspace is unavailable on the execution server: %q: %w; for Docker, mount this host directory at the same absolute path and use its owner's UID/GID", request.Workspace, err)
|
|
}
|
|
info, err := os.Stat(selected)
|
|
if err != nil || !info.IsDir() {
|
|
return nil, fmt.Errorf("run workspace %q must be an accessible directory on the execution server", request.Workspace)
|
|
}
|
|
workspace = selected
|
|
}
|
|
if len(request.WritePaths) > 20 {
|
|
return nil, fmt.Errorf("too many write paths")
|
|
}
|
|
for _, path := range request.WritePaths {
|
|
if filepath.IsAbs(path) || filepath.Clean(path) != path || path == ".." || strings.HasPrefix(path, ".."+string(os.PathSeparator)) || strings.ContainsAny(path, "*?\\") {
|
|
return nil, fmt.Errorf("optional write paths must be exact run-directory-relative paths")
|
|
}
|
|
parent := filepath.Join(workspace, path)
|
|
for {
|
|
_, err := os.Lstat(parent)
|
|
if err == nil {
|
|
break
|
|
}
|
|
if !os.IsNotExist(err) {
|
|
return nil, err
|
|
}
|
|
next := filepath.Dir(parent)
|
|
if next == parent {
|
|
return nil, fmt.Errorf("write path is unavailable")
|
|
}
|
|
parent = next
|
|
}
|
|
resolved, err := filepath.EvalSymlinks(parent)
|
|
if err != nil || (resolved != workspace && !strings.HasPrefix(resolved, workspace+string(os.PathSeparator))) {
|
|
return nil, fmt.Errorf("write path must stay within the run directory")
|
|
}
|
|
}
|
|
request.Input = maps.Clone(request.Input)
|
|
if request.Input == nil {
|
|
request.Input = map[string]any{}
|
|
}
|
|
request.Input["pwd"] = workspace
|
|
if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" {
|
|
return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt")
|
|
}
|
|
d, err := r.store.Get(r.ctx, request.PipelineID, request.Version)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil {
|
|
return nil, err
|
|
}
|
|
for _, s := range d.Graph.Stages {
|
|
if stage.EffectiveType(s.Type) != stage.ModelType {
|
|
continue
|
|
}
|
|
if s.Configs["harness"] == "pi" && r.execute == nil {
|
|
configured, err := harness.Load()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, ok := configured.Profiles["pi"]; !ok {
|
|
return nil, fmt.Errorf("Pi is not configured on this execution server; run pipelines harness setup pi there")
|
|
}
|
|
}
|
|
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !explicit {
|
|
request.AllowTools = true
|
|
}
|
|
for _, tool := range tools {
|
|
if tool == "edit" || tool == "shell" {
|
|
request.AllowTools = true
|
|
}
|
|
}
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if r.ctx.Err() != nil {
|
|
return nil, r.ctx.Err()
|
|
}
|
|
sessions, err := r.store.ActiveAgentSessions(r.ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(sessions) > 0 && len(r.active) == 0 {
|
|
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
|
|
}
|
|
// Idempotent repeats are accepted even while the original run is active.
|
|
if len(r.active) > 0 {
|
|
existing, err := r.store.RunByKey(r.ctx, request.RequestKey)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if existing == 0 {
|
|
// The workspace is the only scope a run reserves: refuse a second
|
|
// run only when an active run is already changing this exact
|
|
// resolved directory. One active run per workspace keeps this scan
|
|
// finding at most one match, so the message is deterministic.
|
|
for id, entry := range r.active {
|
|
if entry.workspace == workspace {
|
|
return nil, fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
id, created, err := r.store.CreateRun(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, retryOf...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if created {
|
|
ctx, cancel := context.WithCancel(r.ctx)
|
|
r.active[id] = activeRun{cancel: cancel, workspace: workspace}
|
|
r.wg.Add(1)
|
|
go func() {
|
|
defer r.wg.Done()
|
|
defer cancel()
|
|
defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }()
|
|
r.run(ctx, id, d, workspace, request)
|
|
}()
|
|
}
|
|
detail, err := r.store.GetRun(r.ctx, id)
|
|
if detail != nil {
|
|
_, detail.Executing = r.active[id]
|
|
}
|
|
return detail, err
|
|
}
|
|
|
|
func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
d, err := r.store.GetRun(r.ctx, id)
|
|
if d != nil {
|
|
_, d.Executing = r.active[id]
|
|
for _, s := range d.Steps {
|
|
if s.AgentActive {
|
|
d.Executing = true
|
|
}
|
|
}
|
|
}
|
|
return d, err
|
|
}
|
|
|
|
func (r *runner) listRuns() ([]protocol.RunSummary, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
items, err := r.store.ListRuns(r.ctx)
|
|
for i := range items {
|
|
_, items[i].Executing = r.active[items[i].ID]
|
|
if items[i].Status == "failed" && !items[i].Executing {
|
|
d, err := r.store.GetRun(r.ctx, items[i].ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, step := range d.Steps {
|
|
if step.AgentActive {
|
|
items[i].Executing = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return items, err
|
|
}
|
|
|
|
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
|
|
if request.RunID <= 0 {
|
|
return nil, fmt.Errorf("retry requires a positive run ID")
|
|
}
|
|
original, err := r.getRun(request.RunID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if original.Executing {
|
|
return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID)
|
|
}
|
|
if original.Status != "failed" || original.FinishedAt == "" {
|
|
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
|
|
}
|
|
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths}, original.ID)
|
|
}
|
|
|
|
func (r *runner) cancel(id int64) error {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
entry, ok := r.active[id]
|
|
if !ok {
|
|
return fmt.Errorf("run is not active")
|
|
}
|
|
entry.cancel()
|
|
return nil
|
|
}
|
|
|
|
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
|
|
input := stage.Values(request.Input)
|
|
key := d.Graph.Stages[0].Key
|
|
visits := map[string]int{}
|
|
budgets := map[string]*stage.Budget{}
|
|
finish := func(status, outcome, diagnostic string) {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
|
|
log.Printf("finish run %d: %v", id, err)
|
|
}
|
|
}
|
|
original, copyErr := copyValues(stage.Values(request.Input))
|
|
if copyErr != nil {
|
|
finish("failed", "error", copyErr.Error())
|
|
return
|
|
}
|
|
memory := map[string]memoryValue{"input": {output: original}}
|
|
defer func() {
|
|
if recovered := recover(); recovered != nil {
|
|
finish("failed", "error", "stage execution panicked")
|
|
}
|
|
}()
|
|
for sequence := 1; sequence <= 30; sequence++ {
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
visits[key]++
|
|
if visits[key] > 5 {
|
|
finish("failed", "error", "stage visit budget exceeded")
|
|
return
|
|
}
|
|
var def stage.Definition
|
|
var selected protocol.Stage
|
|
found := false
|
|
for _, s := range d.Graph.Stages {
|
|
if s.Key == key {
|
|
selected = s
|
|
def = pipeline.StageDefinition(s)
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
finish("failed", "error", "missing stage")
|
|
return
|
|
}
|
|
maxVisits := 5
|
|
if selected.Limits != nil {
|
|
maxVisits = selected.Limits.MaxVisits
|
|
}
|
|
if visits[key] > maxVisits {
|
|
finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits))
|
|
return
|
|
}
|
|
var reads map[string]protocol.MemoryRead
|
|
input, reads, copyErr = memoryInput(input, selected.Memory, memory)
|
|
if copyErr != nil {
|
|
finish("failed", "error", copyErr.Error())
|
|
return
|
|
}
|
|
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
var tools []string
|
|
explicitTools := false
|
|
var toolErr error
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
tools, explicitTools, toolErr = stage.ToolNames(def)
|
|
}
|
|
if toolErr != nil {
|
|
finish("failed", "error", toolErr.Error())
|
|
return
|
|
}
|
|
useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools)
|
|
operations := &stage.OperationLog{}
|
|
client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations}, WritePaths: request.WritePaths, Tools: tools}
|
|
client.OnSession = func(session string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference)
|
|
}
|
|
var result stage.Result
|
|
var piClient *stage.PiClient
|
|
var err error
|
|
stageCtx := ctx
|
|
var modelConfig stage.ModelConfig
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
spec, specErr := (stage.ModelStage{}).Spec(def)
|
|
if specErr != nil {
|
|
finish("failed", "error", specErr.Error())
|
|
return
|
|
}
|
|
modelConfig = spec.Config
|
|
if budgets[key] == nil {
|
|
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
|
|
}
|
|
stageCtx = stage.WithBudget(ctx, budgets[key])
|
|
}
|
|
snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second)
|
|
before := filediff.Capture(snapshotCtx, workspace)
|
|
snapshotCancel()
|
|
var stageSession string
|
|
panicked := false
|
|
func() {
|
|
defer func() {
|
|
if recover() != nil {
|
|
panicked = true
|
|
err = fmt.Errorf("stage execution panicked")
|
|
}
|
|
}()
|
|
if r.execute != nil {
|
|
timeout, _ := stage.StageTimeout(def)
|
|
executeCtx, cancel := context.WithTimeout(stageCtx, timeout)
|
|
defer cancel()
|
|
result, err = r.execute(executeCtx, def, input)
|
|
} else if modelConfig.Harness == "pi" {
|
|
configured, configErr := harness.Load()
|
|
if configErr != nil {
|
|
err = configErr
|
|
return
|
|
}
|
|
profile, ok := configured.Profiles["pi"]
|
|
if !ok {
|
|
err = fmt.Errorf("Pi is not configured on execution server")
|
|
return
|
|
}
|
|
piClient = &stage.PiClient{Executable: profile.Executable, Directory: workspace, Tools: tools, WritePaths: request.WritePaths}
|
|
piClient.OnOwnership = func(session, reference string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return r.store.RunSession(c, id, sequence, session, active, "pi", reference)
|
|
}
|
|
result, err = (stage.ModelStage{Client: piClient}).Run(stageCtx, def, input)
|
|
stageSession = piClient.SessionID
|
|
client.Transcript = piClient.Transcript
|
|
} else if useAgent {
|
|
result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input)
|
|
stageSession = client.SessionID
|
|
} else if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations}
|
|
plain.OnGenerationSession = func(session, backend, reference string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if active {
|
|
stageSession = session
|
|
}
|
|
return r.store.RunSession(c, id, sequence, session, active, backend, reference)
|
|
}
|
|
result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input)
|
|
client.Transcript = plain.FormatAudit
|
|
} else {
|
|
result, err = stage.DefaultRegistry().Run(ctx, def, input)
|
|
}
|
|
}()
|
|
// Capture after cleanup, even when the stage's context was canceled.
|
|
snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
|
after := filediff.Capture(snapshotCtx, workspace, before)
|
|
snapshotCancel()
|
|
changes := filediff.Compare(before, after)
|
|
if errors.Is(err, stage.ErrStageCleanup) {
|
|
changes.Status = "partial"
|
|
changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing."
|
|
}
|
|
step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes}
|
|
if budget := budgets[key]; budget != nil {
|
|
step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries)
|
|
}
|
|
if err != nil {
|
|
step.Status = "failed"
|
|
step.Diagnostic = err.Error()
|
|
step.Outcome = "error"
|
|
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
|
}
|
|
step.Target = def.Edges[step.Outcome]
|
|
step.Control = &protocol.ControlEvidence{Visit: visits[key], MaxVisits: maxVisits, Memory: reads}
|
|
if err == nil && selected.Limits != nil && visits[key] >= maxVisits {
|
|
for _, outcome := range selected.Limits.On {
|
|
if outcome == step.Outcome {
|
|
step.Control.LimitReached = true
|
|
step.Control.ModelTarget = step.Target
|
|
step.Target = protocol.FailureEnd
|
|
step.Diagnostic = fmt.Sprintf("Pipelines stopped %s after %d visits on outcome %s", key, maxVisits, step.Outcome)
|
|
}
|
|
}
|
|
}
|
|
if panicked {
|
|
step.Target = protocol.FailureEnd
|
|
}
|
|
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
persistErr := r.store.FinishStep(save, id, step)
|
|
cancel()
|
|
if persistErr != nil {
|
|
finish("failed", "error", persistErr.Error())
|
|
return
|
|
}
|
|
if step.Status == "completed" {
|
|
values, err := copyValues(stage.Values(step.Output))
|
|
if err != nil {
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
memory[key] = memoryValue{output: values, sequence: sequence}
|
|
}
|
|
for _, s := range d.Graph.Stages {
|
|
if s.Key == key {
|
|
r.notifyStep(id, d, workspace, s.Notify, step)
|
|
break
|
|
}
|
|
}
|
|
input = stage.Values(step.Output)
|
|
if useAgent || errors.Is(err, stage.ErrStageCleanup) {
|
|
sessions, checkErr := r.store.ActiveAgentSessions(context.Background())
|
|
if checkErr != nil || len(sessions) > 0 {
|
|
finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work")
|
|
return
|
|
}
|
|
}
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
if step.Target == protocol.SuccessEnd {
|
|
finish("succeeded", "success", "")
|
|
return
|
|
}
|
|
if step.Target == protocol.FailureEnd {
|
|
finish("failed", "error", step.Diagnostic)
|
|
return
|
|
}
|
|
if step.Target == "" {
|
|
finish("failed", "error", "stage returned an undeclared outcome")
|
|
return
|
|
}
|
|
key = step.Target
|
|
}
|
|
finish("failed", "error", "total stage budget exceeded")
|
|
}
|