package server import ( "context" "errors" "fmt" "log" "maps" "os" "path/filepath" "strconv" "strings" "sync" "time" "git.a-wan.net/alan/pipelines/internal/filediff" "git.a-wan.net/alan/pipelines/internal/harness" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) // activeRun records one in-flight run's cancellation seam and the resolved // workspace it is changing. A single map is the only active-run state, so the // workspace cannot drift from the cancel function. type activeRun struct { cancel context.CancelFunc workspace string } type runner struct { ctx context.Context store *pipeline.Store artifactsRoot string mu sync.Mutex active map[int64]activeRun execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error) wg sync.WaitGroup schedulerWG sync.WaitGroup } func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { sessions, err := s.ActiveSessionOwners(ctx) if err != nil { return nil, err } for _, owner := range sessions { stop, cancel := context.WithTimeout(ctx, 20*time.Second) var err error if owner.Backend == "pi" { err = stage.InterruptPi(stop, owner.Reference) } else { err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID) } cancel() if err != nil { return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err) } } if err := s.RecoverRuns(ctx); err != nil { return nil, err } root := filepath.Join(s.Directory(), "artifacts") return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}}, nil } func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) { if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 { return nil, fmt.Errorf("an explicit version and request key are required") } workspace, err := os.Getwd() if err != nil { return nil, err } workspace, err = filepath.EvalSymlinks(workspace) if err != nil { return nil, err } if request.Workspace != "" { if !filepath.IsAbs(request.Workspace) { return nil, fmt.Errorf("run workspace must be an absolute directory") } selected, err := filepath.EvalSymlinks(request.Workspace) if err != nil { return nil, fmt.Errorf("run workspace is unavailable on the execution server: %q: %w; for Docker, mount this host directory at the same absolute path and use its owner's UID/GID", request.Workspace, err) } info, err := os.Stat(selected) if err != nil || !info.IsDir() { return nil, fmt.Errorf("run workspace %q must be an accessible directory on the execution server", request.Workspace) } workspace = selected } if len(request.WritePaths) > 20 { return nil, fmt.Errorf("too many write paths") } for _, path := range request.WritePaths { if filepath.IsAbs(path) || filepath.Clean(path) != path || path == ".." || strings.HasPrefix(path, ".."+string(os.PathSeparator)) || strings.ContainsAny(path, "*?\\") { return nil, fmt.Errorf("optional write paths must be exact run-directory-relative paths") } parent := filepath.Join(workspace, path) for { _, err := os.Lstat(parent) if err == nil { break } if !os.IsNotExist(err) { return nil, err } next := filepath.Dir(parent) if next == parent { return nil, fmt.Errorf("write path is unavailable") } parent = next } resolved, err := filepath.EvalSymlinks(parent) if err != nil || (resolved != workspace && !strings.HasPrefix(resolved, workspace+string(os.PathSeparator))) { return nil, fmt.Errorf("write path must stay within the run directory") } } request.Input = maps.Clone(request.Input) if request.Input == nil { request.Input = map[string]any{} } request.Input["pwd"] = workspace if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" { return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt") } d, err := r.store.Get(r.ctx, request.PipelineID, request.Version) if err != nil { return nil, err } if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil { return nil, err } for _, s := range d.Graph.Stages { if stage.EffectiveType(s.Type) != stage.ModelType { continue } if s.Configs["harness"] == "pi" && r.execute == nil { configured, err := harness.Load() if err != nil { return nil, err } if _, ok := configured.Profiles["pi"]; !ok { return nil, fmt.Errorf("Pi is not configured on this execution server; run pipelines harness setup pi there") } } tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s)) if err != nil { return nil, err } if !explicit { request.AllowTools = true } for _, tool := range tools { if tool == "edit" || tool == "shell" { request.AllowTools = true } } } r.mu.Lock() defer r.mu.Unlock() if r.ctx.Err() != nil { return nil, r.ctx.Err() } owners, err := r.store.ActiveSessionOwners(r.ctx) if err != nil { return nil, err } // Refuse only an orphaned session: one whose owning run is not active now. // A session owned by a currently active run (another run in another // workspace) must not block new work. for _, owner := range owners { if _, ok := r.active[owner.RunID]; !ok { return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work") } } // Idempotent repeats are accepted even while the original run is active. if len(r.active) > 0 { existing, err := r.store.RunByKey(r.ctx, request.RequestKey) if err != nil { return nil, err } if existing == 0 { // The workspace is the only scope a run reserves: refuse a second // run only when an active run is already changing this exact // resolved directory. One active run per workspace keeps this scan // finding at most one match, so the message is deterministic. for id, entry := range r.active { if entry.workspace == workspace { return nil, fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace) } } } } id, created, err := r.store.CreateRun(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, retryOf...) if err != nil { return nil, err } if created { ctx, cancel := context.WithCancel(r.ctx) r.active[id] = activeRun{cancel: cancel, workspace: workspace} r.wg.Add(1) go func() { defer r.wg.Done() defer cancel() defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }() r.run(ctx, id, d, workspace, request) }() } detail, err := r.store.GetRun(r.ctx, id) if detail != nil { _, detail.Executing = r.active[id] } return detail, err } func (r *runner) getRun(id int64) (*protocol.RunDetail, error) { r.mu.Lock() defer r.mu.Unlock() d, err := r.store.GetRun(r.ctx, id) if d != nil { _, d.Executing = r.active[id] for _, s := range d.Steps { if s.AgentActive { d.Executing = true } } } return d, err } func (r *runner) listRuns() ([]protocol.RunSummary, error) { r.mu.Lock() defer r.mu.Unlock() items, err := r.store.ListRuns(r.ctx) for i := range items { _, items[i].Executing = r.active[items[i].ID] if items[i].Status == "failed" && !items[i].Executing { d, err := r.store.GetRun(r.ctx, items[i].ID) if err != nil { return nil, err } for _, step := range d.Steps { if step.AgentActive { items[i].Executing = true } } } } return items, err } func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) { if request.RunID <= 0 { return nil, fmt.Errorf("retry requires a positive run ID") } original, err := r.getRun(request.RunID) if err != nil { return nil, err } if original.Executing { return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID) } if original.Status != "failed" || original.FinishedAt == "" { return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status) } return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths}, original.ID) } func (r *runner) cancel(id int64) error { r.mu.Lock() defer r.mu.Unlock() entry, ok := r.active[id] if !ok { return fmt.Errorf("run is not active") } entry.cancel() return nil } func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) { input := stage.Values(request.Input) key := d.Graph.Stages[0].Key visits := map[string]int{} budgets := map[string]*stage.Budget{} finish := func(status, outcome, diagnostic string) { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil { log.Printf("finish run %d: %v", id, err) } } original, copyErr := copyValues(stage.Values(request.Input)) if copyErr != nil { finish("failed", "error", copyErr.Error()) return } memory := map[string]memoryValue{"input": {output: original}} defer func() { if recovered := recover(); recovered != nil { finish("failed", "error", "stage execution panicked") } }() for sequence := 1; sequence <= 30; sequence++ { if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return } visits[key]++ if visits[key] > 5 { finish("failed", "error", "stage visit budget exceeded") return } var def stage.Definition var selected protocol.Stage found := false for _, s := range d.Graph.Stages { if s.Key == key { selected = s def = pipeline.StageDefinition(s) found = true break } } if !found { finish("failed", "error", "missing stage") return } maxVisits := 5 if selected.Limits != nil { maxVisits = selected.Limits.MaxVisits } if visits[key] > maxVisits { finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits)) return } var reads map[string]protocol.MemoryRead input, reads, copyErr = memoryInput(input, selected.Memory, memory) if copyErr != nil { finish("failed", "error", copyErr.Error()) return } if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil { if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return } finish("failed", "error", err.Error()) return } var tools []string explicitTools := false var toolErr error if stage.EffectiveType(def.Type) == stage.ModelType { tools, explicitTools, toolErr = stage.ToolNames(def) } if toolErr != nil { finish("failed", "error", toolErr.Error()) return } useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools) // Every agent visit gets a per-run/step directory for the files it // produces. Creation is best-effort: an unusable directory must never // fail the run, and it is only attempted for model stages. artifactsDir := "" if stage.EffectiveType(def.Type) == stage.ModelType { candidate := filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10), strconv.Itoa(sequence)) if mkErr := os.MkdirAll(candidate, 0700); mkErr != nil { log.Printf("stage artifacts directory run=%d step=%d: %v", id, sequence, mkErr) } else { artifactsDir = candidate def.ArtifactsDir = candidate } } operations := &stage.OperationLog{} client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: request.WritePaths, Tools: tools} client.OnSession = func(session string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference) } var result stage.Result var piClient *stage.PiClient var err error stageCtx := ctx var modelConfig stage.ModelConfig if stage.EffectiveType(def.Type) == stage.ModelType { spec, specErr := (stage.ModelStage{}).Spec(def) if specErr != nil { finish("failed", "error", specErr.Error()) return } modelConfig = spec.Config if budgets[key] == nil { budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens) } stageCtx = stage.WithBudget(ctx, budgets[key]) } snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second) before := filediff.Capture(snapshotCtx, workspace) snapshotCancel() var stageSession string panicked := false func() { defer func() { if recover() != nil { panicked = true err = fmt.Errorf("stage execution panicked") } }() if r.execute != nil { timeout, _ := stage.StageTimeout(def) executeCtx, cancel := context.WithTimeout(stageCtx, timeout) defer cancel() result, err = r.execute(executeCtx, def, input) } else if modelConfig.Harness == "pi" { configured, configErr := harness.Load() if configErr != nil { err = configErr return } profile, ok := configured.Profiles["pi"] if !ok { err = fmt.Errorf("Pi is not configured on execution server") return } piClient = &stage.PiClient{Executable: profile.Executable, Directory: workspace, Tools: tools, WritePaths: request.WritePaths, ArtifactsDir: artifactsDir} piClient.OnOwnership = func(session, reference string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return r.store.RunSession(c, id, sequence, session, active, "pi", reference) } result, err = (stage.ModelStage{Client: piClient}).Run(stageCtx, def, input) stageSession = piClient.SessionID client.Transcript = piClient.Transcript } else if useAgent { result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input) stageSession = client.SessionID } else if stage.EffectiveType(def.Type) == stage.ModelType { plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir} plain.OnGenerationSession = func(session, backend, reference string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if active { stageSession = session } return r.store.RunSession(c, id, sequence, session, active, backend, reference) } result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input) client.Transcript = plain.FormatAudit } else { result, err = stage.DefaultRegistry().Run(ctx, def, input) } }() // Capture after cleanup, even when the stage's context was canceled. snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) after := filediff.Capture(snapshotCtx, workspace, before) snapshotCancel() changes := filediff.Compare(before, after) if errors.Is(err, stage.ErrStageCleanup) { changes.Status = "partial" changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing." } artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir) step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes, Artifacts: artifacts, ArtifactsSkipped: artifactsSkipped, ArtifactsTruncated: artifactsTruncated} if budget := budgets[key]; budget != nil { step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries) } if err != nil { step.Status = "failed" step.Diagnostic = err.Error() step.Outcome = "error" step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} } // A failed or timed-out step carries the session pointer even when the // bounded transcript would otherwise be only its stage_limits envelope. if err != nil && stageSession != "" { step.Transcript = stage.RetainSession(step.Transcript, stageSession) } step.Target = def.Edges[step.Outcome] step.Control = &protocol.ControlEvidence{Visit: visits[key], MaxVisits: maxVisits, Memory: reads} if err == nil && selected.Limits != nil && visits[key] >= maxVisits { for _, outcome := range selected.Limits.On { if outcome == step.Outcome { step.Control.LimitReached = true step.Control.ModelTarget = step.Target step.Target = protocol.FailureEnd step.Diagnostic = fmt.Sprintf("Pipelines stopped %s after %d visits on outcome %s", key, maxVisits, step.Outcome) } } } if panicked { step.Target = protocol.FailureEnd } save, cancel := context.WithTimeout(context.Background(), 10*time.Second) persistErr := r.store.FinishStep(save, id, step) cancel() if persistErr != nil { finish("failed", "error", persistErr.Error()) return } if step.Status == "completed" { values, err := copyValues(stage.Values(step.Output)) if err != nil { finish("failed", "error", err.Error()) return } memory[key] = memoryValue{output: values, sequence: sequence} } for _, s := range d.Graph.Stages { if s.Key == key { r.notifyStep(id, d, workspace, s.Notify, step) break } } input = stage.Values(step.Output) if useAgent || errors.Is(err, stage.ErrStageCleanup) { sessions, checkErr := r.store.ActiveAgentSessionsForRun(context.Background(), id) if checkErr != nil || len(sessions) > 0 { finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work") return } } if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return } if step.Target == protocol.SuccessEnd { finish("succeeded", "success", "") return } if step.Target == protocol.FailureEnd { diagnostic := step.Diagnostic if diagnostic == "" { diagnostic = fmt.Sprintf("stage %s returned outcome %q", step.Stage, step.Outcome) } finish("failed", "error", diagnostic) return } if step.Target == "" { finish("failed", "error", "stage returned an undeclared outcome") return } key = step.Target } finish("failed", "error", "total stage budget exceeded") }