package server import ( "context" "fmt" "log" "maps" "os" "path/filepath" "strings" "sync" "time" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) type runner struct { ctx context.Context store *pipeline.Store mu sync.Mutex active map[int64]context.CancelFunc execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error) wg sync.WaitGroup schedulerWG sync.WaitGroup } func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { sessions, err := s.ActiveSessionOwners(ctx) if err != nil { return nil, err } for _, owner := range sessions { stop, cancel := context.WithTimeout(ctx, 20*time.Second) err := (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID) cancel() if err != nil { return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err) } } if err := s.RecoverRuns(ctx); err != nil { return nil, err } return &runner{ctx: ctx, store: s, active: map[int64]context.CancelFunc{}}, nil } func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) { if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 { return nil, fmt.Errorf("an explicit version and request key are required") } workspace, err := os.Getwd() if err != nil { return nil, err } workspace, err = filepath.EvalSymlinks(workspace) if err != nil { return nil, err } if request.Workspace != "" { if !filepath.IsAbs(request.Workspace) { return nil, fmt.Errorf("run workspace must be an absolute directory") } selected, err := filepath.EvalSymlinks(request.Workspace) if err != nil { return nil, fmt.Errorf("run workspace is unavailable") } info, err := os.Stat(selected) if err != nil || !info.IsDir() { return nil, fmt.Errorf("run workspace must be a directory") } workspace = selected } if len(request.WritePaths) > 20 { return nil, fmt.Errorf("too many write paths") } for _, path := range request.WritePaths { if filepath.IsAbs(path) || filepath.Clean(path) != path || path == ".." || strings.HasPrefix(path, ".."+string(os.PathSeparator)) || strings.ContainsAny(path, "*?\\") { return nil, fmt.Errorf("optional write paths must be exact run-directory-relative paths") } parent := filepath.Join(workspace, path) for { _, err := os.Lstat(parent) if err == nil { break } if !os.IsNotExist(err) { return nil, err } next := filepath.Dir(parent) if next == parent { return nil, fmt.Errorf("write path is unavailable") } parent = next } resolved, err := filepath.EvalSymlinks(parent) if err != nil || (resolved != workspace && !strings.HasPrefix(resolved, workspace+string(os.PathSeparator))) { return nil, fmt.Errorf("write path must stay within the run directory") } } request.Input = maps.Clone(request.Input) if request.Input == nil { request.Input = map[string]any{} } request.Input["pwd"] = workspace if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" { return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt") } d, err := r.store.Get(r.ctx, request.PipelineID, request.Version) if err != nil { return nil, err } if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil { return nil, err } for _, s := range d.Graph.Stages { if stage.EffectiveType(s.Type) != stage.ModelType { continue } tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s)) if err != nil { return nil, err } if !explicit { request.AllowTools = true } for _, tool := range tools { if tool == "edit" || tool == "shell" { request.AllowTools = true } } } r.mu.Lock() defer r.mu.Unlock() if r.ctx.Err() != nil { return nil, r.ctx.Err() } sessions, err := r.store.ActiveAgentSessions(r.ctx) if err != nil { return nil, err } if len(sessions) > 0 && len(r.active) == 0 { return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work") } // Idempotent repeats are accepted even while the original run is active. if len(r.active) > 0 { existing, err := r.store.RunByKey(r.ctx, request.RequestKey) if err != nil { return nil, err } if existing == 0 { return nil, fmt.Errorf("another run is active; inspect it before starting more work") } } id, created, err := r.store.CreateRun(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, retryOf...) if err != nil { return nil, err } if created { ctx, cancel := context.WithTimeout(r.ctx, 30*time.Minute) r.active[id] = cancel r.wg.Add(1) go func() { defer r.wg.Done() defer cancel() defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }() r.run(ctx, id, d, workspace, request) }() } detail, err := r.store.GetRun(r.ctx, id) if detail != nil { _, detail.Executing = r.active[id] } return detail, err } func (r *runner) getRun(id int64) (*protocol.RunDetail, error) { r.mu.Lock() defer r.mu.Unlock() d, err := r.store.GetRun(r.ctx, id) if d != nil { _, d.Executing = r.active[id] for _, s := range d.Steps { if s.AgentActive { d.Executing = true } } } return d, err } func (r *runner) listRuns() ([]protocol.RunSummary, error) { r.mu.Lock() defer r.mu.Unlock() items, err := r.store.ListRuns(r.ctx) for i := range items { _, items[i].Executing = r.active[items[i].ID] if items[i].Status == "failed" && !items[i].Executing { d, err := r.store.GetRun(r.ctx, items[i].ID) if err != nil { return nil, err } for _, step := range d.Steps { if step.AgentActive { items[i].Executing = true } } } } return items, err } func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) { if request.RunID <= 0 { return nil, fmt.Errorf("retry requires a positive run ID") } original, err := r.getRun(request.RunID) if err != nil { return nil, err } if original.Executing { return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID) } if original.Status != "failed" || original.FinishedAt == "" { return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status) } return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths}, original.ID) } func (r *runner) cancel(id int64) error { r.mu.Lock() defer r.mu.Unlock() cancel, ok := r.active[id] if !ok { return fmt.Errorf("run is not active") } cancel() return nil } func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) { input := stage.Values(request.Input) key := d.Graph.Stages[0].Key visits := map[string]int{} finish := func(status, outcome, diagnostic string) { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil { log.Printf("finish run %d: %v", id, err) } } defer func() { if recovered := recover(); recovered != nil { finish("failed", "error", "stage execution panicked") } }() for sequence := 1; sequence <= 30; sequence++ { if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return } visits[key]++ if visits[key] > 5 { finish("failed", "error", "stage visit budget exceeded") return } var def stage.Definition found := false for _, s := range d.Graph.Stages { if s.Key == key { def = pipeline.StageDefinition(s) found = true break } } if !found { finish("failed", "error", "missing stage") return } if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil { if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return } finish("failed", "error", err.Error()) return } var tools []string explicitTools := false var toolErr error if stage.EffectiveType(def.Type) == stage.ModelType { tools, explicitTools, toolErr = stage.ToolNames(def) } if toolErr != nil { finish("failed", "error", toolErr.Error()) return } useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools) client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace}, WritePaths: request.WritePaths, Tools: tools} client.OnSession = func(session string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference) } var result stage.Result var err error if r.execute != nil { result, err = r.execute(ctx, def, input) } else if useAgent { result, err = (stage.ModelStage{Client: client}).Run(ctx, def, input) } else if stage.EffectiveType(def.Type) == stage.ModelType { plain := &stage.OpenCodeClient{Directory: workspace} result, err = (stage.ModelStage{Client: plain}).Run(ctx, def, input) client.Transcript = plain.FormatAudit } else { result, err = stage.DefaultRegistry().Run(ctx, def, input) } step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: client.Transcript} if err != nil { step.Status = "failed" step.Diagnostic = err.Error() step.Outcome = "error" step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} } step.Target = def.Edges[step.Outcome] save, cancel := context.WithTimeout(context.Background(), 10*time.Second) persistErr := r.store.FinishStep(save, id, step) cancel() if persistErr != nil { finish("failed", "error", persistErr.Error()) return } for _, s := range d.Graph.Stages { if s.Key == key { r.notifyStep(id, d, workspace, s.Notify, step) break } } input = stage.Values(step.Output) if useAgent { sessions, checkErr := r.store.ActiveAgentSessions(context.Background()) if checkErr != nil || len(sessions) > 0 { finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work") return } } if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return } if step.Target == protocol.SuccessEnd { finish("succeeded", "success", "") return } if step.Target == protocol.FailureEnd { finish("failed", "error", step.Diagnostic) return } if step.Target == "" { finish("failed", "error", "stage returned an undeclared outcome") return } key = step.Target } finish("failed", "error", "total stage budget exceeded") }