306 lines
9.2 KiB
Go
306 lines
9.2 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"pipelines/internal/pipeline"
|
|
"pipelines/internal/protocol"
|
|
"pipelines/internal/stage"
|
|
)
|
|
|
|
type runner struct {
|
|
ctx context.Context
|
|
store *pipeline.Store
|
|
mu sync.Mutex
|
|
active map[int64]context.CancelFunc
|
|
execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error)
|
|
wg sync.WaitGroup
|
|
schedulerWG sync.WaitGroup
|
|
workspaceRoots []string
|
|
}
|
|
|
|
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
|
sessions, err := s.ActiveAgentSessions(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, id := range sessions {
|
|
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
|
|
err := (stage.OpenCodeClient{}).Interrupt(stop, id)
|
|
cancel()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", id, err)
|
|
}
|
|
}
|
|
if err := s.RecoverRuns(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
return &runner{ctx: ctx, store: s, active: map[int64]context.CancelFunc{}}, nil
|
|
}
|
|
|
|
func (r *runner) start(request protocol.Message) (*protocol.RunDetail, error) {
|
|
if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 {
|
|
return nil, fmt.Errorf("an explicit version and request key are required")
|
|
}
|
|
workspace, err := os.Getwd()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
workspace, err = filepath.EvalSymlinks(workspace)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if request.Workspace != "" {
|
|
if !filepath.IsAbs(request.Workspace) {
|
|
return nil, fmt.Errorf("run workspace must be an absolute directory")
|
|
}
|
|
selected, err := filepath.EvalSymlinks(request.Workspace)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("run workspace is unavailable")
|
|
}
|
|
info, err := os.Stat(selected)
|
|
if err != nil || !info.IsDir() {
|
|
return nil, fmt.Errorf("run workspace must be a directory")
|
|
}
|
|
allowed := selected == workspace
|
|
for _, root := range r.workspaceRoots {
|
|
root, err = filepath.Abs(root)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
root, err = filepath.EvalSymlinks(root)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("configured workspace root is unavailable")
|
|
}
|
|
rel, err := filepath.Rel(root, selected)
|
|
if err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
|
allowed = true
|
|
}
|
|
}
|
|
if !allowed {
|
|
return nil, fmt.Errorf("run workspace is outside the server's allowed workspace roots")
|
|
}
|
|
workspace = selected
|
|
}
|
|
if len(request.WritePaths) > 20 {
|
|
return nil, fmt.Errorf("too many write paths")
|
|
}
|
|
for _, path := range request.WritePaths {
|
|
if filepath.IsAbs(path) || filepath.Clean(path) != path || strings.HasPrefix(path, "..") || strings.ContainsAny(path, "*?\\") || !strings.HasSuffix(path, ".go") {
|
|
return nil, fmt.Errorf("write paths must be exact project-local Go files")
|
|
}
|
|
resolved, err := filepath.EvalSymlinks(filepath.Join(workspace, path))
|
|
if err != nil || !strings.HasPrefix(resolved, workspace+string(os.PathSeparator)) {
|
|
return nil, fmt.Errorf("write path must be an existing file within workspace")
|
|
}
|
|
}
|
|
if request.Input == nil {
|
|
request.Input = map[string]any{}
|
|
}
|
|
if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" {
|
|
return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt")
|
|
}
|
|
d, err := r.store.Get(r.ctx, request.PipelineID, request.Version)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil {
|
|
return nil, err
|
|
}
|
|
for _, s := range d.Graph.Stages {
|
|
if stage.EffectiveType(s.Type) != stage.ModelType {
|
|
continue
|
|
}
|
|
tools, _, err := stage.ToolNames(pipeline.StageDefinition(s))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, tool := range tools {
|
|
if (tool == "edit" || tool == "shell") && !request.AllowTools {
|
|
return nil, fmt.Errorf("stage %s: %s requires explicit manual tool approval", s.Key, tool)
|
|
}
|
|
}
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if r.ctx.Err() != nil {
|
|
return nil, r.ctx.Err()
|
|
}
|
|
sessions, err := r.store.ActiveAgentSessions(r.ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(sessions) > 0 && len(r.active) == 0 {
|
|
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
|
|
}
|
|
// Idempotent repeats are accepted even while the original run is active.
|
|
if len(r.active) > 0 {
|
|
existing, err := r.store.RunByKey(r.ctx, request.RequestKey)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if existing == 0 {
|
|
return nil, fmt.Errorf("another run is active; inspect it before starting more work")
|
|
}
|
|
}
|
|
id, created, err := r.store.CreateRun(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if created {
|
|
ctx, cancel := context.WithTimeout(r.ctx, 30*time.Minute)
|
|
r.active[id] = cancel
|
|
r.wg.Add(1)
|
|
go func() {
|
|
defer r.wg.Done()
|
|
defer cancel()
|
|
defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }()
|
|
r.run(ctx, id, d, workspace, request)
|
|
}()
|
|
}
|
|
return r.store.GetRun(r.ctx, id)
|
|
}
|
|
|
|
func (r *runner) cancel(id int64) error {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
cancel, ok := r.active[id]
|
|
if !ok {
|
|
return fmt.Errorf("run is not active")
|
|
}
|
|
cancel()
|
|
return nil
|
|
}
|
|
|
|
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
|
|
input := stage.Values(request.Input)
|
|
key := d.Graph.Stages[0].Key
|
|
visits := map[string]int{}
|
|
finish := func(status, outcome, diagnostic string) {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
|
|
log.Printf("finish run %d: %v", id, err)
|
|
}
|
|
}
|
|
defer func() {
|
|
if recovered := recover(); recovered != nil {
|
|
finish("failed", "error", "stage execution panicked")
|
|
}
|
|
}()
|
|
for sequence := 1; sequence <= 30; sequence++ {
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
visits[key]++
|
|
if visits[key] > 5 {
|
|
finish("failed", "error", "stage visit budget exceeded")
|
|
return
|
|
}
|
|
var def stage.Definition
|
|
found := false
|
|
for _, s := range d.Graph.Stages {
|
|
if s.Key == key {
|
|
def = pipeline.StageDefinition(s)
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
finish("failed", "error", "missing stage")
|
|
return
|
|
}
|
|
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
var tools []string
|
|
explicitTools := false
|
|
var toolErr error
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
tools, explicitTools, toolErr = stage.ToolNames(def)
|
|
}
|
|
if toolErr != nil {
|
|
finish("failed", "error", toolErr.Error())
|
|
return
|
|
}
|
|
useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || (request.AllowTools && !explicitTools))
|
|
client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace}, WritePaths: request.WritePaths, Tools: tools}
|
|
client.OnSession = func(session string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return r.store.RunSession(c, id, sequence, session, active)
|
|
}
|
|
var result stage.Result
|
|
var err error
|
|
if r.execute != nil {
|
|
result, err = r.execute(ctx, def, input)
|
|
} else if useAgent {
|
|
result, err = (stage.ModelStage{Client: client}).Run(ctx, def, input)
|
|
} else if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
result, err = (stage.ModelStage{Client: stage.OpenCodeClient{Directory: workspace}}).Run(ctx, def, input)
|
|
} else {
|
|
result, err = stage.DefaultRegistry().Run(ctx, def, input)
|
|
}
|
|
step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: client.Transcript}
|
|
if err != nil {
|
|
step.Status = "failed"
|
|
step.Diagnostic = err.Error()
|
|
step.Outcome = "error"
|
|
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
|
}
|
|
step.Target = def.Edges[step.Outcome]
|
|
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
persistErr := r.store.FinishStep(save, id, step)
|
|
cancel()
|
|
if persistErr != nil {
|
|
finish("failed", "error", persistErr.Error())
|
|
return
|
|
}
|
|
for _, s := range d.Graph.Stages {
|
|
if s.Key == key {
|
|
r.notifyStep(id, d, workspace, s.Notify, step)
|
|
break
|
|
}
|
|
}
|
|
input = stage.Values(step.Output)
|
|
if useAgent {
|
|
sessions, checkErr := r.store.ActiveAgentSessions(context.Background())
|
|
if checkErr != nil || len(sessions) > 0 {
|
|
finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work")
|
|
return
|
|
}
|
|
}
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
if step.Target == protocol.SuccessEnd {
|
|
finish("succeeded", "success", "")
|
|
return
|
|
}
|
|
if step.Target == protocol.FailureEnd {
|
|
finish("failed", "error", step.Diagnostic)
|
|
return
|
|
}
|
|
if step.Target == "" {
|
|
finish("failed", "error", "stage returned an undeclared outcome")
|
|
return
|
|
}
|
|
key = step.Target
|
|
}
|
|
finish("failed", "error", "total stage budget exceeded")
|
|
}
|