package server import ( "context" "errors" "fmt" "log" "time" "git.a-wan.net/alan/pipelines/internal/opencode" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) // Runs survive a server restart. With detach enabled (the installed server), // a stopping server cancels each run's context with stage.ErrServerShutdown: // stage clients leave their admitted harness session running and owned, and the // loop returns without settling anything. The next process plans, before it // serves requests, how to continue every run it finds in flight: // // - a running visit with a live OpenCode V2 session is re-adopted: the loop // re-enters that visit, attaches to the session, waits for it to settle (or // reads the result it reached while the server was down) and continues the // graph as if it had watched it; // - a visit begun but never handed to a harness is marked interrupted and the // same stage starts again (nothing was admitted, so nothing replays); // - a run between visits continues with its next stage; // - a session whose harness process is gone, or that belongs to a harness the // server owned as a child (Pi, OpenCode V1), falls back to the previous // behaviour: the run is interrupted, with the reason in its diagnostic. // // Only an operator cancel (cancel_run) stops a stage session. type reattachKind int const ( reattachAdopt reattachKind = iota reattachRestart reattachBetween ) // reattachPlan is one run start-up recovery kept for reattachRuns. type reattachPlan struct { runID int64 kind reattachKind // step is the running visit (adopt/restart) or the last completed one. step protocol.RunStep owner pipeline.SessionOwner } // adoptedVisit is the visit a runLoop re-adopts instead of starting. type adoptedVisit struct { sequence int owner pipeline.SessionOwner } // stageServiceAlive reports whether a retained stage session's harness service // is still running. It is a seam for tests. var stageServiceAlive = func(ctx context.Context, owner pipeline.SessionOwner) error { return opencode.ReferenceAlive(ctx, owner.Reference) } // interruptRecoveredSession stops a previously active session that start-up // recovery does not re-adopt. It is a seam for tests. var interruptRecoveredSession = func(ctx context.Context, owner pipeline.SessionOwner) error { if owner.Backend == "pi" { return stage.InterruptPi(ctx, owner.Reference) } return (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(ctx, owner.ID) } // runContext derives one run's execution context. With detach, a server // shutdown cancels it with stage.ErrServerShutdown as the cause, while the // returned cancel (an operator cancel or the run's own end) uses the ordinary // context.Canceled, so stage clients can tell the two apart. func (r *runner) runContext() (context.Context, context.CancelFunc) { if !r.detach { return context.WithCancel(r.ctx) } ctx, cancel := context.WithCancelCause(context.WithoutCancel(r.ctx)) stop := context.AfterFunc(r.ctx, func() { cancel(stage.ErrServerShutdown) }) return ctx, func() { stop(); cancel(context.Canceled) } } // planReattach decides, per in-flight run, whether start-up continues it. // Runs it does not plan keep today's recovery (interrupted); reasons carries the // explanation for the ones that could not be reattached. // gone lists runs whose session's service is not running: there is nothing // left to interrupt, so recovery only clears their ownership. func planReattach(ctx context.Context, s *pipeline.Store) ([]reattachPlan, map[int64]string, map[int64]bool, error) { reasons, gone := map[int64]string{}, map[int64]bool{} rows, err := s.ListRuns(ctx) if err != nil { return nil, nil, nil, err } var plans []reattachPlan for _, row := range rows { if row.Status != protocol.RunStatusRunning || row.CancelRequested || row.CleanupUnconfirmed { continue } detail, err := s.GetRun(ctx, row.ID) if err != nil { return nil, nil, nil, err } if len(detail.Steps) == 0 { continue } last := detail.Steps[0] for _, step := range detail.Steps { if step.Sequence > last.Sequence { last = step } } switch last.Status { case protocol.StepStatusRunning: if last.SessionID == "" { plans = append(plans, reattachPlan{runID: row.ID, kind: reattachRestart, step: last}) continue } owner, err := s.RunSessionOwner(ctx, row.ID, last.Sequence) if err != nil { return nil, nil, nil, err } owner.RunID = row.ID if owner.Backend != "v2" { name := "OpenCode V1" if owner.Backend == "pi" { name = "Pi" } reasons[row.ID] = fmt.Sprintf("%s stage sessions belong to the stopped server process, so stage %s could not be reattached", name, last.Stage) continue } probe, cancel := context.WithTimeout(ctx, 5*time.Second) aliveErr := stageServiceAlive(probe, owner) cancel() if aliveErr != nil { gone[row.ID] = errors.Is(aliveErr, opencode.ErrServiceNotRunning) reasons[row.ID] = fmt.Sprintf("the OpenCode service running stage %s was gone (%v), so its session could not be reattached", last.Stage, aliveErr) continue } plans = append(plans, reattachPlan{runID: row.ID, kind: reattachAdopt, step: last, owner: owner}) case "completed": plans = append(plans, reattachPlan{runID: row.ID, kind: reattachBetween, step: last}) } } return plans, reasons, gone, nil } // reattachRuns continues every run start-up recovery kept. It is called once, // after the runner's collaborators are wired and before requests are served. func (r *runner) reattachRuns() { plans := r.reattach r.reattach = nil for _, plan := range plans { r.reattachRun(plan) } } func (r *runner) reattachRun(plan reattachPlan) { id := plan.runID ctx, cancel := context.WithTimeout(r.ctx, 10*time.Second) defer cancel() if plan.kind == reattachRestart { if err := r.store.RestartStep(ctx, id, plan.step.Sequence, "Server restarted before this stage handed work to a harness; the stage was started again"); err != nil { log.Printf("run %d: restart stage %s after server restart: %v", id, plan.step.Stage, err) } } detail, err := r.store.GetRun(ctx, id) if err != nil { log.Printf("run %d: read for reattach: %v", id, err) return } r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary)) workspace := detail.Workspace request := protocol.Message{WritePaths: detail.WritePaths, AllowTools: detail.AllowTools, Model: detail.Model, StageModels: detail.StageModels} r.mu.Lock() defer r.mu.Unlock() switch plan.kind { case reattachBetween: log.Printf("run %d: continuing after stage %s following a server restart", id, plan.step.Stage) r.launch(id, workspace, func(ctx context.Context) { r.resume(ctx, id, detail, workspace, plan.step) }) default: state := rebuildAdoptState(detail, plan) if plan.kind == reattachAdopt { log.Printf("run %d: reattached stage %s #%d session %s after a server restart", id, plan.step.Stage, plan.step.Sequence, plan.owner.ID) } else { log.Printf("run %d: starting stage %s again after a server restart", id, plan.step.Stage) } r.launch(id, workspace, func(ctx context.Context) { r.runLoop(ctx, id, detail.Graph, detail.Name, workspace, request, state) }) } } // rebuildAdoptState restores the loop state at the planned visit: memory and // visit counts from every other persisted step, the handoff input from the // visit before it, and (for an adopted visit) the session to re-adopt. func rebuildAdoptState(detail *protocol.RunDetail, plan reattachPlan) *runState { decided := protocol.RunStep{Target: plan.step.Stage, Output: detail.Input} previous := 0 for _, step := range detail.Steps { if step.Sequence < plan.step.Sequence && step.Sequence > previous { previous = step.Sequence decided = step decided.Target = plan.step.Stage } } state := rebuildRunState(detail, decided) state.key = plan.step.Stage // The interrupted or adopted visit was counted; the loop counts it again. if state.visits[plan.step.Stage] > 0 { state.visits[plan.step.Stage]-- } if plan.kind == reattachAdopt { state.nextSequence = plan.step.Sequence state.adopt = &adoptedVisit{sequence: plan.step.Sequence, owner: plan.owner} } return state }