Files
pipelines/internal/server/run_reattach.go
T
alan bd07abcde8 Run pipeline stages on Claude Code (harness=claude)
A model stage with harness=claude, provider=anthropic and a Claude model id
runs one claude -p process per visit with the server user's Claude sign-in.
Only the declared tools exist (--tools) and nothing else is approved
(--permission-mode dontAsk); edits are approved only below the run directory
and the stage's artifacts directory. A permissions block, run write paths,
delivery=true and reasoning_effort none/minimal are refused, not widened.

Usage comes from the CLI's final result and counts toward budgets. Failures
are fixed text; billing, quota and rate-limit refusals map to
ErrProviderQuota. The redacted stream is the session export and evidence.
provider-check --harness claude and model overrides are covered.
2026-11-11 10:30:37 +00:00

227 lines
8.4 KiB
Go

package server
import (
"context"
"errors"
"fmt"
"log"
"time"
"git.a-wan.net/alan/pipelines/internal/opencode"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// Runs survive a server restart. With detach enabled (the installed server),
// a stopping server cancels each run's context with stage.ErrServerShutdown:
// stage clients leave their admitted harness session running and owned, and the
// loop returns without settling anything. The next process plans, before it
// serves requests, how to continue every run it finds in flight:
//
// - a running visit with a live OpenCode V2 session is re-adopted: the loop
// re-enters that visit, attaches to the session, waits for it to settle (or
// reads the result it reached while the server was down) and continues the
// graph as if it had watched it;
// - a visit begun but never handed to a harness is marked interrupted and the
// same stage starts again (nothing was admitted, so nothing replays);
// - a run between visits continues with its next stage;
// - a session whose harness process is gone, or that belongs to a harness the
// server owned as a child (Pi, OpenCode V1), falls back to the previous
// behaviour: the run is interrupted, with the reason in its diagnostic.
//
// Only an operator cancel (cancel_run) stops a stage session.
type reattachKind int
const (
reattachAdopt reattachKind = iota
reattachRestart
reattachBetween
)
// reattachPlan is one run start-up recovery kept for reattachRuns.
type reattachPlan struct {
runID int64
kind reattachKind
// step is the running visit (adopt/restart) or the last completed one.
step protocol.RunStep
owner pipeline.SessionOwner
}
// adoptedVisit is the visit a runLoop re-adopts instead of starting.
type adoptedVisit struct {
sequence int
owner pipeline.SessionOwner
}
// stageServiceAlive reports whether a retained stage session's harness service
// is still running. It is a seam for tests.
var stageServiceAlive = func(ctx context.Context, owner pipeline.SessionOwner) error {
return opencode.ReferenceAlive(ctx, owner.Reference)
}
// interruptRecoveredSession stops a previously active session that start-up
// recovery does not re-adopt. It is a seam for tests.
var interruptRecoveredSession = func(ctx context.Context, owner pipeline.SessionOwner) error {
if owner.Backend == "pi" {
return stage.InterruptPi(ctx, owner.Reference)
}
if owner.Backend == stage.ClaudeBackend {
return stage.InterruptClaudeStage(ctx, owner.Reference)
}
return (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(ctx, owner.ID)
}
// runContext derives one run's execution context. With detach, a server
// shutdown cancels it with stage.ErrServerShutdown as the cause, while the
// returned cancel (an operator cancel or the run's own end) uses the ordinary
// context.Canceled, so stage clients can tell the two apart.
func (r *runner) runContext() (context.Context, context.CancelFunc) {
if !r.detach {
return context.WithCancel(r.ctx)
}
ctx, cancel := context.WithCancelCause(context.WithoutCancel(r.ctx))
stop := context.AfterFunc(r.ctx, func() { cancel(stage.ErrServerShutdown) })
return ctx, func() { stop(); cancel(context.Canceled) }
}
// planReattach decides, per in-flight run, whether start-up continues it.
// Runs it does not plan keep today's recovery (interrupted); reasons carries the
// explanation for the ones that could not be reattached.
// gone lists runs whose session's service is not running: there is nothing
// left to interrupt, so recovery only clears their ownership.
func planReattach(ctx context.Context, s *pipeline.Store) ([]reattachPlan, map[int64]string, map[int64]bool, error) {
reasons, gone := map[int64]string{}, map[int64]bool{}
rows, err := s.ListRuns(ctx)
if err != nil {
return nil, nil, nil, err
}
var plans []reattachPlan
for _, row := range rows {
if row.Status != protocol.RunStatusRunning || row.CancelRequested || row.CleanupUnconfirmed {
continue
}
detail, err := s.GetRun(ctx, row.ID)
if err != nil {
return nil, nil, nil, err
}
if len(detail.Steps) == 0 {
continue
}
last := detail.Steps[0]
for _, step := range detail.Steps {
if step.Sequence > last.Sequence {
last = step
}
}
switch last.Status {
case protocol.StepStatusRunning:
if last.SessionID == "" {
plans = append(plans, reattachPlan{runID: row.ID, kind: reattachRestart, step: last})
continue
}
owner, err := s.RunSessionOwner(ctx, row.ID, last.Sequence)
if err != nil {
return nil, nil, nil, err
}
owner.RunID = row.ID
if owner.Backend != "v2" {
name := "OpenCode V1"
if owner.Backend == "pi" {
name = "Pi"
}
if owner.Backend == stage.ClaudeBackend {
name = "Claude Code"
}
reasons[row.ID] = fmt.Sprintf("%s stage sessions belong to the stopped server process, so stage %s could not be reattached", name, last.Stage)
continue
}
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
aliveErr := stageServiceAlive(probe, owner)
cancel()
if aliveErr != nil {
gone[row.ID] = errors.Is(aliveErr, opencode.ErrServiceNotRunning)
reasons[row.ID] = fmt.Sprintf("the OpenCode service running stage %s was gone (%v), so its session could not be reattached", last.Stage, aliveErr)
continue
}
plans = append(plans, reattachPlan{runID: row.ID, kind: reattachAdopt, step: last, owner: owner})
case "completed":
plans = append(plans, reattachPlan{runID: row.ID, kind: reattachBetween, step: last})
}
}
return plans, reasons, gone, nil
}
// reattachRuns continues every run start-up recovery kept. It is called once,
// after the runner's collaborators are wired and before requests are served.
func (r *runner) reattachRuns() {
plans := r.reattach
r.reattach = nil
for _, plan := range plans {
r.reattachRun(plan)
}
}
func (r *runner) reattachRun(plan reattachPlan) {
id := plan.runID
ctx, cancel := context.WithTimeout(r.ctx, 10*time.Second)
defer cancel()
if plan.kind == reattachRestart {
if err := r.store.RestartStep(ctx, id, plan.step.Sequence, "Server restarted before this stage handed work to a harness; the stage was started again"); err != nil {
log.Printf("run %d: restart stage %s after server restart: %v", id, plan.step.Stage, err)
}
}
detail, err := r.store.GetRun(ctx, id)
if err != nil {
log.Printf("run %d: read for reattach: %v", id, err)
return
}
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
workspace := detail.Workspace
request := protocol.Message{WritePaths: detail.WritePaths, AllowTools: detail.AllowTools, Model: detail.Model, StageModels: detail.StageModels}
r.mu.Lock()
defer r.mu.Unlock()
switch plan.kind {
case reattachBetween:
log.Printf("run %d: continuing after stage %s following a server restart", id, plan.step.Stage)
r.launch(id, workspace, func(ctx context.Context) { r.resume(ctx, id, detail, workspace, plan.step) })
default:
state := rebuildAdoptState(detail, plan)
if plan.kind == reattachAdopt {
log.Printf("run %d: reattached stage %s #%d session %s after a server restart", id, plan.step.Stage, plan.step.Sequence, plan.owner.ID)
} else {
log.Printf("run %d: starting stage %s again after a server restart", id, plan.step.Stage)
}
r.launch(id, workspace, func(ctx context.Context) {
r.runLoop(ctx, id, detail.Graph, detail.Name, workspace, request, state)
})
}
}
// rebuildAdoptState restores the loop state at the planned visit: memory and
// visit counts from every other persisted step, the handoff input from the
// visit before it, and (for an adopted visit) the session to re-adopt.
func rebuildAdoptState(detail *protocol.RunDetail, plan reattachPlan) *runState {
decided := protocol.RunStep{Target: plan.step.Stage, Output: detail.Input}
previous := 0
for _, step := range detail.Steps {
if step.Sequence < plan.step.Sequence && step.Sequence > previous {
previous = step.Sequence
decided = step
decided.Target = plan.step.Stage
}
}
state := rebuildRunState(detail, decided)
state.key = plan.step.Stage
// The interrupted or adopted visit was counted; the loop counts it again.
if state.visits[plan.step.Stage] > 0 {
state.visits[plan.step.Stage]--
}
if plan.kind == reattachAdopt {
state.nextSequence = plan.step.Sequence
state.adopt = &adoptedVisit{sequence: plan.step.Sequence, owner: plan.owner}
}
return state
}