bd07abcde8
A model stage with harness=claude, provider=anthropic and a Claude model id runs one claude -p process per visit with the server user's Claude sign-in. Only the declared tools exist (--tools) and nothing else is approved (--permission-mode dontAsk); edits are approved only below the run directory and the stage's artifacts directory. A permissions block, run write paths, delivery=true and reasoning_effort none/minimal are refused, not widened. Usage comes from the CLI's final result and counts toward budgets. Failures are fixed text; billing, quota and rate-limit refusals map to ErrProviderQuota. The redacted stream is the session export and evidence. provider-check --harness claude and model overrides are covered.
227 lines
8.4 KiB
Go
227 lines
8.4 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/opencode"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
// Runs survive a server restart. With detach enabled (the installed server),
|
|
// a stopping server cancels each run's context with stage.ErrServerShutdown:
|
|
// stage clients leave their admitted harness session running and owned, and the
|
|
// loop returns without settling anything. The next process plans, before it
|
|
// serves requests, how to continue every run it finds in flight:
|
|
//
|
|
// - a running visit with a live OpenCode V2 session is re-adopted: the loop
|
|
// re-enters that visit, attaches to the session, waits for it to settle (or
|
|
// reads the result it reached while the server was down) and continues the
|
|
// graph as if it had watched it;
|
|
// - a visit begun but never handed to a harness is marked interrupted and the
|
|
// same stage starts again (nothing was admitted, so nothing replays);
|
|
// - a run between visits continues with its next stage;
|
|
// - a session whose harness process is gone, or that belongs to a harness the
|
|
// server owned as a child (Pi, OpenCode V1), falls back to the previous
|
|
// behaviour: the run is interrupted, with the reason in its diagnostic.
|
|
//
|
|
// Only an operator cancel (cancel_run) stops a stage session.
|
|
|
|
type reattachKind int
|
|
|
|
const (
|
|
reattachAdopt reattachKind = iota
|
|
reattachRestart
|
|
reattachBetween
|
|
)
|
|
|
|
// reattachPlan is one run start-up recovery kept for reattachRuns.
|
|
type reattachPlan struct {
|
|
runID int64
|
|
kind reattachKind
|
|
// step is the running visit (adopt/restart) or the last completed one.
|
|
step protocol.RunStep
|
|
owner pipeline.SessionOwner
|
|
}
|
|
|
|
// adoptedVisit is the visit a runLoop re-adopts instead of starting.
|
|
type adoptedVisit struct {
|
|
sequence int
|
|
owner pipeline.SessionOwner
|
|
}
|
|
|
|
// stageServiceAlive reports whether a retained stage session's harness service
|
|
// is still running. It is a seam for tests.
|
|
var stageServiceAlive = func(ctx context.Context, owner pipeline.SessionOwner) error {
|
|
return opencode.ReferenceAlive(ctx, owner.Reference)
|
|
}
|
|
|
|
// interruptRecoveredSession stops a previously active session that start-up
|
|
// recovery does not re-adopt. It is a seam for tests.
|
|
var interruptRecoveredSession = func(ctx context.Context, owner pipeline.SessionOwner) error {
|
|
if owner.Backend == "pi" {
|
|
return stage.InterruptPi(ctx, owner.Reference)
|
|
}
|
|
if owner.Backend == stage.ClaudeBackend {
|
|
return stage.InterruptClaudeStage(ctx, owner.Reference)
|
|
}
|
|
return (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(ctx, owner.ID)
|
|
}
|
|
|
|
// runContext derives one run's execution context. With detach, a server
|
|
// shutdown cancels it with stage.ErrServerShutdown as the cause, while the
|
|
// returned cancel (an operator cancel or the run's own end) uses the ordinary
|
|
// context.Canceled, so stage clients can tell the two apart.
|
|
func (r *runner) runContext() (context.Context, context.CancelFunc) {
|
|
if !r.detach {
|
|
return context.WithCancel(r.ctx)
|
|
}
|
|
ctx, cancel := context.WithCancelCause(context.WithoutCancel(r.ctx))
|
|
stop := context.AfterFunc(r.ctx, func() { cancel(stage.ErrServerShutdown) })
|
|
return ctx, func() { stop(); cancel(context.Canceled) }
|
|
}
|
|
|
|
// planReattach decides, per in-flight run, whether start-up continues it.
|
|
// Runs it does not plan keep today's recovery (interrupted); reasons carries the
|
|
// explanation for the ones that could not be reattached.
|
|
// gone lists runs whose session's service is not running: there is nothing
|
|
// left to interrupt, so recovery only clears their ownership.
|
|
func planReattach(ctx context.Context, s *pipeline.Store) ([]reattachPlan, map[int64]string, map[int64]bool, error) {
|
|
reasons, gone := map[int64]string{}, map[int64]bool{}
|
|
rows, err := s.ListRuns(ctx)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
var plans []reattachPlan
|
|
for _, row := range rows {
|
|
if row.Status != protocol.RunStatusRunning || row.CancelRequested || row.CleanupUnconfirmed {
|
|
continue
|
|
}
|
|
detail, err := s.GetRun(ctx, row.ID)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
if len(detail.Steps) == 0 {
|
|
continue
|
|
}
|
|
last := detail.Steps[0]
|
|
for _, step := range detail.Steps {
|
|
if step.Sequence > last.Sequence {
|
|
last = step
|
|
}
|
|
}
|
|
switch last.Status {
|
|
case protocol.StepStatusRunning:
|
|
if last.SessionID == "" {
|
|
plans = append(plans, reattachPlan{runID: row.ID, kind: reattachRestart, step: last})
|
|
continue
|
|
}
|
|
owner, err := s.RunSessionOwner(ctx, row.ID, last.Sequence)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
owner.RunID = row.ID
|
|
if owner.Backend != "v2" {
|
|
name := "OpenCode V1"
|
|
if owner.Backend == "pi" {
|
|
name = "Pi"
|
|
}
|
|
if owner.Backend == stage.ClaudeBackend {
|
|
name = "Claude Code"
|
|
}
|
|
reasons[row.ID] = fmt.Sprintf("%s stage sessions belong to the stopped server process, so stage %s could not be reattached", name, last.Stage)
|
|
continue
|
|
}
|
|
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
aliveErr := stageServiceAlive(probe, owner)
|
|
cancel()
|
|
if aliveErr != nil {
|
|
gone[row.ID] = errors.Is(aliveErr, opencode.ErrServiceNotRunning)
|
|
reasons[row.ID] = fmt.Sprintf("the OpenCode service running stage %s was gone (%v), so its session could not be reattached", last.Stage, aliveErr)
|
|
continue
|
|
}
|
|
plans = append(plans, reattachPlan{runID: row.ID, kind: reattachAdopt, step: last, owner: owner})
|
|
case "completed":
|
|
plans = append(plans, reattachPlan{runID: row.ID, kind: reattachBetween, step: last})
|
|
}
|
|
}
|
|
return plans, reasons, gone, nil
|
|
}
|
|
|
|
// reattachRuns continues every run start-up recovery kept. It is called once,
|
|
// after the runner's collaborators are wired and before requests are served.
|
|
func (r *runner) reattachRuns() {
|
|
plans := r.reattach
|
|
r.reattach = nil
|
|
for _, plan := range plans {
|
|
r.reattachRun(plan)
|
|
}
|
|
}
|
|
|
|
func (r *runner) reattachRun(plan reattachPlan) {
|
|
id := plan.runID
|
|
ctx, cancel := context.WithTimeout(r.ctx, 10*time.Second)
|
|
defer cancel()
|
|
if plan.kind == reattachRestart {
|
|
if err := r.store.RestartStep(ctx, id, plan.step.Sequence, "Server restarted before this stage handed work to a harness; the stage was started again"); err != nil {
|
|
log.Printf("run %d: restart stage %s after server restart: %v", id, plan.step.Stage, err)
|
|
}
|
|
}
|
|
detail, err := r.store.GetRun(ctx, id)
|
|
if err != nil {
|
|
log.Printf("run %d: read for reattach: %v", id, err)
|
|
return
|
|
}
|
|
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
|
|
workspace := detail.Workspace
|
|
request := protocol.Message{WritePaths: detail.WritePaths, AllowTools: detail.AllowTools, Model: detail.Model, StageModels: detail.StageModels}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
switch plan.kind {
|
|
case reattachBetween:
|
|
log.Printf("run %d: continuing after stage %s following a server restart", id, plan.step.Stage)
|
|
r.launch(id, workspace, func(ctx context.Context) { r.resume(ctx, id, detail, workspace, plan.step) })
|
|
default:
|
|
state := rebuildAdoptState(detail, plan)
|
|
if plan.kind == reattachAdopt {
|
|
log.Printf("run %d: reattached stage %s #%d session %s after a server restart", id, plan.step.Stage, plan.step.Sequence, plan.owner.ID)
|
|
} else {
|
|
log.Printf("run %d: starting stage %s again after a server restart", id, plan.step.Stage)
|
|
}
|
|
r.launch(id, workspace, func(ctx context.Context) {
|
|
r.runLoop(ctx, id, detail.Graph, detail.Name, workspace, request, state)
|
|
})
|
|
}
|
|
}
|
|
|
|
// rebuildAdoptState restores the loop state at the planned visit: memory and
|
|
// visit counts from every other persisted step, the handoff input from the
|
|
// visit before it, and (for an adopted visit) the session to re-adopt.
|
|
func rebuildAdoptState(detail *protocol.RunDetail, plan reattachPlan) *runState {
|
|
decided := protocol.RunStep{Target: plan.step.Stage, Output: detail.Input}
|
|
previous := 0
|
|
for _, step := range detail.Steps {
|
|
if step.Sequence < plan.step.Sequence && step.Sequence > previous {
|
|
previous = step.Sequence
|
|
decided = step
|
|
decided.Target = plan.step.Stage
|
|
}
|
|
}
|
|
state := rebuildRunState(detail, decided)
|
|
state.key = plan.step.Stage
|
|
// The interrupted or adopted visit was counted; the loop counts it again.
|
|
if state.visits[plan.step.Stage] > 0 {
|
|
state.visits[plan.step.Stage]--
|
|
}
|
|
if plan.kind == reattachAdopt {
|
|
state.nextSequence = plan.step.Sequence
|
|
state.adopt = &adoptedVisit{sequence: plan.step.Sequence, owner: plan.owner}
|
|
}
|
|
return state
|
|
}
|