Files
pipelines/internal/server/websocket.go
T
alan 3bfb3d6b44 Fix decide-after-finish refusal and run size loop data race
A run stays in r.active after finishRunState persists its terminal status
until its goroutine finishes notifying, so a decision in that window was
refused as "already active". decide now refuses only runs whose terminal
state is not yet durable; a finished run gets ResolveDecision's
already-decided / not-waiting answer, and an executing run is still refused.

Tests overwrote runner.artifactsRoot after Handler/newRunner had started
background goroutines (sizeLoop, session export jobs) that read it. The
root is now fixed at construction through newRunnerAt, which
HandlerWithOptions uses for Options.ArtifactsRoot, and tests configure it
there instead of mutating the field afterwards.
2026-11-01 18:58:14 +00:00

438 lines
16 KiB
Go

package server
import (
"context"
"fmt"
"log"
"net/http"
"strings"
"sync"
"time"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
"git.a-wan.net/alan/pipelines/examples"
"git.a-wan.net/alan/pipelines/internal/buildinfo"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/summarise"
)
// Options configures the HTTP handler.
type Options struct {
// AllowAuthoring permits publishing immutable definitions. Execution servers
// default to run-only: a CLI --yes flag or raw RPC cannot grant this server
// capability. Enable deliberately on a separate trusted authoring instance.
AllowAuthoring bool
// ArtifactsRoot overrides where stage artifacts live. Empty uses the store's
// directory (the database's parent), matching the data-directory default.
ArtifactsRoot string
// Chat brokers OpenCode chat traffic over /connect. Nil answers every
// chat_* request with a protocol error; a configured server passes an
// *opencode.Chat bound to the owned endpoint.
Chat chatBroker
// OpenCodeStatus is the honest report of how chat is attached to OpenCode,
// echoed on get_status: "endpoint" when the broker is attached to the
// operator's configured endpoint, "fallback" when that endpoint could not be
// attached and chat fell back to the server's own instance, "spawned" when
// no endpoint is configured, and "unavailable" when chat is disabled. It
// carries the endpoint URL but never a credential.
OpenCodeStatus protocol.ServiceStatus
// OperatorRegistry is the operator-session registry chat_sessions lists and
// chat_create/chat_directories read. When nil, HandlerWithOptions derives it
// from the first store; when neither is available chat_sessions returns an
// empty list (never the endpoint's whole session list) and chat_create is
// refused. Purely pipeline stage ownership is unaffected.
OperatorRegistry operatorRegistry
// RunTitles generates the operator-facing title stored with each new run.
// When nil, run creation stores the prompt-derived fallback without a model
// call, which keeps tests and unconfigured servers deterministic.
RunTitles func(context.Context, string) summarise.Title
// StepVerdicts generates the bounded one-line reason stored with each
// completed stage visit. When nil, the store derives the verdict's reason at
// read time with no model call, which keeps tests and unconfigured servers
// deterministic.
StepVerdicts func(context.Context, map[string]any) string
// VerificationFixtures are explicit server-owned HTTP fixtures for optional
// target-aware verification. Definitions can name only a registered ID; they
// cannot select an endpoint or supply provenance.
VerificationFixtures []VerificationFixture
}
// Handler serves the persistent connection with default options. Canceling ctx
// stops active clients.
func Handler(ctx context.Context, stores ...*pipeline.Store) http.Handler {
return HandlerWithOptions(ctx, Options{}, stores...)
}
// HandlerWithOptions serves the persistent connection and the stage-artifact
// endpoint. Canceling ctx stops active clients.
func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipeline.Store) http.Handler {
build := buildinfo.Current()
started := time.Now()
hub := newChatHub()
events := newRunEventHub()
mux := http.NewServeMux()
var runs *runner
var runErr error
if len(stores) > 0 {
runs, runErr = newRunnerAt(ctx, stores[0], options.ArtifactsRoot)
if runErr == nil {
runs.events = events
}
if runErr == nil && options.RunTitles != nil {
runs.title = options.RunTitles
}
if runErr == nil && options.StepVerdicts != nil {
runs.verdict = options.StepVerdicts
}
if runErr == nil {
fixtures := newVerificationFixtureRegistry(options.VerificationFixtures)
runs.verifyTarget = fixtures.verify
}
if runErr == nil {
runs.schedulerWG.Add(1)
go runs.scheduleLoop()
// The run-size backfill runs in the background so an upgraded
// database never delays startup.
runs.sizeKick = make(chan struct{}, 1)
runs.schedulerWG.Add(1)
go runs.sizeLoop()
}
}
// chat_sessions lists the operator's own recorded sessions. Prefer an
// injected registry; otherwise derive it from the store. With neither, the
// handler fails closed to an empty list.
registry := options.OperatorRegistry
if registry == nil && len(stores) > 0 {
registry = stores[0]
}
hub.baseCtx = ctx
hub.registry = registry
var settingsStore notificationSettingsStore
if len(stores) > 0 {
settingsStore = stores[0]
}
hub.settings = settingsStore
if len(stores) > 0 {
// Prompts queued behind a running turn are durable and owned by their
// session; resume delivering any a previous process left queued.
hub.queueStore = stores[0]
if options.Chat != nil {
go hub.resumeQueued(ctx, options.Chat)
}
}
acks := newAttentionAckHub()
var ackStore attentionAckStore
var notes *appNotifier
if len(stores) > 0 {
ackStore = stores[0]
notes = newAppNotifier(stores[0])
hub.notes = notes
if runs != nil {
runs.notes = notes
}
}
mux.HandleFunc("GET /artifacts/{run}/{step}/{name...}", func(w http.ResponseWriter, r *http.Request) {
if runs == nil {
http.NotFound(w, r)
return
}
runs.serveArtifact(w, r)
})
mux.HandleFunc("POST /chat-uploads", serveChatUpload)
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if runErr != nil {
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = w.Write([]byte(`{"healthy":false}`))
return
}
_, _ = w.Write([]byte(`{"healthy":true}`))
})
connect := func(w http.ResponseWriter, r *http.Request) {
conn, err := websocket.Accept(w, r, nil)
if err != nil {
return
}
defer conn.CloseNow()
conn.SetReadLimit(2 << 20)
// connCtx is canceled when this connection's read loop returns, so a
// chat subscription started for the client stops on disconnect. A single
// write mutex serializes the read loop's responses and the chat stream
// goroutine, which must never interleave partial websocket writes.
connCtx, connCancel := context.WithCancel(ctx)
defer connCancel()
var writeMu sync.Mutex
write := func(message protocol.Message) error {
// Every server frame (responses, the welcome and chat events) carries
// the build identity and protocol so an older client fails fast or
// warns once instead of silently misreading a field. build is
// computed once per handler.
if message.Build == nil {
message.Build = &build
}
if message.Protocol == 0 {
message.Protocol = protocol.ProtocolVersion
}
writeMu.Lock()
defer writeMu.Unlock()
writeCtx, cancel := context.WithTimeout(connCtx, 5*time.Second)
defer cancel()
return wsjson.Write(writeCtx, conn, message)
}
// clientID is this connection's identity for chat session claims: a
// second client is refused a session another connection holds.
clientID := chatClientToken()
// Drop every run subscription this connection holds when the read loop
// returns, so no subscriber, goroutine or timer outlives the connection.
defer events.unsubscribeClient(clientID)
// Drop every chat subscription too, so a chat room held only by this
// connection releases its broker stream and turn runner.
defer hub.detachClient(clientID)
if len(stores) > 0 {
items, err := stores[0].List(ctx)
if err != nil {
log.Printf("list pipelines: %v", err)
return
}
if err := write(protocol.Message{Type: protocol.PipelineList, Pipelines: items, NotificationEndpoints: notificationEndpoints()}); err != nil {
return
}
}
// Shared attention acknowledgements reach every open connection, after
// the welcome frame so a push never precedes it.
acks.attach(clientID, write)
defer acks.detach(clientID)
if notes != nil {
defer notes.detach(clientID)
}
for {
var request protocol.Message
if err := wsjson.Read(connCtx, conn, &request); err != nil {
return
}
log.Printf("command received: client=%s type=%q id=%q pipeline=%d version=%d", r.RemoteAddr, request.Type, request.ID, request.PipelineID, request.Version)
if request.Type == protocol.Submit && !options.AllowAuthoring {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "pipeline authoring is disabled on this execution server; ask the operator to publish through an explicitly enabled authoring server"})
continue
}
if handleChat(connCtx, request, options.Chat, hub, registry, clientID, write) {
continue
}
if handleNotificationSettings(ctx, request, settingsStore, write) {
continue
}
if handleAttentionAcks(ctx, request, ackStore, acks, notes, clientID, write) {
continue
}
if handleAppNotifications(ctx, request, notes, ackStore, acks, clientID, write) {
continue
}
if handleRunEvents(request, events, runs, clientID, write) {
continue
}
response := protocol.Message{Type: protocol.Pong, ID: request.ID}
if request.Type == protocol.List || request.Type == protocol.Submit {
response.NotificationEndpoints = notificationEndpoints()
}
if request.Type == protocol.GetStatus {
response.Type = protocol.Status
response.Status = systemStatus(r, started, runErr)
// How chat is attached to OpenCode is part of the server's status:
// a configured endpoint that could not be attached reports
// "fallback" here rather than silently disappearing.
response.Status.OpenCode = options.OpenCodeStatus
} else if request.Type == protocol.Ping {
response.Time = time.Now().UTC().Format(time.RFC3339)
} else if request.Type == protocol.ListTemplates {
response.Type = protocol.Templates
response.Templates = examples.Catalog()
model, backend := starterDefaults()
for i := range response.Templates {
response.Templates[i].Source = pipeline.WithDefaults(response.Templates[i].Source, model, backend)
}
} else if request.Type == protocol.Validate {
graph, err := pipeline.ParseDefinition(request.Source)
response.Type = protocol.Validated
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
} else {
response.Graph = &graph
}
} else if (request.Type == protocol.ListSchedules || request.Type == protocol.SaveSchedule || request.Type == protocol.DeleteSchedule) && len(stores) > 0 {
var err error
switch request.Type {
case protocol.ListSchedules:
response.Type = protocol.Schedules
if request.PipelineID < 0 {
err = fmt.Errorf("pipeline_id must be nonnegative")
} else {
response.Schedules, err = stores[0].ListSchedules(ctx, request.PipelineID)
}
case protocol.SaveSchedule:
response.Type = protocol.SavedScheduleReply
if request.Schedule == nil {
err = fmt.Errorf("schedule is required")
} else {
response.Schedule, err = stores[0].SaveSchedule(ctx, *request.Schedule, time.Now(), request.RequestKey)
}
case protocol.DeleteSchedule:
response.Type = protocol.ScheduleDeleted
response.ScheduleID = request.ScheduleID
err = stores[0].DeleteSchedule(ctx, request.ScheduleID, request.ExpectedRevision)
}
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
}
} else if request.Type == protocol.Submit && len(stores) > 0 {
response.Type = protocol.Submitted
response.Name = strings.TrimSpace(request.Name)
version, err := stores[0].SubmitChecked(ctx, request.Name, request.Source, request.CreateOnly, request.ExpectedVersion)
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
} else {
response.Version = version
response.Pipelines, err = stores[0].List(ctx)
if err != nil {
log.Printf("list pipelines after submission: %v", err)
}
for _, item := range response.Pipelines {
if item.Name == response.Name {
response.Detail, err = stores[0].Get(ctx, item.ID, version)
if err != nil {
log.Printf("load submitted pipeline: %v", err)
}
break
}
}
}
} else if request.Type == protocol.GetPipeline && len(stores) > 0 {
response.Type = protocol.Pipeline
detail, err := stores[0].Get(ctx, request.PipelineID, request.Version)
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
} else {
response.Detail = detail
}
} else if request.Type == protocol.List && len(stores) > 0 {
response.Type = protocol.PipelineList
response.DefaultModel, response.DefaultHarness = starterDefaults()
items, err := stores[0].List(ctx)
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
} else {
response.Pipelines = items
}
} else if request.Type == protocol.Search {
if len(stores) == 0 {
response.Type = protocol.Error
response.Error = "search unavailable: no store"
} else if request.Search == nil {
response.Type = protocol.Error
response.Error = "search requires a query"
} else {
response.Type = protocol.SearchResults
var err error
response.SearchResults, err = stores[0].Search(ctx, *request.Search, time.Now())
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
}
}
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence || request.Type == protocol.RepairSessionExport || request.Type == protocol.DeleteRuns) {
var err error
switch request.Type {
case protocol.DeleteRuns:
// Like cancel and retry, deleting run history is an execution
// operation and is not gated by AllowAuthoring.
response.Type = protocol.RunsDeleted
response.RunsDeleted, err = runs.deleteRuns(request.RunIDs)
case protocol.RepairSessionExport:
response.Type = protocol.Run
response.RunDetail, err = runs.repairSessionExport(ctx, request.RunID, request.ExportSequence)
case protocol.GetRunEvidence:
response.Type = protocol.RunEvidence
response.Evidence, err = runs.evidence(ctx, request.RunID)
case protocol.RunPipeline:
response.Type = protocol.RunStarted
response.RunDetail, err = runs.start(request)
case protocol.RetryRun:
response.Type = protocol.RunStarted
response.RunDetail, err = runs.retry(request)
case protocol.DecideRun:
response.Type = protocol.Run
response.RunDetail, err = runs.decide(request)
case protocol.SetRunTitle:
response.Type = protocol.Run
response.RunDetail, err = runs.setTitle(request.RunID, request.Title)
case protocol.ListRuns:
response.Type = protocol.Runs
runs.refreshListSizes(connCtx)
response.Runs, err = runs.listRuns()
if err == nil {
response.Usage, err = stores[0].UsageStats(ctx)
}
case protocol.GetRun:
response.Type = protocol.Run
if request.RunID == 0 && request.RequestKey != "" {
var id int64
id, err = stores[0].RunByKey(connCtx, request.RequestKey)
if err == nil && id != 0 {
response.RunDetail, err = runs.getRun(id)
}
} else {
response.RunDetail, err = runs.getRun(request.RunID)
}
case protocol.CancelRun:
response.Type = protocol.Run
err = runs.cancel(request.RunID)
if err == nil {
response.RunDetail, err = runs.getRun(request.RunID)
}
}
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
}
} else {
response.Type = protocol.Error
response.Error = "unknown command: " + request.Type
if runErr != nil {
response.Error = "execution unavailable: " + runErr.Error()
}
}
if err := write(response); err != nil {
log.Printf("write response: %v", err)
return
}
selectedVersion := response.Version
if response.Detail != nil {
selectedVersion = response.Detail.Version
}
log.Printf("response sent: client=%s type=%q id=%q time=%q error=%q version=%d", r.RemoteAddr, response.Type, response.ID, response.Time, response.Error, selectedVersion)
}
}
mux.HandleFunc("GET /connect", connect)
return &runHandler{Handler: mux, runner: runs}
}
type runHandler struct {
http.Handler
runner *runner
}
// WaitRuns drains active execution before the database is closed on shutdown.
func WaitRuns(handler http.Handler) {
if h, ok := handler.(*runHandler); ok && h.runner != nil {
h.runner.schedulerWG.Wait()
h.runner.wg.Wait()
}
}