Files
pipelines/internal/server/websocket.go
T

565 lines
22 KiB
Go

package server
import (
"context"
"fmt"
"log"
"net/http"
"strings"
"sync"
"time"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
"git.a-wan.net/alan/pipelines/examples"
"git.a-wan.net/alan/pipelines/internal/buildinfo"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/summarise"
)
// Options configures the HTTP handler.
type Options struct {
// ServerStatsSampler overrides host sampling for deterministic tests.
ServerStatsSampler ServerStatsSampler
// AllowAuthoring permits publishing immutable definitions. Execution servers
// default to run-only: a CLI --yes flag or raw RPC cannot grant this server
// capability. Enable deliberately on a separate trusted authoring instance.
AllowAuthoring bool
// ArtifactsRoot overrides where stage artifacts live. Empty uses the store's
// directory (the database's parent), matching the data-directory default.
ArtifactsRoot string
// Chat brokers OpenCode chat traffic over /connect. Nil answers every
// chat_* request with a protocol error; a configured server passes an
// *opencode.Chat bound to the owned endpoint.
Chat chatBroker
// OpenCodeStatus is the honest report of how chat is attached to OpenCode,
// echoed on get_status: "endpoint" when the broker is attached to the
// operator's configured endpoint, "fallback" when that endpoint could not be
// attached and chat fell back to the server's own instance, "spawned" when
// no endpoint is configured, and "unavailable" when chat is disabled. It
// carries the endpoint URL but never a credential.
OpenCodeStatus protocol.ServiceStatus
// OperatorRegistry is the operator-session registry chat_sessions lists and
// chat_create/chat_directories read. When nil, HandlerWithOptions derives it
// from the first store; when neither is available chat_sessions returns an
// empty list (never the endpoint's whole session list) and chat_create is
// refused. Purely pipeline stage ownership is unaffected.
OperatorRegistry operatorRegistry
// RunTitles generates the operator-facing title stored with each new run.
// When nil, run creation stores the prompt-derived fallback without a model
// call, which keeps tests and unconfigured servers deterministic.
RunTitles func(context.Context, string) summarise.Title
// StepVerdicts generates the bounded one-line reason stored with each
// completed stage visit. When nil, the store derives the verdict's reason at
// read time with no model call, which keeps tests and unconfigured servers
// deterministic.
StepVerdicts func(context.Context, map[string]any) string
// VerificationFixtures are explicit server-owned HTTP fixtures for optional
// target-aware verification. Definitions can name only a registered ID; they
// cannot select an endpoint or supply provenance.
VerificationFixtures []VerificationFixture
// DetachSessions keeps harness sessions running across a server restart:
// a stopping server leaves stage and chat sessions running (DetachSessions
// on the handler records the chats) and the next start reattaches them.
// False keeps the previous behaviour (tests, embedded servers): a stopping
// server interrupts its runs and the next start marks them interrupted.
DetachSessions bool
}
// Handler serves the persistent connection with default options. Canceling ctx
// stops active clients.
func Handler(ctx context.Context, stores ...*pipeline.Store) http.Handler {
return HandlerWithOptions(ctx, Options{}, stores...)
}
// HandlerWithOptions serves the persistent connection and the stage-artifact
// endpoint. Canceling ctx stops active clients.
func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipeline.Store) http.Handler {
build := buildinfo.Current()
started := time.Now()
hub := newChatHub()
events := newRunEventHub()
mux := http.NewServeMux()
var runs *runner
var runErr error
if len(stores) > 0 {
runs, runErr = newRunnerReattaching(ctx, stores[0], options.ArtifactsRoot, options.DetachSessions)
if runErr == nil {
runs.events = events
}
if runErr == nil && options.RunTitles != nil {
runs.title = options.RunTitles
}
if runErr == nil && options.StepVerdicts != nil {
runs.verdict = options.StepVerdicts
}
if runErr == nil {
fixtures := newVerificationFixtureRegistry(options.VerificationFixtures)
runs.verifyTarget = fixtures.verify
}
if runErr == nil {
runs.schedulerWG.Add(1)
go runs.scheduleLoop()
// The run-size backfill runs in the background so an upgraded
// database never delays startup.
runs.sizeKick = make(chan struct{}, 1)
runs.schedulerWG.Add(1)
go runs.sizeLoop()
}
}
var storage *storageManager
if runs != nil {
storage = newStorageManager(runs)
runs.schedulerWG.Add(1)
go storage.loop(ctx)
}
// chat_sessions lists the operator's own recorded sessions. Prefer an
// injected registry; otherwise derive it from the store. With neither, the
// handler fails closed to an empty list.
registry := options.OperatorRegistry
if registry == nil && len(stores) > 0 {
registry = stores[0]
}
hub.baseCtx = ctx
hub.registry = registry
var settingsStore notificationSettingsStore
if len(stores) > 0 {
settingsStore = stores[0]
}
hub.settings = settingsStore
if len(stores) > 0 {
// Prompts queued behind a running turn are durable and owned by their
// session; resume delivering any a previous process left queued.
hub.queueStore = stores[0]
hub.liveStore = stores[0]
}
acks := newAttentionAckHub()
var ackStore attentionAckStore
var notes *appNotifier
if len(stores) > 0 {
ackStore = stores[0]
notes = newAppNotifier(stores[0])
hub.notes = notes
if runs != nil {
runs.notes = notes
}
}
mux.HandleFunc("GET /artifacts/{run}/{step}/{name...}", func(w http.ResponseWriter, r *http.Request) {
if runs == nil {
http.NotFound(w, r)
return
}
runs.serveArtifact(w, r)
})
if len(stores) > 0 {
// The Pipelines chat tools' MCP endpoint, for harnesses on this machine.
hub.shown = stores[0]
registerChatTools(mux, newChatToolService(stores[0], stores[0], hub, registry), build.Version)
}
mux.HandleFunc("GET /discovery", serveServerDiscovery)
mux.HandleFunc("GET /files", serveHomeFile)
mux.HandleFunc("POST /files", serveHomeFileUpload)
mux.HandleFunc("POST /chat-uploads", serveChatUpload)
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if runErr != nil {
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = w.Write([]byte(`{"healthy":false}`))
return
}
_, _ = w.Write([]byte(`{"healthy":true}`))
})
connect := func(w http.ResponseWriter, r *http.Request) {
conn, err := websocket.Accept(w, r, nil)
if err != nil {
return
}
defer conn.CloseNow()
conn.SetReadLimit(2 << 20)
// connCtx is canceled when this connection's read loop returns, so a
// chat subscription started for the client stops on disconnect. A single
// write mutex serializes the read loop's responses and the chat stream
// goroutine, which must never interleave partial websocket writes.
connCtx, connCancel := context.WithCancel(ctx)
defer connCancel()
var writeMu sync.Mutex
write := func(message protocol.Message) error {
// Every server frame (responses, the welcome and chat events) carries
// the build identity and protocol so an older client fails fast or
// warns once instead of silently misreading a field. build is
// computed once per handler.
if message.Build == nil {
message.Build = &build
}
if message.Protocol == 0 {
message.Protocol = protocol.ProtocolVersion
}
writeMu.Lock()
defer writeMu.Unlock()
writeCtx, cancel := context.WithTimeout(connCtx, 5*time.Second)
defer cancel()
return wsjson.Write(writeCtx, conn, message)
}
// clientID is this connection's identity for chat session claims: a
// second client is refused a session another connection holds.
clientID := chatClientToken()
// Drop every run subscription this connection holds when the read loop
// returns, so no subscriber, goroutine or timer outlives the connection.
defer events.unsubscribeClient(clientID)
// Drop every chat subscription too, so a chat room held only by this
// connection releases its broker stream and turn runner.
defer hub.detachClient(clientID)
if len(stores) > 0 {
items, err := stores[0].List(ctx)
if err != nil {
log.Printf("list pipelines: %v", err)
return
}
if err := write(protocol.Message{Type: protocol.PipelineList, Pipelines: items, NotificationEndpoints: notificationEndpoints()}); err != nil {
return
}
}
// Shared attention acknowledgements reach every open connection, after
// the welcome frame so a push never precedes it.
acks.attach(clientID, write)
defer acks.detach(clientID)
// Shared chat-list metadata (colour, star, title) reaches every open
// connection the same way.
hub.sessionChanges.attach(clientID, write)
defer hub.sessionChanges.detach(clientID)
if notes != nil {
defer notes.detach(clientID)
}
// A chat search runs beside the read loop so its transcript readback
// never delays the run search sent with it; a newer one cancels the
// previous, whose reply the client would discard as stale anyway.
cancelChatSearch := func() {}
defer func() { cancelChatSearch() }()
for {
var request protocol.Message
if err := wsjson.Read(connCtx, conn, &request); err != nil {
return
}
log.Printf("command received: client=%s type=%q id=%q pipeline=%d version=%d", r.RemoteAddr, request.Type, request.ID, request.PipelineID, request.Version)
if request.Type == protocol.Submit && !options.AllowAuthoring {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "pipeline authoring is disabled on this execution server; ask the operator to publish through an explicitly enabled authoring server"})
continue
}
if isChatSearch(request) {
cancelChatSearch()
searchCtx, cancel := context.WithCancel(connCtx)
cancelChatSearch = cancel
go func(request protocol.Message) {
defer cancel()
_ = write(chatSearchReply(searchCtx, request, options.Chat, registry))
}(request)
continue
}
// Prevent a new operator workspace admission racing a worktree
// removal. Ordinary reads/prompts do not wait for storage scans.
admission := storage != nil && (request.Type == protocol.ChatCreate || request.Type == protocol.ChatAdopt)
if admission {
storage.workspaceMu.RLock()
}
handledChat := handleChat(connCtx, request, options.Chat, hub, registry, clientID, write)
if admission {
storage.workspaceMu.RUnlock()
}
if handledChat {
continue
}
if isStorageRequest(request.Type) {
if storage == nil {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "storage management unavailable: no execution store"})
} else {
// Disk/Git inventory must never stall chat permissions/prompts
// arriving on this same persistent connection.
storage.dispatch(connCtx, request, write)
}
continue
}
if handleDirectoryListing(request, write) {
continue
}
if handleHomeFileMutation(request, write) {
continue
}
if request.Type == protocol.DiscoverServersRequest {
go handleServerDiscovery(connCtx, request, write)
continue
}
if request.Type == protocol.ServerStatsRequest {
go handleServerStats(connCtx, request, runs, options.ServerStatsSampler, started, build, write)
continue
}
if handleNotificationSettings(ctx, request, settingsStore, write) {
continue
}
if handleAttentionAcks(ctx, request, ackStore, acks, notes, clientID, write) {
continue
}
if handleAppNotifications(ctx, request, notes, ackStore, acks, clientID, write) {
continue
}
if handleRunEvents(request, events, runs, clientID, write) {
continue
}
response := protocol.Message{Type: protocol.Pong, ID: request.ID}
if request.Type == protocol.List || request.Type == protocol.Submit {
response.NotificationEndpoints = notificationEndpoints()
}
if request.Type == protocol.GetStatus {
response.Type = protocol.Status
response.Status = systemStatus(r, started, runErr)
// How chat is attached to OpenCode is part of the server's status:
// a configured endpoint that could not be attached reports
// "fallback" here rather than silently disappearing.
response.Status.OpenCode = options.OpenCodeStatus
} else if request.Type == protocol.Ping {
response.Time = time.Now().UTC().Format(time.RFC3339)
} else if request.Type == protocol.ListTemplates {
response.Type = protocol.Templates
response.Templates = examples.Catalog()
model, backend := starterDefaults()
for i := range response.Templates {
response.Templates[i].Source = pipeline.WithDefaults(response.Templates[i].Source, model, backend)
}
} else if request.Type == protocol.Validate {
graph, warnings, err := pipeline.ParseDefinitionReport(request.Source)
response.Type = protocol.Validated
response.Warnings = warnings
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
} else {
response.Graph = &graph
}
} else if (request.Type == protocol.ListSchedules || request.Type == protocol.SaveSchedule || request.Type == protocol.DeleteSchedule) && len(stores) > 0 {
var err error
switch request.Type {
case protocol.ListSchedules:
response.Type = protocol.Schedules
if request.PipelineID < 0 {
err = fmt.Errorf("pipeline_id must be nonnegative")
} else {
response.Schedules, err = stores[0].ListSchedules(ctx, request.PipelineID)
}
case protocol.SaveSchedule:
response.Type = protocol.SavedScheduleReply
if request.Schedule == nil {
err = fmt.Errorf("schedule is required")
} else {
response.Schedule, err = stores[0].SaveSchedule(ctx, *request.Schedule, time.Now(), request.RequestKey)
}
case protocol.DeleteSchedule:
response.Type = protocol.ScheduleDeleted
response.ScheduleID = request.ScheduleID
err = stores[0].DeleteSchedule(ctx, request.ScheduleID, request.ExpectedRevision)
}
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
}
} else if request.Type == protocol.Submit && len(stores) > 0 {
response.Type = protocol.Submitted
response.Name = strings.TrimSpace(request.Name)
version, err := stores[0].PublishChecked(ctx, request.Name, request.Source, request.CreateOnly, request.ExpectedVersion)
_, response.Warnings, _ = pipeline.ParseDefinitionReport(request.Source)
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
} else {
response.Version = version
response.Pipelines, err = stores[0].List(ctx)
if err != nil {
log.Printf("list pipelines after submission: %v", err)
}
for _, item := range response.Pipelines {
if item.Name == response.Name {
response.Detail, err = stores[0].Get(ctx, item.ID, version)
if err != nil {
log.Printf("load submitted pipeline: %v", err)
}
break
}
}
}
} else if request.Type == protocol.GetPipeline && len(stores) > 0 {
response.Type = protocol.Pipeline
detail, err := stores[0].Get(ctx, request.PipelineID, request.Version)
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
} else {
response.Detail = detail
}
} else if request.Type == protocol.List && len(stores) > 0 {
response.Type = protocol.PipelineList
response.DefaultModel, response.DefaultHarness = starterDefaults()
items, err := stores[0].ListPipelines(ctx, request.IncludeArchived)
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
} else {
response.Pipelines = items
}
} else if (request.Type == protocol.ArchivePipeline || request.Type == protocol.UnarchivePipeline) && len(stores) > 0 {
detail, err := archivePipeline(ctx, stores[0], request)
response.Detail = detail
response.Type = protocol.PipelineArchived
if request.Type == protocol.UnarchivePipeline {
response.Type = protocol.PipelineUnarchived
}
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
}
} else if request.Type == protocol.Search {
if len(stores) == 0 {
response.Type = protocol.Error
response.Error = "search unavailable: no store"
} else if request.Search == nil {
response.Type = protocol.Error
response.Error = "search requires a query"
} else {
response.Type = protocol.SearchResults
var err error
response.SearchResults, err = stores[0].Search(ctx, *request.Search, time.Now())
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
}
}
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence || request.Type == protocol.RepairSessionExport || request.Type == protocol.DeleteRuns) {
var err error
switch request.Type {
case protocol.DeleteRuns:
// Like cancel and retry, deleting run history is an execution
// operation and is not gated by AllowAuthoring.
response.Type = protocol.RunsDeleted
response.RunsDeleted, err = runs.deleteRuns(request.RunIDs)
case protocol.RepairSessionExport:
response.Type = protocol.Run
response.RunDetail, err = runs.repairSessionExport(ctx, request.RunID, request.ExportSequence)
case protocol.GetRunEvidence:
response.Type = protocol.RunEvidence
response.Evidence, err = runs.evidence(ctx, request.RunID)
case protocol.RunPipeline:
response.Type = protocol.RunStarted
response.RunDetail, err = runs.start(request)
case protocol.RetryRun:
response.Type = protocol.RunStarted
response.RunDetail, err = runs.retry(request)
case protocol.DecideRun:
response.Type = protocol.Run
response.RunDetail, err = runs.decide(request)
case protocol.SetRunTitle:
response.Type = protocol.Run
response.RunDetail, err = runs.setTitle(request.RunID, request.Title)
case protocol.ListRuns:
response.Type = protocol.Runs
runs.refreshListSizes(connCtx)
response.Runs, err = runs.listRuns()
if err == nil {
response.Usage, err = stores[0].UsageStats(ctx)
}
case protocol.GetRun:
response.Type = protocol.Run
if request.RunID == 0 && request.RequestKey != "" {
var id int64
id, err = stores[0].RunByKey(connCtx, request.RequestKey)
if err == nil && id != 0 {
response.RunDetail, err = runs.getRun(id)
}
} else {
response.RunDetail, err = runs.getRun(request.RunID)
}
case protocol.CancelRun:
response.Type = protocol.Run
err = runs.cancel(request.RunID)
if err == nil {
response.RunDetail, err = runs.getRun(request.RunID)
}
}
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
}
} else {
response.Type = protocol.Error
response.Error = "unknown command: " + request.Type
if runErr != nil {
response.Error = "execution unavailable: " + runErr.Error()
}
}
if err := write(response); err != nil {
log.Printf("write response: %v", err)
return
}
selectedVersion := response.Version
if response.Detail != nil {
selectedVersion = response.Detail.Version
}
log.Printf("response sent: client=%s type=%q id=%q time=%q error=%q version=%d", r.RemoteAddr, response.Type, response.ID, response.Time, response.Error, selectedVersion)
}
}
mux.HandleFunc("GET /connect", connect)
if options.Chat != nil && registry != nil {
// Reattach the chats a previous process left live (and fold in turns
// that ended while it was down), then keep observing any other chat
// whose turn is still running, so a reply or question still notifies
// with no client attached.
done := make(chan struct{})
ready := make(chan struct{})
hub.recoveryDone = done
hub.recoveryReady = ready
go func() {
hub.reattachSessions(ctx, options.Chat)
// Restore live turn ownership before queue rooms can start runners.
hub.resumeQueued(ctx, options.Chat)
close(ready)
hub.observeRunningTurns(ctx, options.Chat)
close(done)
hub.retryReattachments(ctx, options.Chat)
}()
}
if runs != nil {
// Continue the runs a previous process left in flight, now that the
// runner's collaborators are wired and before any request is served.
runs.reattachRuns()
}
return &runHandler{Handler: mux, runner: runs, hub: hub}
}
type runHandler struct {
http.Handler
runner *runner
hub *chatHub
}
// DetachSessions records the chats left live on their harness so the next
// start reattaches them. Call it when the server is stopping, before the
// handler's context is canceled (rooms are torn down after that) and before the
// store closes. It never interrupts a session.
func DetachSessions(handler http.Handler) {
h, ok := handler.(*runHandler)
if !ok || h.hub == nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := h.hub.detachSessions(ctx); err != nil {
log.Printf("record live chat sessions: %v", err)
}
}
// WaitRuns drains active execution before the database is closed on shutdown.
func WaitRuns(handler http.Handler) {
if h, ok := handler.(*runHandler); ok && h.runner != nil {
h.runner.schedulerWG.Wait()
h.runner.wg.Wait()
}
}