package server import ( "context" "fmt" "log" "net/http" "strings" "sync" "time" "github.com/coder/websocket" "github.com/coder/websocket/wsjson" "git.a-wan.net/alan/pipelines/examples" "git.a-wan.net/alan/pipelines/internal/buildinfo" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/summarise" ) // Options configures the HTTP handler. type Options struct { // ServerStatsSampler overrides host sampling for deterministic tests. ServerStatsSampler ServerStatsSampler // AllowAuthoring permits publishing immutable definitions. Execution servers // default to run-only: a CLI --yes flag or raw RPC cannot grant this server // capability. Enable deliberately on a separate trusted authoring instance. AllowAuthoring bool // ArtifactsRoot overrides where stage artifacts live. Empty uses the store's // directory (the database's parent), matching the data-directory default. ArtifactsRoot string // Chat brokers OpenCode chat traffic over /connect. Nil answers every // chat_* request with a protocol error; a configured server passes an // *opencode.Chat bound to the owned endpoint. Chat chatBroker // OpenCodeStatus is the honest report of how chat is attached to OpenCode, // echoed on get_status: "endpoint" when the broker is attached to the // operator's configured endpoint, "fallback" when that endpoint could not be // attached and chat fell back to the server's own instance, "spawned" when // no endpoint is configured, and "unavailable" when chat is disabled. It // carries the endpoint URL but never a credential. OpenCodeStatus protocol.ServiceStatus // OperatorRegistry is the operator-session registry chat_sessions lists and // chat_create/chat_directories read. When nil, HandlerWithOptions derives it // from the first store; when neither is available chat_sessions returns an // empty list (never the endpoint's whole session list) and chat_create is // refused. Purely pipeline stage ownership is unaffected. OperatorRegistry operatorRegistry // RunTitles generates the operator-facing title stored with each new run. // When nil, run creation stores the prompt-derived fallback without a model // call, which keeps tests and unconfigured servers deterministic. RunTitles func(context.Context, string) summarise.Title // StepVerdicts generates the bounded one-line reason stored with each // completed stage visit. When nil, the store derives the verdict's reason at // read time with no model call, which keeps tests and unconfigured servers // deterministic. StepVerdicts func(context.Context, map[string]any) string // VerificationFixtures are explicit server-owned HTTP fixtures for optional // target-aware verification. Definitions can name only a registered ID; they // cannot select an endpoint or supply provenance. VerificationFixtures []VerificationFixture // DetachSessions keeps harness sessions running across a server restart: // a stopping server leaves stage and chat sessions running (DetachSessions // on the handler records the chats) and the next start reattaches them. // False keeps the previous behaviour (tests, embedded servers): a stopping // server interrupts its runs and the next start marks them interrupted. DetachSessions bool // AccessToken, when set, is required on every request (servertoken.Header) // except the chat-tools MCP endpoint and /healthz (request_boundary.go). AccessToken string // WebAuthFile is the web sign-in password file step-up re-authentication // verifies against (see elevation.go); empty disables the gate. // ElevationTTL caps how long one confirmation lasts (default 24 hours). WebAuthFile string ElevationTTL time.Duration } // Handler serves the persistent connection with default options. Canceling ctx // stops active clients. func Handler(ctx context.Context, stores ...*pipeline.Store) http.Handler { return HandlerWithOptions(ctx, Options{}, stores...) } // HandlerWithOptions serves the persistent connection and the stage-artifact // endpoint. Canceling ctx stops active clients. func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipeline.Store) http.Handler { build := buildinfo.Current() started := time.Now() hub := newChatHub() events := newRunEventHub() mux := http.NewServeMux() var runs *runner var runErr error if len(stores) > 0 { runs, runErr = newRunnerReattaching(ctx, stores[0], options.ArtifactsRoot, options.DetachSessions) if runErr == nil { runs.events = events } if runErr == nil && options.RunTitles != nil { runs.title = options.RunTitles } if runErr == nil && options.StepVerdicts != nil { runs.verdict = options.StepVerdicts } if runErr == nil { fixtures := newVerificationFixtureRegistry(options.VerificationFixtures) runs.verifyTarget = fixtures.verify } if runErr == nil { runs.schedulerWG.Add(1) go runs.scheduleLoop() // The run-size backfill runs in the background so an upgraded // database never delays startup. runs.sizeKick = make(chan struct{}, 1) runs.schedulerWG.Add(1) go runs.sizeLoop() } } var storage *storageManager if runs != nil { storage = newStorageManager(runs) runs.schedulerWG.Add(1) go storage.loop(ctx) } // chat_sessions lists the operator's own recorded sessions. Prefer an // injected registry; otherwise derive it from the store. With neither, the // handler fails closed to an empty list. registry := options.OperatorRegistry if registry == nil && len(stores) > 0 { registry = stores[0] } hub.baseCtx = ctx hub.registry = registry var settingsStore notificationSettingsStore if len(stores) > 0 { settingsStore = stores[0] } hub.settings = settingsStore if len(stores) > 0 { // Prompts queued behind a running turn are durable and owned by their // session; resume delivering any a previous process left queued. hub.queueStore = stores[0] hub.liveStore = stores[0] hub.turnDiffStore = stores[0] } // Operator shell commands persist beside the operator sessions they belong // to: the execution store, else an injected store-backed registry. if len(stores) > 0 { hub.startShellRunner(ctx, stores[0]) } else if store, ok := registry.(chatShellRunStore); ok { hub.startShellRunner(ctx, store) } // Machine terminals are recorded beside the operator sessions, in the // same store; the ones a previous process left running are listed as // ended. var terminalRecords terminalStore if len(stores) > 0 { terminalRecords = stores[0] } else if store, ok := registry.(terminalStore); ok { terminalRecords = store } terminals := newTerminalManager(ctx, terminalRecords) terminals.recover(ctx) acks := newAttentionAckHub() var ackStore attentionAckStore var notes *appNotifier if len(stores) > 0 { ackStore = stores[0] notes = newAppNotifier(stores[0]) hub.notes = notes if runs != nil { runs.notes = notes } } mux.HandleFunc("GET /artifacts/{run}/{step}/{name...}", func(w http.ResponseWriter, r *http.Request) { if runs == nil { http.NotFound(w, r) return } runs.serveArtifact(w, r) }) if len(stores) > 0 { // The Pipelines chat tools' MCP endpoint, for harnesses on this machine. hub.shown = stores[0] registerChatTools(mux, newChatToolService(stores[0], stores[0], hub, registry), build.Version) } mux.HandleFunc("GET /discovery", serveServerDiscovery) mux.HandleFunc("GET /files", serveHomeFile) elevation := newElevationGate(options.WebAuthFile, options.ElevationTTL) mux.HandleFunc("POST /files", elevation.requireElevationHTTP(serveHomeFileUpload)) mux.HandleFunc("POST /chat-uploads", serveChatUpload) mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") if runErr != nil { w.WriteHeader(http.StatusServiceUnavailable) _, _ = w.Write([]byte(`{"healthy":false}`)) return } _, _ = w.Write([]byte(`{"healthy":true}`)) }) connect := func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) if err != nil { return } defer conn.CloseNow() conn.SetReadLimit(2 << 20) // connCtx is canceled when this connection's read loop returns, so a // chat subscription started for the client stops on disconnect. A single // write mutex serializes the read loop's responses and the chat stream // goroutine, which must never interleave partial websocket writes. connCtx, connCancel := context.WithCancel(ctx) defer connCancel() var writeMu sync.Mutex write := func(message protocol.Message) error { // Every server frame (responses, the welcome and chat events) carries // the build identity and protocol so an older client fails fast or // warns once instead of silently misreading a field. build is // computed once per handler. if message.Build == nil { message.Build = &build } if message.Protocol == 0 { message.Protocol = protocol.ProtocolVersion } writeMu.Lock() defer writeMu.Unlock() writeCtx, cancel := context.WithTimeout(connCtx, 5*time.Second) defer cancel() return wsjson.Write(writeCtx, conn, message) } // clientID is this connection's identity for chat session claims: a // second client is refused a session another connection holds. clientID := chatClientToken() // Elevation belongs to the web session the proxy reported, so a // reconnect stays elevated; without one, to this connection alone. elevationKey := elevationKeyFor(trustedSession(r), clientID) defer elevation.closed(elevationKey) // Drop every run subscription this connection holds when the read loop // returns, so no subscriber, goroutine or timer outlives the connection. defer events.unsubscribeClient(clientID) // Drop every chat subscription too, so a chat room held only by this // connection releases its broker stream and turn runner. defer hub.detachClient(clientID) if len(stores) > 0 { items, err := stores[0].List(ctx) if err != nil { log.Printf("list pipelines: %v", err) return } if err := write(protocol.Message{Type: protocol.PipelineList, Pipelines: items, NotificationEndpoints: notificationEndpoints()}); err != nil { return } } // Shared attention acknowledgements reach every open connection, after // the welcome frame so a push never precedes it. acks.attach(clientID, write) defer acks.detach(clientID) // Shared chat-list metadata (colour, star, title) reaches every open // connection the same way. hub.sessionChanges.attach(clientID, write) defer hub.sessionChanges.detach(clientID) // Terminal lifecycle changes reach every connection; output only the // connections that opened or read that terminal. terminals.clients.attach(clientID, write) defer terminals.clients.detach(clientID) if notes != nil { defer notes.detach(clientID) } // A chat search runs beside the read loop so its transcript readback // never delays the run search sent with it; a newer one cancels the // previous, whose reply the client would discard as stale anyway. cancelChatSearch := func() {} defer func() { cancelChatSearch() }() for { var request protocol.Message if err := wsjson.Read(connCtx, conn, &request); err != nil { return } if request.Type != protocol.ChatShellInput && request.Type != protocol.ChatShellResize && request.Type != protocol.TerminalInput && request.Type != protocol.TerminalResize { // Keystrokes would be one line each; their data is never logged. log.Printf("command received: client=%s type=%q id=%q pipeline=%d version=%d", r.RemoteAddr, request.Type, request.ID, request.PipelineID, request.Version) } if request.Type == protocol.Submit && !options.AllowAuthoring { _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "pipeline authoring is disabled on this execution server; ask the operator to publish through an explicitly enabled authoring server"}) continue } if elevation.handle(request, elevationKey, write) || elevation.refuse(request, elevationKey, write) { continue } if isChatSearch(request) { cancelChatSearch() searchCtx, cancel := context.WithCancel(connCtx) cancelChatSearch = cancel go func(request protocol.Message) { defer cancel() _ = write(chatSearchReply(searchCtx, request, options.Chat, registry)) }(request) continue } // Prevent a new operator workspace admission racing a worktree // removal. Ordinary reads/prompts do not wait for storage scans. admission := storage != nil && (request.Type == protocol.ChatCreate || request.Type == protocol.ChatAdopt) if admission { storage.workspaceMu.RLock() } handledChat := handleChat(connCtx, request, options.Chat, hub, registry, clientID, write) if admission { storage.workspaceMu.RUnlock() } if handledChat { continue } if isStorageRequest(request.Type) { if storage == nil { _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "storage management unavailable: no execution store"}) } else { // Disk/Git inventory must never stall chat permissions/prompts // arriving on this same persistent connection. storage.dispatch(connCtx, request, write) } continue } if isTerminalRequest(request.Type) { // Closing waits for the shell to go; it never delays input // to the other terminals behind it. if request.Type == protocol.TerminalClose { go handleTerminalRequest(connCtx, request, terminals, clientID, write) } else { handleTerminalRequest(connCtx, request, terminals, clientID, write) } continue } if handleDirectoryListing(request, write) { continue } if handleHomeFileMutation(request, write) { continue } if request.Type == protocol.DiscoverServersRequest { go handleServerDiscovery(connCtx, request, write) continue } if request.Type == protocol.ServerStatsRequest { go handleServerStats(connCtx, request, runs, options.ServerStatsSampler, started, build, write) continue } if handleNotificationSettings(ctx, request, settingsStore, write) { continue } if handleAttentionAcks(ctx, request, ackStore, acks, notes, clientID, write) { continue } if handleAppNotifications(ctx, request, notes, ackStore, acks, clientID, write) { continue } if handleRunEvents(request, events, runs, clientID, write) { continue } response := protocol.Message{Type: protocol.Pong, ID: request.ID} if request.Type == protocol.List || request.Type == protocol.Submit { response.NotificationEndpoints = notificationEndpoints() } if request.Type == protocol.GetStatus { response.Type = protocol.Status response.Status = systemStatus(r, started, runErr) // How chat is attached to OpenCode is part of the server's status: // a configured endpoint that could not be attached reports // "fallback" here rather than silently disappearing. response.Status.OpenCode = options.OpenCodeStatus } else if request.Type == protocol.Ping { response.Time = time.Now().UTC().Format(time.RFC3339) } else if request.Type == protocol.ListTemplates { response.Type = protocol.Templates response.Templates = examples.Catalog() model, backend := starterDefaults() for i := range response.Templates { response.Templates[i].Source = pipeline.WithDefaults(response.Templates[i].Source, model, backend) } } else if request.Type == protocol.Validate { graph, warnings, err := pipeline.ParseDefinitionReport(request.Source) response.Type = protocol.Validated response.Warnings = warnings if err != nil { response.Type = protocol.Error response.Error = err.Error() } else { response.Graph = &graph } } else if (request.Type == protocol.ListSchedules || request.Type == protocol.SaveSchedule || request.Type == protocol.DeleteSchedule) && len(stores) > 0 { var err error switch request.Type { case protocol.ListSchedules: response.Type = protocol.Schedules if request.PipelineID < 0 { err = fmt.Errorf("pipeline_id must be nonnegative") } else { response.Schedules, err = stores[0].ListSchedules(ctx, request.PipelineID) } case protocol.SaveSchedule: response.Type = protocol.SavedScheduleReply if request.Schedule == nil { err = fmt.Errorf("schedule is required") } else { response.Schedule, err = stores[0].SaveSchedule(ctx, *request.Schedule, time.Now(), request.RequestKey) } case protocol.DeleteSchedule: response.Type = protocol.ScheduleDeleted response.ScheduleID = request.ScheduleID err = stores[0].DeleteSchedule(ctx, request.ScheduleID, request.ExpectedRevision) } if err != nil { response.Type = protocol.Error response.Error = err.Error() } } else if request.Type == protocol.Submit && len(stores) > 0 { response.Type = protocol.Submitted response.Name = strings.TrimSpace(request.Name) version, err := stores[0].PublishChecked(ctx, request.Name, request.Source, request.CreateOnly, request.ExpectedVersion) _, response.Warnings, _ = pipeline.ParseDefinitionReport(request.Source) if err != nil { response.Type = protocol.Error response.Error = err.Error() } else { response.Version = version response.Pipelines, err = stores[0].List(ctx) if err != nil { log.Printf("list pipelines after submission: %v", err) } for _, item := range response.Pipelines { if item.Name == response.Name { response.Detail, err = stores[0].Get(ctx, item.ID, version) if err != nil { log.Printf("load submitted pipeline: %v", err) } break } } } } else if request.Type == protocol.GetPipeline && len(stores) > 0 { response.Type = protocol.Pipeline detail, err := stores[0].Get(ctx, request.PipelineID, request.Version) if err != nil { response.Type = protocol.Error response.Error = err.Error() } else { response.Detail = detail } } else if request.Type == protocol.List && len(stores) > 0 { response.Type = protocol.PipelineList response.DefaultModel, response.DefaultHarness = starterDefaults() items, err := stores[0].ListPipelines(ctx, request.IncludeArchived) if err != nil { response.Type = protocol.Error response.Error = err.Error() } else { response.Pipelines = items } } else if (request.Type == protocol.ArchivePipeline || request.Type == protocol.UnarchivePipeline) && len(stores) > 0 { detail, err := archivePipeline(ctx, stores[0], request) response.Detail = detail response.Type = protocol.PipelineArchived if request.Type == protocol.UnarchivePipeline { response.Type = protocol.PipelineUnarchived } if err != nil { response.Type = protocol.Error response.Error = err.Error() } } else if request.Type == protocol.Search { if len(stores) == 0 { response.Type = protocol.Error response.Error = "search unavailable: no store" } else if request.Search == nil { response.Type = protocol.Error response.Error = "search requires a query" } else { response.Type = protocol.SearchResults var err error response.SearchResults, err = stores[0].Search(ctx, *request.Search, time.Now()) if err != nil { response.Type = protocol.Error response.Error = err.Error() } } } else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence || request.Type == protocol.RepairSessionExport || request.Type == protocol.DeleteRuns) { var err error switch request.Type { case protocol.DeleteRuns: // Like cancel and retry, deleting run history is an execution // operation and is not gated by AllowAuthoring. response.Type = protocol.RunsDeleted response.RunsDeleted, err = runs.deleteRuns(request.RunIDs) case protocol.RepairSessionExport: response.Type = protocol.Run response.RunDetail, err = runs.repairSessionExport(ctx, request.RunID, request.ExportSequence) case protocol.GetRunEvidence: response.Type = protocol.RunEvidence response.Evidence, err = runs.evidence(ctx, request.RunID) case protocol.RunPipeline: response.Type = protocol.RunStarted response.RunDetail, err = runs.start(request) case protocol.RetryRun: response.Type = protocol.RunStarted response.RunDetail, err = runs.retry(request) case protocol.DecideRun: response.Type = protocol.Run response.RunDetail, err = runs.decide(request) case protocol.SetRunTitle: response.Type = protocol.Run response.RunDetail, err = runs.setTitle(request.RunID, request.Title) case protocol.ListRuns: response.Type = protocol.Runs runs.refreshListSizes(connCtx) response.Runs, err = runs.listRuns() if err == nil { response.Usage, err = stores[0].UsageStats(ctx) } case protocol.GetRun: response.Type = protocol.Run if request.RunID == 0 && request.RequestKey != "" { var id int64 id, err = stores[0].RunByKey(connCtx, request.RequestKey) if err == nil && id != 0 { response.RunDetail, err = runs.getRun(id) } } else { response.RunDetail, err = runs.getRun(request.RunID) } case protocol.CancelRun: response.Type = protocol.Run err = runs.cancel(request.RunID) if err == nil { response.RunDetail, err = runs.getRun(request.RunID) } } if err != nil { response.Type = protocol.Error response.Error = err.Error() } } else { response.Type = protocol.Error response.Error = "unknown command: " + request.Type if runErr != nil { response.Error = "execution unavailable: " + runErr.Error() } } if err := write(response); err != nil { log.Printf("write response: %v", err) return } selectedVersion := response.Version if response.Detail != nil { selectedVersion = response.Detail.Version } log.Printf("response sent: client=%s type=%q id=%q time=%q error=%q version=%d", r.RemoteAddr, response.Type, response.ID, response.Time, response.Error, selectedVersion) } } mux.HandleFunc("GET /connect", connect) if options.Chat != nil && registry != nil { // Reattach the chats a previous process left live (and fold in turns // that ended while it was down), then keep observing any other chat // whose turn is still running, so a reply or question still notifies // with no client attached. done := make(chan struct{}) ready := make(chan struct{}) hub.recoveryDone = done hub.recoveryReady = ready go func() { hub.reattachSessions(ctx, options.Chat) // Restore live turn ownership before queue rooms can start runners. hub.resumeQueued(ctx, options.Chat) close(ready) hub.observeRunningTurns(ctx, options.Chat) close(done) hub.retryReattachments(ctx, options.Chat) }() } if runs != nil { // Continue the runs a previous process left in flight, now that the // runner's collaborators are wired and before any request is served. runs.reattachRuns() } return &runHandler{Handler: guardRequests(mux, options.AccessToken), runner: runs, hub: hub} } type runHandler struct { http.Handler runner *runner hub *chatHub } // DetachSessions records the chats left live on their harness so the next // start reattaches them. Call it when the server is stopping, before the // handler's context is canceled (rooms are torn down after that) and before the // store closes. It never interrupts a session. func DetachSessions(handler http.Handler) { h, ok := handler.(*runHandler) if !ok || h.hub == nil { return } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := h.hub.detachSessions(ctx); err != nil { log.Printf("record live chat sessions: %v", err) } } // WaitRuns drains active execution before the database is closed on shutdown. func WaitRuns(handler http.Handler) { if h, ok := handler.(*runHandler); ok && h.runner != nil { h.runner.schedulerWG.Wait() h.runner.wg.Wait() } }