50535dc5ea
# Conflicts: # cmd/pipelines-server/main.go # internal/server/websocket.go # scripts/manage.py # scripts/setup-native.py # web/serve.mjs # web/src/peerRelay.js
615 lines
24 KiB
Go
615 lines
24 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/coder/websocket"
|
|
"github.com/coder/websocket/wsjson"
|
|
|
|
"git.a-wan.net/alan/pipelines/examples"
|
|
"git.a-wan.net/alan/pipelines/internal/buildinfo"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/summarise"
|
|
)
|
|
|
|
// Options configures the HTTP handler.
|
|
type Options struct {
|
|
// ServerStatsSampler overrides host sampling for deterministic tests.
|
|
ServerStatsSampler ServerStatsSampler
|
|
// AllowAuthoring permits publishing immutable definitions. Execution servers
|
|
// default to run-only: a CLI --yes flag or raw RPC cannot grant this server
|
|
// capability. Enable deliberately on a separate trusted authoring instance.
|
|
AllowAuthoring bool
|
|
// ArtifactsRoot overrides where stage artifacts live. Empty uses the store's
|
|
// directory (the database's parent), matching the data-directory default.
|
|
ArtifactsRoot string
|
|
// Chat brokers OpenCode chat traffic over /connect. Nil answers every
|
|
// chat_* request with a protocol error; a configured server passes an
|
|
// *opencode.Chat bound to the owned endpoint.
|
|
Chat chatBroker
|
|
// OpenCodeStatus is the honest report of how chat is attached to OpenCode,
|
|
// echoed on get_status: "endpoint" when the broker is attached to the
|
|
// operator's configured endpoint, "fallback" when that endpoint could not be
|
|
// attached and chat fell back to the server's own instance, "spawned" when
|
|
// no endpoint is configured, and "unavailable" when chat is disabled. It
|
|
// carries the endpoint URL but never a credential.
|
|
OpenCodeStatus protocol.ServiceStatus
|
|
// OperatorRegistry is the operator-session registry chat_sessions lists and
|
|
// chat_create/chat_directories read. When nil, HandlerWithOptions derives it
|
|
// from the first store; when neither is available chat_sessions returns an
|
|
// empty list (never the endpoint's whole session list) and chat_create is
|
|
// refused. Purely pipeline stage ownership is unaffected.
|
|
OperatorRegistry operatorRegistry
|
|
// RunTitles generates the operator-facing title stored with each new run.
|
|
// When nil, run creation stores the prompt-derived fallback without a model
|
|
// call, which keeps tests and unconfigured servers deterministic.
|
|
RunTitles func(context.Context, string) summarise.Title
|
|
// StepVerdicts generates the bounded one-line reason stored with each
|
|
// completed stage visit. When nil, the store derives the verdict's reason at
|
|
// read time with no model call, which keeps tests and unconfigured servers
|
|
// deterministic.
|
|
StepVerdicts func(context.Context, map[string]any) string
|
|
// VerificationFixtures are explicit server-owned HTTP fixtures for optional
|
|
// target-aware verification. Definitions can name only a registered ID; they
|
|
// cannot select an endpoint or supply provenance.
|
|
VerificationFixtures []VerificationFixture
|
|
// DetachSessions keeps harness sessions running across a server restart:
|
|
// a stopping server leaves stage and chat sessions running (DetachSessions
|
|
// on the handler records the chats) and the next start reattaches them.
|
|
// False keeps the previous behaviour (tests, embedded servers): a stopping
|
|
// server interrupts its runs and the next start marks them interrupted.
|
|
DetachSessions bool
|
|
// AccessToken, when set, is required on every request (servertoken.Header)
|
|
// except the chat-tools MCP endpoint and /healthz (request_boundary.go).
|
|
AccessToken string
|
|
// WebAuthFile is the web sign-in password file step-up re-authentication
|
|
// verifies against (see elevation.go); empty disables the gate.
|
|
// ElevationTTL is how long one confirmation lasts (default 15 minutes).
|
|
WebAuthFile string
|
|
ElevationTTL time.Duration
|
|
}
|
|
|
|
// Handler serves the persistent connection with default options. Canceling ctx
|
|
// stops active clients.
|
|
func Handler(ctx context.Context, stores ...*pipeline.Store) http.Handler {
|
|
return HandlerWithOptions(ctx, Options{}, stores...)
|
|
}
|
|
|
|
// HandlerWithOptions serves the persistent connection and the stage-artifact
|
|
// endpoint. Canceling ctx stops active clients.
|
|
func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipeline.Store) http.Handler {
|
|
build := buildinfo.Current()
|
|
started := time.Now()
|
|
hub := newChatHub()
|
|
events := newRunEventHub()
|
|
mux := http.NewServeMux()
|
|
var runs *runner
|
|
var runErr error
|
|
if len(stores) > 0 {
|
|
runs, runErr = newRunnerReattaching(ctx, stores[0], options.ArtifactsRoot, options.DetachSessions)
|
|
if runErr == nil {
|
|
runs.events = events
|
|
}
|
|
if runErr == nil && options.RunTitles != nil {
|
|
runs.title = options.RunTitles
|
|
}
|
|
if runErr == nil && options.StepVerdicts != nil {
|
|
runs.verdict = options.StepVerdicts
|
|
}
|
|
if runErr == nil {
|
|
fixtures := newVerificationFixtureRegistry(options.VerificationFixtures)
|
|
runs.verifyTarget = fixtures.verify
|
|
}
|
|
if runErr == nil {
|
|
runs.schedulerWG.Add(1)
|
|
go runs.scheduleLoop()
|
|
// The run-size backfill runs in the background so an upgraded
|
|
// database never delays startup.
|
|
runs.sizeKick = make(chan struct{}, 1)
|
|
runs.schedulerWG.Add(1)
|
|
go runs.sizeLoop()
|
|
}
|
|
}
|
|
var storage *storageManager
|
|
if runs != nil {
|
|
storage = newStorageManager(runs)
|
|
runs.schedulerWG.Add(1)
|
|
go storage.loop(ctx)
|
|
}
|
|
// chat_sessions lists the operator's own recorded sessions. Prefer an
|
|
// injected registry; otherwise derive it from the store. With neither, the
|
|
// handler fails closed to an empty list.
|
|
registry := options.OperatorRegistry
|
|
if registry == nil && len(stores) > 0 {
|
|
registry = stores[0]
|
|
}
|
|
hub.baseCtx = ctx
|
|
hub.registry = registry
|
|
var settingsStore notificationSettingsStore
|
|
if len(stores) > 0 {
|
|
settingsStore = stores[0]
|
|
}
|
|
hub.settings = settingsStore
|
|
if len(stores) > 0 {
|
|
// Prompts queued behind a running turn are durable and owned by their
|
|
// session; resume delivering any a previous process left queued.
|
|
hub.queueStore = stores[0]
|
|
hub.liveStore = stores[0]
|
|
hub.turnDiffStore = stores[0]
|
|
}
|
|
// Operator shell commands persist beside the operator sessions they belong
|
|
// to: the execution store, else an injected store-backed registry.
|
|
if len(stores) > 0 {
|
|
hub.startShellRunner(ctx, stores[0])
|
|
} else if store, ok := registry.(chatShellRunStore); ok {
|
|
hub.startShellRunner(ctx, store)
|
|
}
|
|
// Machine terminals are recorded beside the operator sessions, in the
|
|
// same store; the ones a previous process left running are listed as
|
|
// ended.
|
|
var terminalRecords terminalStore
|
|
if len(stores) > 0 {
|
|
terminalRecords = stores[0]
|
|
} else if store, ok := registry.(terminalStore); ok {
|
|
terminalRecords = store
|
|
}
|
|
terminals := newTerminalManager(ctx, terminalRecords)
|
|
terminals.recover(ctx)
|
|
acks := newAttentionAckHub()
|
|
var ackStore attentionAckStore
|
|
var notes *appNotifier
|
|
if len(stores) > 0 {
|
|
ackStore = stores[0]
|
|
notes = newAppNotifier(stores[0])
|
|
hub.notes = notes
|
|
if runs != nil {
|
|
runs.notes = notes
|
|
}
|
|
}
|
|
mux.HandleFunc("GET /artifacts/{run}/{step}/{name...}", func(w http.ResponseWriter, r *http.Request) {
|
|
if runs == nil {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
runs.serveArtifact(w, r)
|
|
})
|
|
if len(stores) > 0 {
|
|
// The Pipelines chat tools' MCP endpoint, for harnesses on this machine.
|
|
hub.shown = stores[0]
|
|
registerChatTools(mux, newChatToolService(stores[0], stores[0], hub, registry), build.Version)
|
|
}
|
|
mux.HandleFunc("GET /discovery", serveServerDiscovery)
|
|
mux.HandleFunc("GET /files", serveHomeFile)
|
|
elevation := newElevationGate(options.WebAuthFile, options.ElevationTTL)
|
|
mux.HandleFunc("POST /files", elevation.requireElevationHTTP(serveHomeFileUpload))
|
|
mux.HandleFunc("POST /chat-uploads", serveChatUpload)
|
|
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
if runErr != nil {
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
_, _ = w.Write([]byte(`{"healthy":false}`))
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte(`{"healthy":true}`))
|
|
})
|
|
connect := func(w http.ResponseWriter, r *http.Request) {
|
|
conn, err := websocket.Accept(w, r, nil)
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer conn.CloseNow()
|
|
conn.SetReadLimit(2 << 20)
|
|
// connCtx is canceled when this connection's read loop returns, so a
|
|
// chat subscription started for the client stops on disconnect. A single
|
|
// write mutex serializes the read loop's responses and the chat stream
|
|
// goroutine, which must never interleave partial websocket writes.
|
|
connCtx, connCancel := context.WithCancel(ctx)
|
|
defer connCancel()
|
|
var writeMu sync.Mutex
|
|
write := func(message protocol.Message) error {
|
|
// Every server frame (responses, the welcome and chat events) carries
|
|
// the build identity and protocol so an older client fails fast or
|
|
// warns once instead of silently misreading a field. build is
|
|
// computed once per handler.
|
|
if message.Build == nil {
|
|
message.Build = &build
|
|
}
|
|
if message.Protocol == 0 {
|
|
message.Protocol = protocol.ProtocolVersion
|
|
}
|
|
writeMu.Lock()
|
|
defer writeMu.Unlock()
|
|
writeCtx, cancel := context.WithTimeout(connCtx, 5*time.Second)
|
|
defer cancel()
|
|
return wsjson.Write(writeCtx, conn, message)
|
|
}
|
|
// clientID is this connection's identity for chat session claims: a
|
|
// second client is refused a session another connection holds.
|
|
clientID := chatClientToken()
|
|
defer elevation.forget(clientID)
|
|
// Drop every run subscription this connection holds when the read loop
|
|
// returns, so no subscriber, goroutine or timer outlives the connection.
|
|
defer events.unsubscribeClient(clientID)
|
|
// Drop every chat subscription too, so a chat room held only by this
|
|
// connection releases its broker stream and turn runner.
|
|
defer hub.detachClient(clientID)
|
|
if len(stores) > 0 {
|
|
items, err := stores[0].List(ctx)
|
|
if err != nil {
|
|
log.Printf("list pipelines: %v", err)
|
|
return
|
|
}
|
|
if err := write(protocol.Message{Type: protocol.PipelineList, Pipelines: items, NotificationEndpoints: notificationEndpoints()}); err != nil {
|
|
return
|
|
}
|
|
}
|
|
// Shared attention acknowledgements reach every open connection, after
|
|
// the welcome frame so a push never precedes it.
|
|
acks.attach(clientID, write)
|
|
defer acks.detach(clientID)
|
|
// Shared chat-list metadata (colour, star, title) reaches every open
|
|
// connection the same way.
|
|
hub.sessionChanges.attach(clientID, write)
|
|
defer hub.sessionChanges.detach(clientID)
|
|
// Terminal lifecycle changes reach every connection; output only the
|
|
// connections that opened or read that terminal.
|
|
terminals.clients.attach(clientID, write)
|
|
defer terminals.clients.detach(clientID)
|
|
if notes != nil {
|
|
defer notes.detach(clientID)
|
|
}
|
|
// A chat search runs beside the read loop so its transcript readback
|
|
// never delays the run search sent with it; a newer one cancels the
|
|
// previous, whose reply the client would discard as stale anyway.
|
|
cancelChatSearch := func() {}
|
|
defer func() { cancelChatSearch() }()
|
|
for {
|
|
var request protocol.Message
|
|
if err := wsjson.Read(connCtx, conn, &request); err != nil {
|
|
return
|
|
}
|
|
if request.Type != protocol.ChatShellInput && request.Type != protocol.ChatShellResize &&
|
|
request.Type != protocol.TerminalInput && request.Type != protocol.TerminalResize {
|
|
// Keystrokes would be one line each; their data is never logged.
|
|
log.Printf("command received: client=%s type=%q id=%q pipeline=%d version=%d", r.RemoteAddr, request.Type, request.ID, request.PipelineID, request.Version)
|
|
}
|
|
if request.Type == protocol.Submit && !options.AllowAuthoring {
|
|
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "pipeline authoring is disabled on this execution server; ask the operator to publish through an explicitly enabled authoring server"})
|
|
continue
|
|
}
|
|
if elevation.handle(request, clientID, write) || elevation.refuse(request, clientID, write) {
|
|
continue
|
|
}
|
|
if isChatSearch(request) {
|
|
cancelChatSearch()
|
|
searchCtx, cancel := context.WithCancel(connCtx)
|
|
cancelChatSearch = cancel
|
|
go func(request protocol.Message) {
|
|
defer cancel()
|
|
_ = write(chatSearchReply(searchCtx, request, options.Chat, registry))
|
|
}(request)
|
|
continue
|
|
}
|
|
// Prevent a new operator workspace admission racing a worktree
|
|
// removal. Ordinary reads/prompts do not wait for storage scans.
|
|
admission := storage != nil && (request.Type == protocol.ChatCreate || request.Type == protocol.ChatAdopt)
|
|
if admission {
|
|
storage.workspaceMu.RLock()
|
|
}
|
|
handledChat := handleChat(connCtx, request, options.Chat, hub, registry, clientID, write)
|
|
if admission {
|
|
storage.workspaceMu.RUnlock()
|
|
}
|
|
if handledChat {
|
|
continue
|
|
}
|
|
if isStorageRequest(request.Type) {
|
|
if storage == nil {
|
|
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "storage management unavailable: no execution store"})
|
|
} else {
|
|
// Disk/Git inventory must never stall chat permissions/prompts
|
|
// arriving on this same persistent connection.
|
|
storage.dispatch(connCtx, request, write)
|
|
}
|
|
continue
|
|
}
|
|
if isTerminalRequest(request.Type) {
|
|
// Closing waits for the shell to go; it never delays input
|
|
// to the other terminals behind it.
|
|
if request.Type == protocol.TerminalClose {
|
|
go handleTerminalRequest(connCtx, request, terminals, clientID, write)
|
|
} else {
|
|
handleTerminalRequest(connCtx, request, terminals, clientID, write)
|
|
}
|
|
continue
|
|
}
|
|
if handleDirectoryListing(request, write) {
|
|
continue
|
|
}
|
|
if handleHomeFileMutation(request, write) {
|
|
continue
|
|
}
|
|
if request.Type == protocol.DiscoverServersRequest {
|
|
go handleServerDiscovery(connCtx, request, write)
|
|
continue
|
|
}
|
|
if request.Type == protocol.ServerStatsRequest {
|
|
go handleServerStats(connCtx, request, runs, options.ServerStatsSampler, started, build, write)
|
|
continue
|
|
}
|
|
if handleNotificationSettings(ctx, request, settingsStore, write) {
|
|
continue
|
|
}
|
|
if handleAttentionAcks(ctx, request, ackStore, acks, notes, clientID, write) {
|
|
continue
|
|
}
|
|
if handleAppNotifications(ctx, request, notes, ackStore, acks, clientID, write) {
|
|
continue
|
|
}
|
|
if handleRunEvents(request, events, runs, clientID, write) {
|
|
continue
|
|
}
|
|
response := protocol.Message{Type: protocol.Pong, ID: request.ID}
|
|
if request.Type == protocol.List || request.Type == protocol.Submit {
|
|
response.NotificationEndpoints = notificationEndpoints()
|
|
}
|
|
if request.Type == protocol.GetStatus {
|
|
response.Type = protocol.Status
|
|
response.Status = systemStatus(r, started, runErr)
|
|
// How chat is attached to OpenCode is part of the server's status:
|
|
// a configured endpoint that could not be attached reports
|
|
// "fallback" here rather than silently disappearing.
|
|
response.Status.OpenCode = options.OpenCodeStatus
|
|
} else if request.Type == protocol.Ping {
|
|
response.Time = time.Now().UTC().Format(time.RFC3339)
|
|
} else if request.Type == protocol.ListTemplates {
|
|
response.Type = protocol.Templates
|
|
response.Templates = examples.Catalog()
|
|
model, backend := starterDefaults()
|
|
for i := range response.Templates {
|
|
response.Templates[i].Source = pipeline.WithDefaults(response.Templates[i].Source, model, backend)
|
|
}
|
|
} else if request.Type == protocol.Validate {
|
|
graph, warnings, err := pipeline.ParseDefinitionReport(request.Source)
|
|
response.Type = protocol.Validated
|
|
response.Warnings = warnings
|
|
if err != nil {
|
|
response.Type = protocol.Error
|
|
response.Error = err.Error()
|
|
} else {
|
|
response.Graph = &graph
|
|
}
|
|
} else if (request.Type == protocol.ListSchedules || request.Type == protocol.SaveSchedule || request.Type == protocol.DeleteSchedule) && len(stores) > 0 {
|
|
var err error
|
|
switch request.Type {
|
|
case protocol.ListSchedules:
|
|
response.Type = protocol.Schedules
|
|
if request.PipelineID < 0 {
|
|
err = fmt.Errorf("pipeline_id must be nonnegative")
|
|
} else {
|
|
response.Schedules, err = stores[0].ListSchedules(ctx, request.PipelineID)
|
|
}
|
|
case protocol.SaveSchedule:
|
|
response.Type = protocol.SavedScheduleReply
|
|
if request.Schedule == nil {
|
|
err = fmt.Errorf("schedule is required")
|
|
} else {
|
|
response.Schedule, err = stores[0].SaveSchedule(ctx, *request.Schedule, time.Now(), request.RequestKey)
|
|
}
|
|
case protocol.DeleteSchedule:
|
|
response.Type = protocol.ScheduleDeleted
|
|
response.ScheduleID = request.ScheduleID
|
|
err = stores[0].DeleteSchedule(ctx, request.ScheduleID, request.ExpectedRevision)
|
|
}
|
|
if err != nil {
|
|
response.Type = protocol.Error
|
|
response.Error = err.Error()
|
|
}
|
|
} else if request.Type == protocol.Submit && len(stores) > 0 {
|
|
response.Type = protocol.Submitted
|
|
response.Name = strings.TrimSpace(request.Name)
|
|
version, err := stores[0].PublishChecked(ctx, request.Name, request.Source, request.CreateOnly, request.ExpectedVersion)
|
|
_, response.Warnings, _ = pipeline.ParseDefinitionReport(request.Source)
|
|
if err != nil {
|
|
response.Type = protocol.Error
|
|
response.Error = err.Error()
|
|
} else {
|
|
response.Version = version
|
|
response.Pipelines, err = stores[0].List(ctx)
|
|
if err != nil {
|
|
log.Printf("list pipelines after submission: %v", err)
|
|
}
|
|
for _, item := range response.Pipelines {
|
|
if item.Name == response.Name {
|
|
response.Detail, err = stores[0].Get(ctx, item.ID, version)
|
|
if err != nil {
|
|
log.Printf("load submitted pipeline: %v", err)
|
|
}
|
|
break
|
|
}
|
|
}
|
|
}
|
|
} else if request.Type == protocol.GetPipeline && len(stores) > 0 {
|
|
response.Type = protocol.Pipeline
|
|
detail, err := stores[0].Get(ctx, request.PipelineID, request.Version)
|
|
if err != nil {
|
|
response.Type = protocol.Error
|
|
response.Error = err.Error()
|
|
} else {
|
|
response.Detail = detail
|
|
}
|
|
} else if request.Type == protocol.List && len(stores) > 0 {
|
|
response.Type = protocol.PipelineList
|
|
response.DefaultModel, response.DefaultHarness = starterDefaults()
|
|
items, err := stores[0].ListPipelines(ctx, request.IncludeArchived)
|
|
if err != nil {
|
|
response.Type = protocol.Error
|
|
response.Error = err.Error()
|
|
} else {
|
|
response.Pipelines = items
|
|
}
|
|
} else if (request.Type == protocol.ArchivePipeline || request.Type == protocol.UnarchivePipeline) && len(stores) > 0 {
|
|
detail, err := archivePipeline(ctx, stores[0], request)
|
|
response.Detail = detail
|
|
response.Type = protocol.PipelineArchived
|
|
if request.Type == protocol.UnarchivePipeline {
|
|
response.Type = protocol.PipelineUnarchived
|
|
}
|
|
if err != nil {
|
|
response.Type = protocol.Error
|
|
response.Error = err.Error()
|
|
}
|
|
} else if request.Type == protocol.Search {
|
|
if len(stores) == 0 {
|
|
response.Type = protocol.Error
|
|
response.Error = "search unavailable: no store"
|
|
} else if request.Search == nil {
|
|
response.Type = protocol.Error
|
|
response.Error = "search requires a query"
|
|
} else {
|
|
response.Type = protocol.SearchResults
|
|
var err error
|
|
response.SearchResults, err = stores[0].Search(ctx, *request.Search, time.Now())
|
|
if err != nil {
|
|
response.Type = protocol.Error
|
|
response.Error = err.Error()
|
|
}
|
|
}
|
|
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence || request.Type == protocol.RepairSessionExport || request.Type == protocol.DeleteRuns) {
|
|
var err error
|
|
switch request.Type {
|
|
case protocol.DeleteRuns:
|
|
// Like cancel and retry, deleting run history is an execution
|
|
// operation and is not gated by AllowAuthoring.
|
|
response.Type = protocol.RunsDeleted
|
|
response.RunsDeleted, err = runs.deleteRuns(request.RunIDs)
|
|
case protocol.RepairSessionExport:
|
|
response.Type = protocol.Run
|
|
response.RunDetail, err = runs.repairSessionExport(ctx, request.RunID, request.ExportSequence)
|
|
case protocol.GetRunEvidence:
|
|
response.Type = protocol.RunEvidence
|
|
response.Evidence, err = runs.evidence(ctx, request.RunID)
|
|
case protocol.RunPipeline:
|
|
response.Type = protocol.RunStarted
|
|
response.RunDetail, err = runs.start(request)
|
|
case protocol.RetryRun:
|
|
response.Type = protocol.RunStarted
|
|
response.RunDetail, err = runs.retry(request)
|
|
case protocol.DecideRun:
|
|
response.Type = protocol.Run
|
|
response.RunDetail, err = runs.decide(request)
|
|
case protocol.SetRunTitle:
|
|
response.Type = protocol.Run
|
|
response.RunDetail, err = runs.setTitle(request.RunID, request.Title)
|
|
case protocol.ListRuns:
|
|
response.Type = protocol.Runs
|
|
runs.refreshListSizes(connCtx)
|
|
response.Runs, err = runs.listRuns()
|
|
if err == nil {
|
|
response.Usage, err = stores[0].UsageStats(ctx)
|
|
}
|
|
case protocol.GetRun:
|
|
response.Type = protocol.Run
|
|
if request.RunID == 0 && request.RequestKey != "" {
|
|
var id int64
|
|
id, err = stores[0].RunByKey(connCtx, request.RequestKey)
|
|
if err == nil && id != 0 {
|
|
response.RunDetail, err = runs.getRun(id)
|
|
}
|
|
} else {
|
|
response.RunDetail, err = runs.getRun(request.RunID)
|
|
}
|
|
case protocol.CancelRun:
|
|
response.Type = protocol.Run
|
|
err = runs.cancel(request.RunID)
|
|
if err == nil {
|
|
response.RunDetail, err = runs.getRun(request.RunID)
|
|
}
|
|
}
|
|
if err != nil {
|
|
response.Type = protocol.Error
|
|
response.Error = err.Error()
|
|
}
|
|
} else {
|
|
response.Type = protocol.Error
|
|
response.Error = "unknown command: " + request.Type
|
|
if runErr != nil {
|
|
response.Error = "execution unavailable: " + runErr.Error()
|
|
}
|
|
}
|
|
if err := write(response); err != nil {
|
|
log.Printf("write response: %v", err)
|
|
return
|
|
}
|
|
selectedVersion := response.Version
|
|
if response.Detail != nil {
|
|
selectedVersion = response.Detail.Version
|
|
}
|
|
log.Printf("response sent: client=%s type=%q id=%q time=%q error=%q version=%d", r.RemoteAddr, response.Type, response.ID, response.Time, response.Error, selectedVersion)
|
|
}
|
|
}
|
|
mux.HandleFunc("GET /connect", connect)
|
|
if options.Chat != nil && registry != nil {
|
|
// Reattach the chats a previous process left live (and fold in turns
|
|
// that ended while it was down), then keep observing any other chat
|
|
// whose turn is still running, so a reply or question still notifies
|
|
// with no client attached.
|
|
done := make(chan struct{})
|
|
ready := make(chan struct{})
|
|
hub.recoveryDone = done
|
|
hub.recoveryReady = ready
|
|
go func() {
|
|
hub.reattachSessions(ctx, options.Chat)
|
|
// Restore live turn ownership before queue rooms can start runners.
|
|
hub.resumeQueued(ctx, options.Chat)
|
|
close(ready)
|
|
hub.observeRunningTurns(ctx, options.Chat)
|
|
close(done)
|
|
hub.retryReattachments(ctx, options.Chat)
|
|
}()
|
|
}
|
|
if runs != nil {
|
|
// Continue the runs a previous process left in flight, now that the
|
|
// runner's collaborators are wired and before any request is served.
|
|
runs.reattachRuns()
|
|
}
|
|
return &runHandler{Handler: guardRequests(mux, options.AccessToken), runner: runs, hub: hub}
|
|
}
|
|
|
|
type runHandler struct {
|
|
http.Handler
|
|
runner *runner
|
|
hub *chatHub
|
|
}
|
|
|
|
// DetachSessions records the chats left live on their harness so the next
|
|
// start reattaches them. Call it when the server is stopping, before the
|
|
// handler's context is canceled (rooms are torn down after that) and before the
|
|
// store closes. It never interrupts a session.
|
|
func DetachSessions(handler http.Handler) {
|
|
h, ok := handler.(*runHandler)
|
|
if !ok || h.hub == nil {
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
if err := h.hub.detachSessions(ctx); err != nil {
|
|
log.Printf("record live chat sessions: %v", err)
|
|
}
|
|
}
|
|
|
|
// WaitRuns drains active execution before the database is closed on shutdown.
|
|
func WaitRuns(handler http.Handler) {
|
|
if h, ok := handler.(*runHandler); ok && h.runner != nil {
|
|
h.runner.schedulerWG.Wait()
|
|
h.runner.wg.Wait()
|
|
}
|
|
}
|