e9d902213e
Reconcile authoritative shell snapshots with bounded lifecycle journals so concurrent exits and creates cannot be overwritten, coalesce observation setup, and retain verified ownership across reconnects. Add deterministic reconciliation and cancellation regressions plus stable limited-visibility handling.\n\nChecked packaged templates and authoring skill references: none need updating. The task-owned docs note records the stale shared chat documentation for docs-fold; shared documentation was not edited.
484 lines
15 KiB
Go
484 lines
15 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"strconv"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/opencode"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// chatRetainedLimit bounds the state one session may retain so a stalled reader
|
|
// or a runaway turn cannot grow memory without limit. It matches the OpenCode
|
|
// response bound (1 MiB).
|
|
const chatRetainedLimit = opencode.MaxResponse
|
|
|
|
// chatMaxTurns bounds how many sessions retain turn state. A long-lived server
|
|
// cannot accumulate state for every session ever prompted, so the
|
|
// least-recently-updated turn is evicted once the cap is exceeded.
|
|
const chatMaxTurns = 128
|
|
|
|
// chatShellLimit bounds how many running shells one session retains for replay
|
|
// to a client that reattaches, so a runaway turn cannot grow the retained state
|
|
// without limit. It matches the approval-replay bound.
|
|
const chatShellLimit = 16
|
|
|
|
// chatHub retains the accumulated turn state per session, independently of any
|
|
// single connection, so a reconnecting client can re-sync what it missed. State
|
|
// is keyed by session and then by message id (a session holds many turns), and
|
|
// is capped at chatRetainedLimit. It also owns the per-session live rooms: the
|
|
// one broker stream, the prompt queue and the fan-out to every attached client.
|
|
type chatHub struct {
|
|
mu sync.Mutex
|
|
turns map[string]*chatTurn
|
|
rooms map[string]*chatRoom
|
|
baseCtx context.Context
|
|
registry operatorRegistry
|
|
now func() time.Time
|
|
// userSeq is a monotonic source of ids for the user turns the server echoes
|
|
// to the other clients, so each retained user message has a stable id.
|
|
userSeq int
|
|
}
|
|
|
|
// chatPart is the retained form of one part. Text and reasoning deltas append to
|
|
// text (amortized O(1)); a tool frame replaces part outright.
|
|
type chatPart struct {
|
|
part protocol.ChatPart
|
|
text []byte
|
|
}
|
|
|
|
// chatMessage is one retained turn. Its parts keep the exact ids the supporting
|
|
// chat_event frames carried, so a resume preserves attribution. Role and client
|
|
// are carried for a user turn the server echoed (a prompt another client sent),
|
|
// so a resumed transcript still says who sent it.
|
|
type chatMessage struct {
|
|
id string
|
|
role string
|
|
client string
|
|
order []string
|
|
parts map[string]*chatPart
|
|
}
|
|
|
|
type chatTurn struct {
|
|
sessionID string
|
|
order []string
|
|
messages map[string]*chatMessage
|
|
step *protocol.ChatStep
|
|
usage *protocol.ChatUsage
|
|
size int
|
|
overflow bool
|
|
updated time.Time
|
|
// permissions holds the request ids already pushed to the client so a
|
|
// repeated poll cannot push the same pending request twice; permissionOrder
|
|
// bounds it to chatPermissionRetainedLimit.
|
|
permissions map[string]struct{}
|
|
permissionOrder []string
|
|
// resolutions records the shared exactly-once answers to permission requests,
|
|
// keyed by request id, so a second answer is refused and the recorded outcome
|
|
// survives the clients that gave it; resolutionOrder bounds it to
|
|
// chatPermissionRetainedLimit like the dedup set above.
|
|
resolutions map[string]*chatResolution
|
|
resolutionOrder []string
|
|
// shells holds the session's still-running shell commands, keyed by the
|
|
// shell id the stream gave them, so a client that reattaches can be re-synced
|
|
// the background processes it cannot have seen. shellOrder preserves start
|
|
// order and bounds the retained set to chatShellLimit; a turn that has
|
|
// finished retains nothing here, so a background process surviving the turn
|
|
// is never mistaken for the turn still running.
|
|
shells map[string]protocol.ChatShell
|
|
shellOrder []string
|
|
}
|
|
|
|
// chatResolution is the shared answer to one permission request. answered is
|
|
// false while the first answer is still being delivered to the agent, so a
|
|
// concurrent second answer is refused without ever reaching the agent twice.
|
|
type chatResolution struct {
|
|
client string
|
|
decision string
|
|
answered bool
|
|
}
|
|
|
|
func newChatHub() *chatHub {
|
|
return &chatHub{turns: map[string]*chatTurn{}, rooms: map[string]*chatRoom{}, baseCtx: context.Background(), now: time.Now}
|
|
}
|
|
|
|
// nextUserMessageID mints a stable id for one echoed user turn.
|
|
func (h *chatHub) nextUserMessageID() string {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
h.userSeq++
|
|
return "usr_" + strconv.Itoa(h.userSeq)
|
|
}
|
|
|
|
func (h *chatHub) clock() time.Time {
|
|
if h.now != nil {
|
|
return h.now()
|
|
}
|
|
return time.Now()
|
|
}
|
|
|
|
// apply folds one pushed frame into the retained state. Text and reasoning
|
|
// deltas append to their message's part; a tool frame upserts the part (merging
|
|
// the fields it carries onto what is retained, and accounting for the whole
|
|
// size) so the latest lifecycle status, name, input, content, metadata, error and
|
|
// time are retained under the frame's message id and a later frame that omits a
|
|
// field never blanks it. Step and usage frames update the session's latest
|
|
// progress.
|
|
func (h *chatHub) apply(frame protocol.ChatEventFrame) {
|
|
if frame.SessionID == "" {
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
h.applyLocked(frame)
|
|
}
|
|
|
|
func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) {
|
|
if frame.SessionID == "" {
|
|
return
|
|
}
|
|
turn := h.turnLocked(frame.SessionID)
|
|
switch frame.Kind {
|
|
case protocol.ChatEventStep:
|
|
turn.step = frame.Step
|
|
return
|
|
case protocol.ChatEventUsage:
|
|
turn.usage = frame.Usage
|
|
return
|
|
case protocol.ChatEventShell:
|
|
// A shell frame carries no message id (it outlives the tool call that
|
|
// started it), so it is folded before the message-id gate below.
|
|
if room := h.rooms[frame.SessionID]; room != nil {
|
|
room.shellReadback.Record(frame.Shell)
|
|
}
|
|
turn.applyShell(frame.Shell)
|
|
return
|
|
}
|
|
if frame.MessageID == "" {
|
|
return
|
|
}
|
|
message := turn.messages[frame.MessageID]
|
|
if message == nil {
|
|
role := protocol.ChatRoleAssistant
|
|
client := ""
|
|
if frame.Kind == protocol.ChatEventUser {
|
|
role = protocol.ChatRoleUser
|
|
client = frame.Client
|
|
}
|
|
message = &chatMessage{id: frame.MessageID, role: role, client: client, parts: map[string]*chatPart{}}
|
|
turn.order = append(turn.order, frame.MessageID)
|
|
turn.messages[frame.MessageID] = message
|
|
} else if message.client == "" && frame.Kind == protocol.ChatEventUser {
|
|
message.client = frame.Client
|
|
}
|
|
switch frame.Kind {
|
|
case protocol.ChatEventUser:
|
|
if frame.Part == nil || frame.Part.ID == "" || frame.Part.Text == "" || turn.overflow {
|
|
return
|
|
}
|
|
if !turn.retain(len(frame.Part.Text)) {
|
|
return
|
|
}
|
|
part, ok := message.parts[frame.Part.ID]
|
|
if !ok {
|
|
part = &chatPart{part: protocol.ChatPart{Type: protocol.ChatPartText, ID: frame.Part.ID}}
|
|
message.order = append(message.order, frame.Part.ID)
|
|
message.parts[frame.Part.ID] = part
|
|
}
|
|
part.text = append(part.text, frame.Part.Text...)
|
|
case protocol.ChatEventText, protocol.ChatEventReasoning:
|
|
id := ""
|
|
if frame.Part != nil {
|
|
id = frame.Part.ID
|
|
}
|
|
if id == "" || frame.Delta == "" {
|
|
return
|
|
}
|
|
if !turn.retain(len(frame.Delta)) {
|
|
return
|
|
}
|
|
part, ok := message.parts[id]
|
|
if !ok {
|
|
part = &chatPart{part: protocol.ChatPart{Type: frame.Kind, ID: id}}
|
|
message.order = append(message.order, id)
|
|
message.parts[id] = part
|
|
}
|
|
part.text = append(part.text, frame.Delta...)
|
|
case protocol.ChatEventTool:
|
|
if frame.Part == nil || frame.Part.ID == "" || turn.overflow {
|
|
return
|
|
}
|
|
// The tool part is retained whole, so size the complete, merged part
|
|
// (input, content, metadata, error and time) and charge the difference
|
|
// against the cap on every upsert, not just the first-seen name.
|
|
part, ok := message.parts[frame.Part.ID]
|
|
if !ok {
|
|
merged := mergeChatToolPart(nil, frame.Part)
|
|
if !turn.retain(chatPartBytes(&merged)) {
|
|
return
|
|
}
|
|
message.order = append(message.order, frame.Part.ID)
|
|
message.parts[frame.Part.ID] = &chatPart{part: merged}
|
|
return
|
|
}
|
|
merged := mergeChatToolPart(&part.part, frame.Part)
|
|
incoming := chatPartBytes(&merged)
|
|
previous := chatPartBytes(&part.part)
|
|
if incoming > previous {
|
|
if !turn.retain(incoming - previous) {
|
|
return
|
|
}
|
|
} else {
|
|
turn.shrink(previous - incoming)
|
|
}
|
|
part.part = merged
|
|
}
|
|
}
|
|
|
|
// applyShell folds one shell frame into the session's retained shells. A running
|
|
// shell is upserted (keyed by its id) so a client that reattaches can be
|
|
// re-synced the processes still running; an exited shell is dropped, which is
|
|
// what clears the indicator. The retained set is bounded both by count
|
|
// (chatShellLimit) and by the same per-session byte cap the other retained state
|
|
// uses, so a long turn with many shells cannot grow it without limit. A shell
|
|
// exit for a shell the session never retained is ignored, so a frame that is not
|
|
// this session's cannot clear another session's process.
|
|
func (t *chatTurn) applyShell(shell *protocol.ChatShell) {
|
|
if shell == nil || shell.ID == "" {
|
|
return
|
|
}
|
|
if shell.Status != protocol.ChatShellRunning {
|
|
old, ok := t.shells[shell.ID]
|
|
if !ok {
|
|
return
|
|
}
|
|
t.shellOrder = removeShellID(t.shellOrder, shell.ID)
|
|
delete(t.shells, shell.ID)
|
|
t.shrink(len(old.Command))
|
|
return
|
|
}
|
|
if t.shells == nil {
|
|
t.shells = map[string]protocol.ChatShell{}
|
|
}
|
|
if _, ok := t.shells[shell.ID]; !ok {
|
|
if !t.retain(len(shell.Command)) {
|
|
return
|
|
}
|
|
t.shellOrder = append(t.shellOrder, shell.ID)
|
|
}
|
|
t.shells[shell.ID] = *shell
|
|
for len(t.shellOrder) > chatShellLimit {
|
|
id := t.shellOrder[0]
|
|
t.shellOrder = t.shellOrder[1:]
|
|
if dropped, ok := t.shells[id]; ok {
|
|
delete(t.shells, id)
|
|
t.shrink(len(dropped.Command))
|
|
}
|
|
}
|
|
}
|
|
|
|
// removeShellID drops id from order, preserving the order of the rest.
|
|
func removeShellID(order []string, id string) []string {
|
|
for i, existing := range order {
|
|
if existing == id {
|
|
return append(order[:i:i], order[i+1:]...)
|
|
}
|
|
}
|
|
return order
|
|
}
|
|
|
|
// mergeChatToolPart folds one tool frame onto the part retained from the frames
|
|
// before it. A tool call is reported across several frames that each carry only
|
|
// part of it — input.started the name, input.ended the arguments as text, called
|
|
// the parsed input, progress metadata, and success the output with no name and no
|
|
// input — so a field the broker already learned is never blanked by a later frame
|
|
// that omits it. A non-empty incoming field wins; otherwise the retained value is
|
|
// kept.
|
|
func mergeChatToolPart(previous, incoming *protocol.ChatPart) protocol.ChatPart {
|
|
if previous == nil {
|
|
return *incoming
|
|
}
|
|
merged := *incoming
|
|
if merged.Type == "" {
|
|
merged.Type = previous.Type
|
|
}
|
|
if merged.Name == "" {
|
|
merged.Name = previous.Name
|
|
}
|
|
merged.Executed = merged.Executed || previous.Executed
|
|
if merged.Time == nil {
|
|
merged.Time = previous.Time
|
|
}
|
|
switch {
|
|
case incoming.State == nil:
|
|
merged.State = previous.State
|
|
case previous.State == nil:
|
|
// The incoming state stands on its own; merged already carries it.
|
|
default:
|
|
state := *incoming.State
|
|
if state.Status == "" {
|
|
state.Status = previous.State.Status
|
|
}
|
|
if len(state.Input) == 0 {
|
|
state.Input = previous.State.Input
|
|
}
|
|
if len(state.Content) == 0 {
|
|
state.Content = previous.State.Content
|
|
}
|
|
if state.Metadata == nil {
|
|
state.Metadata = previous.State.Metadata
|
|
}
|
|
if state.Error == nil {
|
|
state.Error = previous.State.Error
|
|
}
|
|
merged.State = &state
|
|
}
|
|
return merged
|
|
}
|
|
|
|
// chatPartBytes estimates the retained size of a part so the per-session cap
|
|
// accounts for its whole state instead of only the first-seen name. It walks the
|
|
// fields rather than re-marshalling, so the cost matches the bytes it stores.
|
|
func chatPartBytes(part *protocol.ChatPart) int {
|
|
if part == nil {
|
|
return 0
|
|
}
|
|
n := len(part.Type) + len(part.ID) + len(part.Name)
|
|
if part.State == nil {
|
|
return n
|
|
}
|
|
n += len(part.State.Status) + len(part.State.Input)
|
|
for _, content := range part.State.Content {
|
|
n += len(content.Type) + len(content.Text) + len(content.URI) + len(content.Mime) + len(content.Name)
|
|
}
|
|
if part.State.Error != nil {
|
|
n += len(part.State.Error.Type) + len(part.State.Error.Message)
|
|
}
|
|
for key, value := range part.State.Metadata {
|
|
n += len(key)
|
|
if text, ok := value.(string); ok {
|
|
n += len(text)
|
|
} else {
|
|
n += 8
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// evictLocked drops the least-recently-updated turn when the session cap is
|
|
// reached, keeping retained memory bounded across session churn.
|
|
func (h *chatHub) evictLocked() {
|
|
if len(h.turns) < chatMaxTurns {
|
|
return
|
|
}
|
|
oldestID := ""
|
|
var oldest time.Time
|
|
for id, turn := range h.turns {
|
|
if oldestID == "" || turn.updated.Before(oldest) {
|
|
oldestID, oldest = id, turn.updated
|
|
}
|
|
}
|
|
if oldestID != "" {
|
|
delete(h.turns, oldestID)
|
|
}
|
|
}
|
|
|
|
// turnLocked returns the session's retained turn, creating it (and evicting the
|
|
// least-recently-updated turn first when the cap is reached) and refreshing its
|
|
// update time. It is the shared turn-initialization path for apply and
|
|
// seenPermission, so the two cannot drift. Callers must hold h.mu.
|
|
func (h *chatHub) turnLocked(sessionID string) *chatTurn {
|
|
turn := h.turns[sessionID]
|
|
if turn == nil {
|
|
h.evictLocked()
|
|
turn = &chatTurn{sessionID: sessionID, messages: map[string]*chatMessage{}}
|
|
h.turns[sessionID] = turn
|
|
}
|
|
turn.updated = h.clock()
|
|
return turn
|
|
}
|
|
|
|
// retain records n more bytes and reports whether the turn is still within the
|
|
// bound. The first overflow records one diagnostic and then drops further state
|
|
// instead of growing without limit.
|
|
func (t *chatTurn) retain(n int) bool {
|
|
if t.overflow {
|
|
return false
|
|
}
|
|
if t.size+n > chatRetainedLimit {
|
|
t.overflow = true
|
|
log.Printf("chat session %s retained state exceeds %d bytes; dropping further state", t.sessionID, chatRetainedLimit)
|
|
return false
|
|
}
|
|
t.size += n
|
|
return true
|
|
}
|
|
|
|
// shrink releases n bytes when a tool upsert replaces a larger retained part.
|
|
func (t *chatTurn) shrink(n int) {
|
|
t.size -= n
|
|
if t.size < 0 {
|
|
t.size = 0
|
|
}
|
|
}
|
|
|
|
// snapshot returns the accumulated session as one assistant message per
|
|
// retained message id, with the same stable part ids, or nil when nothing is
|
|
// retained. It is the resume source: a reconnecting client re-syncs the whole
|
|
// session once instead of replaying deltas, and each turn keeps its own id.
|
|
func (h *chatHub) snapshot(sessionID string) []protocol.ChatMessage {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
turn := h.turns[sessionID]
|
|
if turn == nil {
|
|
return nil
|
|
}
|
|
turn.updated = h.clock()
|
|
var messages []protocol.ChatMessage
|
|
for _, id := range turn.order {
|
|
message := turn.messages[id]
|
|
if message == nil || len(message.order) == 0 {
|
|
continue
|
|
}
|
|
out := protocol.ChatMessage{ID: id, SessionID: sessionID, Role: protocol.ChatRoleAssistant}
|
|
if message.role != "" {
|
|
out.Role = message.role
|
|
}
|
|
out.Client = message.client
|
|
for _, partID := range message.order {
|
|
if part, ok := message.parts[partID]; ok {
|
|
piece := part.part
|
|
piece.Text = string(part.text)
|
|
out.Parts = append(out.Parts, piece)
|
|
}
|
|
}
|
|
messages = append(messages, out)
|
|
}
|
|
return messages
|
|
}
|
|
|
|
// shellSnapshot returns the session's still-running shells, in the order they
|
|
// started, so a client attaching to the session's stream can be re-synced the
|
|
// background processes it cannot have seen. It is the shell half of the resume
|
|
// path — the message snapshot cannot carry a shell, because a shell is not a
|
|
// turn — and returns nil when the session has none running. A shell that has
|
|
// exited is not retained, so a stale process is never replayed.
|
|
func (h *chatHub) shellSnapshot(sessionID string) []protocol.ChatShell {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
turn := h.turns[sessionID]
|
|
if turn == nil || len(turn.shells) == 0 {
|
|
return nil
|
|
}
|
|
shells := make([]protocol.ChatShell, 0, len(turn.shells))
|
|
for _, id := range turn.shellOrder {
|
|
if shell, ok := turn.shells[id]; ok {
|
|
shells = append(shells, shell)
|
|
}
|
|
}
|
|
return shells
|
|
}
|