diff --git a/docs/cli.md b/docs/cli.md index 9e1a1112..c5e3f0df 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -435,13 +435,31 @@ sessions pipeline runs create. The protocol types are: client that watches a session sees every frame any client's turn produces — the other devices' prompts and the turn's streaming output — and can answer a shared approval. Attaching is idempotent per connection; the subscription is released - when the connection ends. + when the connection ends. The reply carries `chat_queue` (`session_id`, + `prompts`): the prompts the server still holds for the session, each with + `operation_id`, `message_id`, `native_message_id`, `text`, `client`, + `attachments` and `queued_at`, so a reloaded client draws them as queued. - `chat_prompt` may come from any client. A prompt whose session already has a turn - in flight is accepted into that session's bounded queue (currently 8) and - acknowledged with `chat_queued: true`; the agent never receives two interleaved - turns. The prompt is echoed to the other watching clients as a `chat_event` of - kind `user`, carried with the sender's `client` label (a device name), while the - sender's own transcript keeps its local copy. + in flight is handed to that turn at once when the harness accepts mid-turn input + (Claude Code reads it from stdin at its next step; Codex takes it with + `turn/steer` pinned to the active turn) and is acknowledged as sent. Otherwise + (OpenCode, or a turn that ended meanwhile) it is written to the session's durable + server-side queue (bounded, currently 8) and acknowledged with + `chat_queued: true`; the agent never receives two interleaved turns. Queued + prompts belong to the session, not the connection: they are delivered in order + when the turn ends even with no client attached, and after a server restart once + the harness is idle. A retry with the same `operation_id` is acknowledged again, + never queued or delivered twice. The prompt is echoed to the other watching + clients as a `chat_event` of kind `user`, carried with the sender's `client` + label (a device name), while the sender's own transcript keeps its local copy; + a queued prompt's user event is published again as it is delivered. +- `chat_queue` is pushed to every watching client whenever a session's queue + changes, with the current `prompts` and the change: `delivered`, `removed`, or + `failed` with `error` (the harness refused it; it is not retried). +- `chat_unqueue` (body `chat_unqueue`: `session_id`, `operation_id`) removes one + prompt still waiting in the queue and replies with the updated `chat_queue`. A + prompt already handed to the harness is refused as already sent. Closing a chat + drops its queue. - `chat_reply` is exactly-once: the first answer to a pending request wins, is recorded with the answering client's label and is pushed back to every watching client as the same `chat_permission` frame with `resolved: true`, `decision` and diff --git a/internal/claude/chat.go b/internal/claude/chat.go index 189f856a..cc5cc841 100644 --- a/internal/claude/chat.go +++ b/internal/claude/chat.go @@ -151,6 +151,8 @@ func (c *Chat) PromptWithMessageIDModelFiles(ctx context.Context, id, text, mess if directory == "" { return errors.New("the Claude session has no working directory") } + s.promptMu.Lock() + defer s.promptMu.Unlock() proc, err := s.ensureProcess(ctx) if err != nil { return err @@ -182,10 +184,9 @@ func (c *Chat) PromptWithMessageIDModelFiles(ctx context.Context, id, text, mess s.mu.Lock() s.turnID = turn s.running = true - s.prompts = append(s.prompts, promptRecord{id: messageID, text: strings.TrimSpace(text)}) - if len(s.prompts) > 200 { - s.prompts = s.prompts[len(s.prompts)-200:] - } + s.steered = 0 + s.awaitFollowUp = false + s.recordPromptLocked(messageID, text) s.mu.Unlock() s.emit(protocol.ChatEventFrame{SessionID: id, NativeTurnID: turn, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: turn}}) payload := map[string]any{ @@ -199,6 +200,72 @@ func (c *Chat) PromptWithMessageIDModelFiles(ctx context.Context, id, text, mess return nil } +func (s *session) recordPromptLocked(messageID, text string) { + s.prompts = append(s.prompts, promptRecord{id: messageID, text: strings.TrimSpace(text)}) + if len(s.prompts) > 200 { + s.prompts = s.prompts[len(s.prompts)-200:] + } +} + +// SteerPrompt hands a user message to the turn Claude Code is running now. The +// stream-json CLI accepts user messages on stdin while it works and reads them +// at its next step, surfacing them to the model within the running turn (or, +// when the turn ends first, as its own follow-up turn). It reports false when +// the session has no live process with a turn in flight, so the caller queues +// the prompt instead. The model/mode selection applies at the next turn. +func (c *Chat) SteerPrompt(ctx context.Context, id, text, messageID string, files []opencode.PromptFile) (bool, error) { + s := c.session(id) + if s == nil { + return false, nil + } + if strings.TrimSpace(text) == "" { + return false, errors.New("a nonempty chat prompt is required") + } + s.promptMu.Lock() + defer s.promptMu.Unlock() + s.mu.Lock() + proc := s.proc + running := s.running + s.mu.Unlock() + if proc == nil || !running { + return false, nil + } + content, err := promptContent(text, files) + if err != nil { + return false, err + } + payload := map[string]any{ + "type": "user", + "message": map[string]any{"role": "user", "content": content}, + } + if err := proc.write(payload); err != nil { + return false, fmt.Errorf("Claude Code is not accepting input: %w", err) + } + s.mu.Lock() + s.steered++ + s.lastUsed = time.Now() + s.recordPromptLocked(messageID, text) + s.mu.Unlock() + return true, nil +} + +// adoptFollowUpTurn starts a turn for output the CLI produces after a result +// while messages written mid-turn were outstanding: the CLI is running them as +// a turn of its own, which clients must see as running until its result. +func (s *session) adoptFollowUpTurn() { + s.mu.Lock() + if s.running || !s.awaitFollowUp { + s.mu.Unlock() + return + } + s.awaitFollowUp = false + turn := fmt.Sprintf("claude-%d", time.Now().UnixNano()) + s.turnID = turn + s.running = true + s.mu.Unlock() + s.emit(protocol.ChatEventFrame{SessionID: s.id, NativeTurnID: turn, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: turn}}) +} + // finishTurn settles a turn that never reached a CLI result. func (s *session) finishTurn(turn, diagnostic string) { s.mu.Lock() diff --git a/internal/claude/claude.go b/internal/claude/claude.go index d0ff4cca..c3ca7fbb 100644 --- a/internal/claude/claude.go +++ b/internal/claude/claude.go @@ -243,8 +243,15 @@ type session struct { startErr error turnID string running bool - lastUsed time.Time - lastErr string + // promptMu serializes writing user turns, so a prompt and a mid-turn + // message never both decide the CLI is idle. steered counts messages + // written into the running turn; awaitFollowUp is set when a turn ended + // with some outstanding, because the CLI then runs them as its own turn. + promptMu sync.Mutex + steered int + awaitFollowUp bool + lastUsed time.Time + lastErr string subs map[chan protocol.ChatEventFrame]struct{} approvals map[string]*pendingApproval diff --git a/internal/claude/process.go b/internal/claude/process.go index 4b1a4773..2b3dfc9b 100644 --- a/internal/claude/process.go +++ b/internal/claude/process.go @@ -352,6 +352,8 @@ func (s *session) processExit(p *process, err error) { turn := s.turnID s.running = false s.turnID = "" + s.steered = 0 + s.awaitFollowUp = false s.resetStreamLocked() s.mu.Unlock() if !running { diff --git a/internal/claude/steer_test.go b/internal/claude/steer_test.go new file mode 100644 index 00000000..37ce4a9f --- /dev/null +++ b/internal/claude/steer_test.go @@ -0,0 +1,89 @@ +package claude + +import ( + "context" + "os" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// A message sent while a turn runs is written to the CLI at once: the CLI +// reads it at the running turn's next step, and the transcript records it as +// the operator's turn at that point, carrying the caller's message id. +func TestSteerPromptReachesTheRunningTurn(t *testing.T) { + chat := testChat(t) + session := newCreatedChat(t, chat) + frames := subscribe(t, chat, session.ID) + if delivered, err := chat.SteerPrompt(context.Background(), session.ID, "too early", "msg_early", nil); err != nil || delivered { + t.Fatalf("idle session took a mid-turn message: delivered=%v err=%v", delivered, err) + } + if err := chat.Prompt(context.Background(), session.ID, "long task"); err != nil { + t.Fatal(err) + } + started := 0 + waitFrame(t, frames, "turn started", func(frame protocol.ChatEventFrame) bool { + if frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepStarted { + started++ + return true + } + return false + }) + delivered, err := chat.SteerPrompt(context.Background(), session.ID, "also check the docs", "msg_steer", nil) + if err != nil || !delivered { + t.Fatalf("running turn refused a mid-turn message: delivered=%v err=%v", delivered, err) + } + // The fake CLI only finishes the long turn once it has read the message. + waitFrame(t, frames, "steered reply", func(frame protocol.ChatEventFrame) bool { + return frame.Kind == protocol.ChatEventTool || (frame.Kind == protocol.ChatEventText && strings.Contains(frame.Delta, "Noted")) || + (frame.Part != nil && strings.Contains(frame.Part.Text, "Noted")) + }) + waitStepEnded(t, frames) + steered, err := os.ReadFile(os.Getenv("FAKE_CLAUDE_LOG") + ".steer") + if err != nil || strings.TrimSpace(string(steered)) != "also check the docs" { + t.Fatalf("CLI did not receive the message mid-turn: %q %v", steered, err) + } + if chat.TurnRunning(session.ID) { + t.Fatal("turn still running after its result") + } + page, err := chat.Messages(context.Background(), session.ID, 50, "") + if err != nil { + t.Fatal(err) + } + var users []string + for _, message := range page.Messages { + if message.Role == protocol.ChatRoleUser { + users = append(users, message.ID+"="+message.Parts[0].Text) + } + } + if len(users) != 2 || users[1] != "msg_steer=also check the docs" { + t.Fatalf("transcript users = %v, want the mid-turn message recorded once with its id", users) + } +} + +// When the CLI ends the turn before reading a mid-turn message it runs that +// message as a turn of its own; its output must be reported as a running turn +// with a start and an end rather than activity outside any turn. +func TestSteerPromptFollowUpTurnIsAdopted(t *testing.T) { + chat := testChat(t) + session := newCreatedChat(t, chat) + frames := subscribe(t, chat, session.ID) + s := chat.session(session.ID) + s.mu.Lock() + s.awaitFollowUp = true + s.mu.Unlock() + s.adoptFollowUpTurn() + frame := waitFrame(t, frames, "follow-up start", func(frame protocol.ChatEventFrame) bool { + return frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepStarted + }) + if frame.Step.TurnID == "" || !chat.TurnRunning(session.ID) { + t.Fatalf("follow-up turn not adopted: %+v", frame.Step) + } + s.handleResult(lineEnvelope{Type: "result", Subtype: "success"}) + waitStepEnded(t, frames) + s.adoptFollowUpTurn() + if chat.TurnRunning(session.ID) { + t.Fatal("a turn without outstanding messages was adopted") + } +} diff --git a/internal/claude/testdata/fake-claude.py b/internal/claude/testdata/fake-claude.py index ecf056b3..1d5f6cb8 100644 --- a/internal/claude/testdata/fake-claude.py +++ b/internal/claude/testdata/fake-claude.py @@ -211,8 +211,31 @@ def handle(line): if images and log_path: with open(log_path + ".content", "a", encoding="utf-8") as log: log.write(json.dumps(content) + "\n") + pending = state["pending"] + if pending and pending["kind"] == "long": + steer(pending, text) + return user_message(text) +def steer(pending, text): + """A message written while a turn runs is read at the turn's next step, + recorded as a queued_command attachment, and the same turn finishes.""" + if log_path: + with open(log_path + ".steer", "a", encoding="utf-8") as log: + log.write(text + "\n") + state["pending"] = None + append({"type": "attachment", "uuid": "steer-%d" % state["turn"], "timestamp": stamp(), + "attachment": {"type": "queued_command", "commandMode": "prompt", + "prompt": [{"type": "text", "text": text}]}}) + reply = "Noted: " + text + append({"type": "assistant", "uuid": "asst-steer-%d" % state["turn"], "timestamp": stamp(), + "message": {"id": pending["message"] + "_steer", "role": "assistant", "model": "claude-test", + "content": [{"type": "text", "text": reply}]}}) + send({"type": "assistant", "message": {"id": pending["message"] + "_steer", "role": "assistant", + "model": "claude-test", "content": [{"type": "text", "text": reply}]}}) + finish(pending["message"]) + + for line in sys.stdin: handle(line.strip()) diff --git a/internal/claude/transcript.go b/internal/claude/transcript.go index 7fe1a480..6cf2bc38 100644 --- a/internal/claude/transcript.go +++ b/internal/claude/transcript.go @@ -35,6 +35,14 @@ type transcriptEntry struct { } `json:"message"` ToolUseResult json.RawMessage `json:"toolUseResult"` Compact json.RawMessage `json:"compactMetadata"` + // Attachment is set on "attachment" entries; a queued_command with + // commandMode "prompt" is a user message the CLI read mid-turn. Prompt is + // either a string or an array of content blocks. + Attachment struct { + Type string `json:"type"` + Prompt json.RawMessage `json:"prompt"` + CommandMode string `json:"commandMode"` + } `json:"attachment"` } func readTranscript(path string) ([]transcriptEntry, error) { @@ -150,6 +158,22 @@ func (s *session) transcriptMessages(limit int, cursor string) (protocol.ChatHis messages := []protocol.ChatMessage{} messageIndex := map[string]int{} toolIndex := map[string][2]int{} // tool id -> message/part index + addUser := func(entry transcriptEntry, text string, files []protocol.ChatPart) { + id := takePromptID(text) + if id == "" { + id = entry.UUID + } + if id == "" { + return + } + created, _ := transcriptTime(entry.Timestamp) + parts := []protocol.ChatPart{} + if text != "" { + parts = append(parts, protocol.ChatPart{Type: protocol.ChatPartText, ID: id + "#text", Text: text}) + } + parts = append(parts, files...) + messages = append(messages, protocol.ChatMessage{ID: id, SessionID: s.id, Role: protocol.ChatRoleUser, CreatedAt: created, Parts: parts}) + } for _, entry := range entries { if entry.IsSidechain { continue @@ -239,20 +263,29 @@ func (s *session) transcriptMessages(limit int, cursor string) (protocol.ChatHis if hasResult || (text == "" && len(files) == 0) { continue } - id := takePromptID(text) - if id == "" { - id = entry.UUID - } - if id == "" { + addUser(entry, text, files) + case "attachment": + // A message the operator sent while the turn was running is recorded + // as a queued_command attachment where the CLI surfaced it to the + // model, so it is the user's turn at that point of the transcript. + // Other queued commands (task notifications, agent messages) are + // harness input, not the operator's. + if entry.Attachment.Type != "queued_command" || entry.Attachment.CommandMode != "prompt" { continue } - created, _ := transcriptTime(entry.Timestamp) - parts := []protocol.ChatPart{} - if text != "" { - parts = append(parts, protocol.ChatPart{Type: protocol.ChatPartText, ID: id + "#text", Text: text}) + var text string + for _, block := range decodeBlocks(entry.Attachment.Prompt) { + if block.Type == "text" && strings.TrimSpace(block.Text) != "" { + if text != "" { + text += "\n" + } + text += block.Text + } } - parts = append(parts, files...) - messages = append(messages, protocol.ChatMessage{ID: id, SessionID: s.id, Role: protocol.ChatRoleUser, CreatedAt: created, Parts: parts}) + if strings.TrimSpace(text) == "" { + continue + } + addUser(entry, text, []protocol.ChatPart{}) } } // Drop assistant messages that ended with no renderable part (for example a diff --git a/internal/claude/translate.go b/internal/claude/translate.go index ee750ea7..c3ae8a1e 100644 --- a/internal/claude/translate.go +++ b/internal/claude/translate.go @@ -102,8 +102,10 @@ func (s *session) handleLine(p *process, line []byte) { case "control_response": s.handleControlResponse(p, envelope) case "stream_event": + s.adoptFollowUpTurn() s.handleStreamEvent(envelope) case "assistant": + s.adoptFollowUpTurn() s.handleAssistant(envelope) case "user": s.handleUser(envelope) @@ -552,6 +554,10 @@ func (s *session) handleResult(envelope lineEnvelope) { proc := s.proc s.running = false s.turnID = "" + // Messages written mid-turn that the CLI did not fold into this turn run + // next as their own turn; its output adopts a new turn id. + s.awaitFollowUp = s.steered > 0 + s.steered = 0 s.approvals = map[string]*pendingApproval{} s.forms = map[string]*pendingForm{} s.resetStreamLocked() @@ -592,6 +598,7 @@ func (s *session) handleSystem(envelope lineEnvelope) { s.emit(protocol.ChatEventFrame{SessionID: s.id, NativeTurnID: s.currentTurn(), Kind: protocol.ChatEventCompaction, Compaction: &protocol.ChatCompaction{Status: "completed"}}) case "init": // The CLI re-emits init at each turn; keep only the authoritative identity. + s.adoptFollowUpTurn() if envelope.SessionID != "" && envelope.SessionID != s.id { s.mu.Lock() s.lastErr = "Claude Code opened unexpected session " + envelope.SessionID diff --git a/internal/codex/chat.go b/internal/codex/chat.go index 49a36732..b1aca9c7 100644 --- a/internal/codex/chat.go +++ b/internal/codex/chat.go @@ -457,6 +457,38 @@ func (c *Chat) PromptWithMessageIDModelFiles(ctx context.Context, id, text, mess return err } +// SteerPrompt adds a user message to the thread's active turn with the App +// Server's turn/steer, which the agent reads at its next step. expectedTurnId +// pins the turn: when it already ended (or none is active) the App Server +// refuses and false is returned so the caller queues the prompt for the next +// turn instead. A transport failure is an error: it may have been admitted and +// must not be blindly repeated. +func (c *Chat) SteerPrompt(ctx context.Context, id, text, messageID string, files []opencode.PromptFile) (bool, error) { + c.mu.Lock() + turn := c.turns[id] + c.mu.Unlock() + if turn == "" { + return false, nil + } + input, err := promptInput(text, files) + if err != nil { + return false, err + } + params := map[string]any{"threadId": id, "expectedTurnId": turn, "input": input} + if messageID != "" { + params["clientUserMessageId"] = messageID + } + _, err = c.call(ctx, "turn/steer", params) + var refused *rpcCallError + if errors.As(err, &refused) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + // ActivateSkill is an OpenCode-only chat concept. func (c *Chat) ActivateSkill(context.Context, string, string) error { return errors.New("Codex chat has no out-of-band skill activation") diff --git a/internal/codex/steer_test.go b/internal/codex/steer_test.go new file mode 100644 index 00000000..262d2e69 --- /dev/null +++ b/internal/codex/steer_test.go @@ -0,0 +1,47 @@ +package codex + +import ( + "context" + "encoding/json" + "testing" +) + +// A prompt for a thread with an active turn is steered into that turn, pinned +// by expectedTurnId; a refusal (the turn already ended) reports not delivered +// so the server queues it, and a thread with no active turn is not steered. +func TestSteerPromptTargetsTheActiveTurn(t *testing.T) { + var steered []map[string]any + refuse := false + chat := modelTestChat(t, func(method string, raw json.RawMessage) any { + if method != "turn/steer" { + t.Errorf("unexpected method %s", method) + return map[string]any{} + } + var params map[string]any + if err := json.Unmarshal(raw, ¶ms); err != nil { + t.Error(err) + } + steered = append(steered, params) + if refuse { + return &rpcError{Code: -32600, Message: "expected turn is not active"} + } + return map[string]any{"turnId": params["expectedTurnId"]} + }) + if delivered, err := chat.SteerPrompt(context.Background(), fixtureThread, "idle", "msg_idle", nil); err != nil || delivered { + t.Fatalf("idle thread steered: delivered=%v err=%v", delivered, err) + } + chat.mu.Lock() + chat.turns[fixtureThread] = "turn-7" + chat.mu.Unlock() + delivered, err := chat.SteerPrompt(context.Background(), fixtureThread, "also this", "msg_steer", nil) + if err != nil || !delivered { + t.Fatalf("active turn not steered: delivered=%v err=%v", delivered, err) + } + if len(steered) != 1 || steered[0]["expectedTurnId"] != "turn-7" || steered[0]["threadId"] != fixtureThread || steered[0]["clientUserMessageId"] != "msg_steer" { + t.Fatalf("steer params = %+v", steered) + } + refuse = true + if delivered, err := chat.SteerPrompt(context.Background(), fixtureThread, "late", "msg_late", nil); err != nil || delivered { + t.Fatalf("refused steer reported delivered=%v err=%v", delivered, err) + } +} diff --git a/internal/pipeline/chat_prompt_queue.go b/internal/pipeline/chat_prompt_queue.go new file mode 100644 index 00000000..c7d5df52 --- /dev/null +++ b/internal/pipeline/chat_prompt_queue.go @@ -0,0 +1,170 @@ +package pipeline + +import ( + "context" + "database/sql" + "errors" + "time" +) + +// ErrChatQueueFull refuses a prompt that would exceed a session's queue bound. +var ErrChatQueueFull = errors.New("chat prompt queue is full") + +// ChatQueuedPrompt is one durable prompt waiting for its session's harness to +// finish the turn in flight. Files and Parts are the server-prepared attachment +// references (JSON) the prompt carries; the bytes stay in the attachment store. +type ChatQueuedPrompt struct { + ID int64 + SessionID string + OperationID string + MessageID string + NativeID string + Text string + Client string + FilesJSON string + PartsJSON string + CreatedAt time.Time +} + +// initChatPromptQueue creates the additive queue table. A prompt belongs to +// its chat session, not to a connection: rows are written when a prompt is +// queued behind a running turn and deleted once it is delivered or removed. +// (session_id, operation_id) is unique so a client retry never enqueues twice. +func (s *Store) initChatPromptQueue() error { + _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS chat_prompt_queue ( +id INTEGER PRIMARY KEY AUTOINCREMENT, +session_id TEXT NOT NULL, +operation_id TEXT NOT NULL, +message_id TEXT NOT NULL, +native_message_id TEXT NOT NULL DEFAULT '', +text TEXT NOT NULL, +client TEXT NOT NULL DEFAULT '', +files_json TEXT NOT NULL DEFAULT '[]', +parts_json TEXT NOT NULL DEFAULT '[]', +created_at INTEGER NOT NULL, +UNIQUE(session_id, operation_id)); +CREATE INDEX IF NOT EXISTS chat_prompt_queue_session ON chat_prompt_queue(session_id, id);`) + return err +} + +// EnqueueChatPrompt appends prompt to its session's queue. When the session +// already holds the same operation id the existing row is returned with +// created false and nothing is written. A new row that would exceed limit is +// refused with ErrChatQueueFull. +func (s *Store) EnqueueChatPrompt(ctx context.Context, prompt ChatQueuedPrompt, limit int) (ChatQueuedPrompt, bool, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return ChatQueuedPrompt{}, false, err + } + defer tx.Rollback() + existing, err := scanChatQueuedPrompt(tx.QueryRowContext(ctx, chatQueueSelect+` WHERE session_id = ? AND operation_id = ?`, prompt.SessionID, prompt.OperationID)) + if err == nil { + return existing, false, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return ChatQueuedPrompt{}, false, err + } + var count int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM chat_prompt_queue WHERE session_id = ?`, prompt.SessionID).Scan(&count); err != nil { + return ChatQueuedPrompt{}, false, err + } + if limit > 0 && count >= limit { + return ChatQueuedPrompt{}, false, ErrChatQueueFull + } + if prompt.CreatedAt.IsZero() { + prompt.CreatedAt = time.Now() + } + if prompt.FilesJSON == "" { + prompt.FilesJSON = "[]" + } + if prompt.PartsJSON == "" { + prompt.PartsJSON = "[]" + } + result, err := tx.ExecContext(ctx, `INSERT INTO chat_prompt_queue +(session_id, operation_id, message_id, native_message_id, text, client, files_json, parts_json, created_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + prompt.SessionID, prompt.OperationID, prompt.MessageID, prompt.NativeID, prompt.Text, prompt.Client, + prompt.FilesJSON, prompt.PartsJSON, prompt.CreatedAt.UnixMilli()) + if err != nil { + return ChatQueuedPrompt{}, false, err + } + if prompt.ID, err = result.LastInsertId(); err != nil { + return ChatQueuedPrompt{}, false, err + } + if err := tx.Commit(); err != nil { + return ChatQueuedPrompt{}, false, err + } + return prompt, true, nil +} + +// DeleteChatQueuedPrompt removes one queued prompt and reports whether a row +// was removed. +func (s *Store) DeleteChatQueuedPrompt(ctx context.Context, sessionID, operationID string) (bool, error) { + result, err := s.db.ExecContext(ctx, `DELETE FROM chat_prompt_queue WHERE session_id = ? AND operation_id = ?`, sessionID, operationID) + if err != nil { + return false, err + } + n, err := result.RowsAffected() + return n > 0, err +} + +// ClearChatQueuedPrompts drops every queued prompt of one session. +func (s *Store) ClearChatQueuedPrompts(ctx context.Context, sessionID string) error { + _, err := s.db.ExecContext(ctx, `DELETE FROM chat_prompt_queue WHERE session_id = ?`, sessionID) + return err +} + +// ChatQueuedPrompts lists one session's queued prompts in delivery order. +func (s *Store) ChatQueuedPrompts(ctx context.Context, sessionID string) ([]ChatQueuedPrompt, error) { + rows, err := s.db.QueryContext(ctx, chatQueueSelect+` WHERE session_id = ? ORDER BY id`, sessionID) + if err != nil { + return nil, err + } + defer rows.Close() + var prompts []ChatQueuedPrompt + for rows.Next() { + prompt, err := scanChatQueuedPrompt(rows) + if err != nil { + return nil, err + } + prompts = append(prompts, prompt) + } + return prompts, rows.Err() +} + +// ChatQueuedSessions lists the sessions holding queued prompts, oldest first, +// so a restarted server can resume their delivery. +func (s *Store) ChatQueuedSessions(ctx context.Context) ([]string, error) { + rows, err := s.db.QueryContext(ctx, `SELECT session_id FROM chat_prompt_queue GROUP BY session_id ORDER BY MIN(id)`) + if err != nil { + return nil, err + } + defer rows.Close() + var sessions []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + sessions = append(sessions, id) + } + return sessions, rows.Err() +} + +const chatQueueSelect = `SELECT id, session_id, operation_id, message_id, native_message_id, text, client, files_json, parts_json, created_at FROM chat_prompt_queue` + +type chatQueueScanner interface { + Scan(...any) error +} + +func scanChatQueuedPrompt(row chatQueueScanner) (ChatQueuedPrompt, error) { + var prompt ChatQueuedPrompt + var created int64 + err := row.Scan(&prompt.ID, &prompt.SessionID, &prompt.OperationID, &prompt.MessageID, &prompt.NativeID, + &prompt.Text, &prompt.Client, &prompt.FilesJSON, &prompt.PartsJSON, &created) + if err != nil { + return ChatQueuedPrompt{}, err + } + prompt.CreatedAt = time.UnixMilli(created) + return prompt, nil +} diff --git a/internal/pipeline/chat_prompt_queue_test.go b/internal/pipeline/chat_prompt_queue_test.go new file mode 100644 index 00000000..2eb1cb99 --- /dev/null +++ b/internal/pipeline/chat_prompt_queue_test.go @@ -0,0 +1,45 @@ +package pipeline + +import ( + "context" + "errors" + "path/filepath" + "testing" +) + +func TestChatPromptQueueOrderDedupeAndBound(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "queue.db") + store, err := Open(path) + if err != nil { + t.Fatal(err) + } + for _, op := range []string{"a", "b"} { + if _, created, err := store.EnqueueChatPrompt(ctx, ChatQueuedPrompt{SessionID: "ses_1", OperationID: op, MessageID: "m-" + op, Text: op}, 2); err != nil || !created { + t.Fatalf("enqueue %s: created=%v err=%v", op, created, err) + } + } + if row, created, err := store.EnqueueChatPrompt(ctx, ChatQueuedPrompt{SessionID: "ses_1", OperationID: "a", Text: "again"}, 2); err != nil || created || row.Text != "a" { + t.Fatalf("duplicate operation: row=%+v created=%v err=%v", row, created, err) + } + if _, _, err := store.EnqueueChatPrompt(ctx, ChatQueuedPrompt{SessionID: "ses_1", OperationID: "c", Text: "c"}, 2); !errors.Is(err, ErrChatQueueFull) { + t.Fatalf("bound not enforced: %v", err) + } + store.Close() + store, err = Open(path) + if err != nil { + t.Fatal(err) + } + defer store.Close() + sessions, err := store.ChatQueuedSessions(ctx) + if err != nil || len(sessions) != 1 || sessions[0] != "ses_1" { + t.Fatalf("sessions = %v %v", sessions, err) + } + if removed, err := store.DeleteChatQueuedPrompt(ctx, "ses_1", "a"); err != nil || !removed { + t.Fatalf("delete: %v %v", removed, err) + } + rows, err := store.ChatQueuedPrompts(ctx, "ses_1") + if err != nil || len(rows) != 1 || rows[0].OperationID != "b" || rows[0].MessageID != "m-b" { + t.Fatalf("rows = %+v %v", rows, err) + } +} diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go index 7e3f9a13..d3cb9aff 100644 --- a/internal/pipeline/store.go +++ b/internal/pipeline/store.go @@ -81,6 +81,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU db.Close() return nil, err } + if err := s.initChatPromptQueue(); err != nil { + db.Close() + return nil, err + } if err := s.initAppNotifications(); err != nil { db.Close() return nil, err diff --git a/internal/protocol/chat.go b/internal/protocol/chat.go index 737afef5..43709ce4 100644 --- a/internal/protocol/chat.go +++ b/internal/protocol/chat.go @@ -69,6 +69,16 @@ const ( // body is a ChatQuery naming the session and the response echoes the request // id once the subscription is attached. ChatSubscribe = "chat_subscribe" + // ChatQueue is pushed to every client watching a session whenever its + // server-side prompt queue changes (a prompt queued, delivered, removed or + // refused by the harness). The chat_subscribe reply carries the same body so + // a reloaded client draws the queued prompts the server still holds. + ChatQueue = "chat_queue" + // ChatUnqueue removes one prompt the server still holds in a session's + // queue, named by its operation id. Its body is ChatUnqueueRequest and the + // reply carries the updated queue in ChatQueue. A prompt already delivered + // to the harness is no longer removable and is refused. + ChatUnqueue = "chat_unqueue" // ChatDirectories is the request/response type for the operator's directory // choices: the home directory plus the directories of previous operator // sessions, most recently used first, deduplicated and bounded. It is a @@ -354,6 +364,37 @@ type ChatPromptRequest struct { Attachments []ChatAttachment `json:"attachments,omitempty"` } +// ChatQueuedPrompt is one prompt the server holds for a session until its +// harness finishes the turn in flight. It is durable: it survives the sending +// client disconnecting and a server restart, and is delivered in order. +type ChatQueuedPrompt struct { + OperationID string `json:"operation_id"` + MessageID string `json:"message_id"` + NativeMessageID string `json:"native_message_id,omitempty"` + Text string `json:"text"` + Client string `json:"client,omitempty"` + Attachments []ChatPart `json:"attachments,omitempty"` + QueuedAt int64 `json:"queued_at,omitempty"` +} + +// ChatQueueFrame is a session's current server-side prompt queue, in delivery +// order. Delivered, Removed or Failed name the operation whose change produced +// the frame; Error carries the harness refusal for Failed. +type ChatQueueFrame struct { + SessionID string `json:"session_id"` + Prompts []ChatQueuedPrompt `json:"prompts"` + Delivered string `json:"delivered,omitempty"` + Removed string `json:"removed,omitempty"` + Failed string `json:"failed,omitempty"` + Error string `json:"error,omitempty"` +} + +// ChatUnqueueRequest names one queued prompt to remove. +type ChatUnqueueRequest struct { + SessionID string `json:"session_id"` + OperationID string `json:"operation_id"` +} + // ChatAttachment is one browser attachment carried by a chat prompt. Data is // standard base64 (no data: prefix); Name is a display name and Mime is the // browser's content type. The server derives the on-disk name and never trusts diff --git a/internal/protocol/message.go b/internal/protocol/message.go index be475185..942fd650 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -221,6 +221,11 @@ type Message struct { // accepted into the session's queue instead of being sent immediately. ChatSubscribe *ChatQuery `json:"chat_subscribe,omitempty"` ChatQueued bool `json:"chat_queued,omitempty"` + // ChatQueue is the session's server-held prompt queue: pushed as a + // chat_queue frame when it changes and carried by the chat_subscribe and + // chat_unqueue replies. + ChatQueue *ChatQueueFrame `json:"chat_queue,omitempty"` + ChatUnqueue *ChatUnqueueRequest `json:"chat_unqueue,omitempty"` // ChatDirectories carries the operator's directory choices on a // chat_directories response (home first, then recent operator-session // directories, deduplicated and bounded). Additive/omitempty so older peers diff --git a/internal/server/chat.go b/internal/server/chat.go index a9664772..a042e88f 100644 --- a/internal/server/chat.go +++ b/internal/server/chat.go @@ -167,6 +167,9 @@ func handleChat(ctx context.Context, request protocol.Message, broker chatBroker case protocol.ChatSubscribe: handleChatSubscribe(request, broker, hub, clientID, write) return true + case protocol.ChatUnqueue: + handleChatUnqueue(ctx, request, hub, write) + return true case protocol.ChatMessages: handleChatMessages(ctx, request, broker, hub, write) return true diff --git a/internal/server/chat_close.go b/internal/server/chat_close.go index f232a0c1..53202481 100644 --- a/internal/server/chat_close.go +++ b/internal/server/chat_close.go @@ -56,6 +56,8 @@ func handleChatClose(ctx context.Context, request protocol.Message, broker chatB _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) return } + // A closed chat takes no more turns: prompts it still held are dropped. + hub.clearQueue(ctx, sessionID) session, found, err := registry.OperatorSession(ctx, sessionID) if err != nil { _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) diff --git a/internal/server/chat_contract_test.go b/internal/server/chat_contract_test.go index a17e029f..78e0c4c4 100644 --- a/internal/server/chat_contract_test.go +++ b/internal/server/chat_contract_test.go @@ -34,3 +34,9 @@ func TestAdvertisedChatHarnessesExposeInteractiveWebContract(t *testing.T) { } } } + +// Claude Code and Codex take a prompt into a running turn; OpenCode does not +// (its prompts are queued durably on the server behind the running turn). +var _ chatMidTurn = (*claude.Chat)(nil) +var _ chatMidTurn = (*codex.Chat)(nil) +var _ chatMidTurn = (*chatManager)(nil) diff --git a/internal/server/chat_hub.go b/internal/server/chat_hub.go index 57645fbf..1f8a220d 100644 --- a/internal/server/chat_hub.go +++ b/internal/server/chat_hub.go @@ -51,6 +51,13 @@ type chatHub struct { attentionSeen map[string]struct{} attentionOrder []string attentionWG sync.WaitGroup + // queueStore persists prompts queued behind a running turn; nil keeps the + // queue in memory only (tests and store-less servers). + queueStore chatPromptQueueStore + // recentOps/recentOpOrder are the bounded record of accepted prompt + // operation ids, so a client retry is acknowledged instead of re-sent. + recentOps map[string]struct{} + recentOpOrder []string } // chatPart is the retained form of one part. Text and reasoning deltas append to diff --git a/internal/server/chat_manager.go b/internal/server/chat_manager.go index de06dffc..3e94047a 100644 --- a/internal/server/chat_manager.go +++ b/internal/server/chat_manager.go @@ -388,6 +388,26 @@ func (m *chatManager) PromptWithMessageIDFiles(ctx context.Context, sessionID, t return errChatAttachmentsUnsupported } +// SteerPrompt forwards a prompt into the routed harness's running turn when +// that harness accepts mid-turn input (Claude Code, Codex). A harness without +// it reports not delivered, so the hub queues the prompt durably instead. +func (m *chatManager) SteerPrompt(ctx context.Context, sessionID, text, messageID string, files []opencode.PromptFile) (bool, error) { + adapter, err := m.brokerFor(ctx, sessionID) + if err != nil { + return false, err + } + steerer, ok := adapter.(chatMidTurn) + if !ok { + return false, nil + } + if len(files) > 0 { + if _, ok := adapter.(chatPromptWithFiles); !ok { + return false, errChatAttachmentsUnsupported + } + } + return steerer.SteerPrompt(ctx, sessionID, text, messageID, files) +} + var errChatAttachmentsUnsupported = errors.New("this chat harness does not support attachments") var errChatFormReadbackUnsupported = errors.New("this chat harness does not support pending question readback") diff --git a/internal/server/chat_queue.go b/internal/server/chat_queue.go new file mode 100644 index 00000000..ed3f8cae --- /dev/null +++ b/internal/server/chat_queue.go @@ -0,0 +1,476 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net/url" + "path/filepath" + "time" + + "git.a-wan.net/alan/pipelines/internal/opencode" + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// A prompt that arrives while its session's turn is running belongs to the +// server, not to the connection that sent it. Where the harness accepts input +// mid-turn (Claude Code's stream-json stdin, Codex turn/steer) the prompt is +// forwarded at once and the harness reads it at its next step. Otherwise it is +// written to the durable chat_prompt_queue and delivered in order when the turn +// ends, whether or not any client is still attached, and after a restart. + +// chatQueueBusyPoll is how often a room holding queued prompts re-checks a +// harness that reports its own turn in flight. TurnRunning performs no I/O. +var chatQueueBusyPoll = time.Second + +// chatQueueRecoverySettle gives a restarted server's freshly attached stream a +// moment to announce a turn the harness is still running, so a recovered +// prompt is not delivered into a busy harness that only reports activity +// through its stream. +var chatQueueRecoverySettle = 1500 * time.Millisecond + +// chatRecentOpsLimit bounds the remembered accepted operation ids used to +// refuse a duplicate delivery of a client retry. +const chatRecentOpsLimit = 256 + +// chatPromptQueueStore is the durable queue seam; *pipeline.Store implements it. +type chatPromptQueueStore interface { + EnqueueChatPrompt(ctx context.Context, prompt pipeline.ChatQueuedPrompt, limit int) (pipeline.ChatQueuedPrompt, bool, error) + DeleteChatQueuedPrompt(ctx context.Context, sessionID, operationID string) (bool, error) + ClearChatQueuedPrompts(ctx context.Context, sessionID string) error + ChatQueuedPrompts(ctx context.Context, sessionID string) ([]pipeline.ChatQueuedPrompt, error) + ChatQueuedSessions(ctx context.Context) ([]string, error) +} + +// chatMidTurn is implemented by a broker that can hand a user message to a +// turn already running. delivered is false (with a nil error) when the session +// has no turn the harness can take input for right now, and the caller then +// queues the prompt instead. An error means the harness refused the message. +type chatMidTurn interface { + SteerPrompt(ctx context.Context, sessionID, text, messageID string, files []opencode.PromptFile) (bool, error) +} + +// chatAdmission is the outcome of offering one prompt to a session. +type chatAdmission int + +const ( + chatAdmitRun chatAdmission = iota + chatAdmitQueued + chatAdmitSteered + chatAdmitRefused +) + +type chatStoredFile struct { + URI string `json:"uri"` + Name string `json:"name"` +} + +// admitPrompt starts p when the session is idle, forwards it into the running +// turn when the harness supports mid-turn input, and otherwise queues it +// (durably when a store is configured). It never blocks on the turn itself. +func (h *chatHub) admitPrompt(ctx context.Context, sessionID string, broker chatBroker, p *chatQueuedPrompt) (chatAdmission, error) { + h.mu.Lock() + room := h.roomLocked(sessionID) + if room.broker == nil { + room.broker = broker + } + busy := room.running || len(room.queue) > 0 || room.observedTurn + queuedAhead := len(room.queue) > 0 + h.mu.Unlock() + if !busy { + busy = brokerTurnRunning(broker, sessionID) + } + if !busy { + h.mu.Lock() + room.queue = append(room.queue, p) + signalWake(room.wake) + h.mu.Unlock() + return chatAdmitRun, nil + } + if steerer, ok := broker.(chatMidTurn); ok && !queuedAhead { + delivered, err := steerer.SteerPrompt(ctx, sessionID, p.text, p.nativeID, p.files) + if err != nil { + return chatAdmitRefused, err + } + if delivered { + releaseChatAttachments(p.files) + return chatAdmitSteered, nil + } + } + p.acked = true + p.queuedAt = h.clock() + if store := h.queueStore; store != nil { + row, created, err := store.EnqueueChatPrompt(ctx, chatQueueRow(sessionID, p), chatRoomQueueLimit) + if errors.Is(err, pipeline.ErrChatQueueFull) { + return chatAdmitRefused, errChatQueueFull(sessionID) + } + if err != nil { + return chatAdmitRefused, fmt.Errorf("queue prompt: %w", err) + } + if !created { + // A retry of a prompt the server already holds: acknowledge it as + // queued again without a second entry. + releaseChatAttachments(p.files) + return chatAdmitQueued, nil + } + p.durable = true + p.queuedAt = row.CreatedAt + } + h.mu.Lock() + if !p.durable && len(room.queue) >= chatRoomQueueLimit { + h.mu.Unlock() + return chatAdmitRefused, errChatQueueFull(sessionID) + } + room.queue = append(room.queue, p) + signalWake(room.wake) + h.mu.Unlock() + h.fanoutQueue(sessionID, protocol.ChatQueueFrame{}) + return chatAdmitQueued, nil +} + +func errChatQueueFull(sessionID string) error { + return errors.New("session " + sessionID + " already has the maximum number of prompts queued") +} + +// brokerTurnRunning asks a harness that knows its own turn state without I/O. +func brokerTurnRunning(broker chatBroker, sessionID string) bool { + reporter, ok := broker.(chatTurnReporter) + return ok && reporter.TurnRunning(sessionID) +} + +func chatQueueRow(sessionID string, p *chatQueuedPrompt) pipeline.ChatQueuedPrompt { + files := make([]chatStoredFile, 0, len(p.files)) + for _, file := range p.files { + files = append(files, chatStoredFile{URI: file.URI, Name: file.Name}) + } + filesJSON, _ := json.Marshal(files) + parts := p.parts + if parts == nil { + parts = []protocol.ChatPart{} + } + partsJSON, _ := json.Marshal(parts) + return pipeline.ChatQueuedPrompt{ + SessionID: sessionID, + OperationID: p.operationID, + MessageID: p.messageID, + NativeID: p.nativeID, + Text: p.text, + Client: p.client, + FilesJSON: string(filesJSON), + PartsJSON: string(partsJSON), + CreatedAt: p.queuedAt, + } +} + +// promptFromRow rebuilds a queued prompt read back from the store. Its files +// must still name the server's own attachment directory; they are leased again +// so retention cannot collect them before delivery. +func promptFromRow(row pipeline.ChatQueuedPrompt) (*chatQueuedPrompt, error) { + p := &chatQueuedPrompt{ + text: row.Text, + operationID: row.OperationID, + messageID: row.MessageID, + nativeID: row.NativeID, + client: row.Client, + acked: true, + durable: true, + queuedAt: row.CreatedAt, + } + _ = json.Unmarshal([]byte(row.PartsJSON), &p.parts) + var stored []chatStoredFile + if err := json.Unmarshal([]byte(row.FilesJSON), &stored); err != nil { + return p, fmt.Errorf("queued prompt attachments are unreadable") + } + if len(stored) == 0 { + return p, nil + } + dir, err := chatAttachmentDir() + if err != nil { + return p, err + } + chatAttachmentStore.Lock() + defer chatAttachmentStore.Unlock() + for _, file := range stored { + u, err := url.Parse(file.URI) + if err != nil || u.Scheme != "file" || filepath.Dir(u.Path) != dir { + return p, fmt.Errorf("queued prompt attachment %q is outside the attachment store", file.Name) + } + chatAttachmentStore.leased[u.Path] = true + p.files = append(p.files, opencode.NewPreparedPromptFile(file.URI, file.Name)) + } + return p, nil +} + +// queueFrame is the session's current server-held queue. A live room is the +// authority while it exists; otherwise the store answers. +func (h *chatHub) queueFrame(sessionID string) protocol.ChatQueueFrame { + frame := protocol.ChatQueueFrame{SessionID: sessionID, Prompts: []protocol.ChatQueuedPrompt{}} + h.mu.Lock() + room := h.rooms[sessionID] + if room != nil { + for _, p := range room.queue { + if p.acked { + frame.Prompts = append(frame.Prompts, p.wire()) + } + } + h.mu.Unlock() + return frame + } + h.mu.Unlock() + if h.queueStore == nil { + return frame + } + rows, err := h.queueStore.ChatQueuedPrompts(h.baseCtx, sessionID) + if err != nil { + log.Printf("chat queue read session=%s: %v", sessionID, err) + return frame + } + for _, row := range rows { + p := &chatQueuedPrompt{text: row.Text, operationID: row.OperationID, messageID: row.MessageID, nativeID: row.NativeID, client: row.Client, queuedAt: row.CreatedAt} + _ = json.Unmarshal([]byte(row.PartsJSON), &p.parts) + frame.Prompts = append(frame.Prompts, p.wire()) + } + return frame +} + +func (p *chatQueuedPrompt) wire() protocol.ChatQueuedPrompt { + out := protocol.ChatQueuedPrompt{ + OperationID: p.operationID, + MessageID: p.messageID, + NativeMessageID: p.nativeID, + Text: p.text, + Client: p.client, + Attachments: p.parts, + } + if !p.queuedAt.IsZero() { + out.QueuedAt = p.queuedAt.UnixMilli() + } + return out +} + +// fanoutQueue pushes the session's current queue to every attached client, +// naming the change in change's Delivered/Removed/Failed/Error fields. +func (h *chatHub) fanoutQueue(sessionID string, change protocol.ChatQueueFrame) { + frame := h.queueFrame(sessionID) + frame.Delivered, frame.Removed, frame.Failed, frame.Error = change.Delivered, change.Removed, change.Failed, change.Error + h.fanout(sessionID, protocol.Message{Type: protocol.ChatQueue, ChatQueue: &frame}, "") +} + +// forgetQueued drops a delivered or failed prompt's durable row. +func (h *chatHub) forgetQueued(sessionID string, p *chatQueuedPrompt) { + if !p.durable || h.queueStore == nil { + return + } + if _, err := h.queueStore.DeleteChatQueuedPrompt(h.baseCtx, sessionID, p.operationID); err != nil { + log.Printf("chat queue delete session=%s: %v", sessionID, err) + } +} + +// unqueue removes one prompt still waiting in the session's queue. It reports +// false when the session holds no such prompt (it was delivered already). +func (h *chatHub) unqueue(ctx context.Context, sessionID, operationID string) (bool, error) { + var removed *chatQueuedPrompt + h.mu.Lock() + room := h.rooms[sessionID] + if room != nil { + // The live room is authoritative: a prompt its runner already took is + // being delivered and is no longer removable, even though its durable row + // is only deleted once the harness accepted it. + for i, p := range room.queue { + if p.acked && p.operationID == operationID { + removed = p + room.queue = append(room.queue[:i:i], room.queue[i+1:]...) + break + } + } + } + h.mu.Unlock() + found := removed != nil + if removed != nil { + releaseChatAttachments(removed.files) + } + if h.queueStore != nil && (room == nil || removed != nil) { + deleted, err := h.queueStore.DeleteChatQueuedPrompt(ctx, sessionID, operationID) + if err != nil { + return found, err + } + found = found || deleted + } + if found { + h.forgetOp(sessionID, operationID) + h.fanoutQueue(sessionID, protocol.ChatQueueFrame{Removed: operationID}) + } + return found, nil +} + +// clearQueue drops every prompt a closed session still holds. +func (h *chatHub) clearQueue(ctx context.Context, sessionID string) { + var dropped []*chatQueuedPrompt + h.mu.Lock() + if room := h.rooms[sessionID]; room != nil { + kept := room.queue[:0] + for _, p := range room.queue { + if p.acked { + dropped = append(dropped, p) + } else { + kept = append(kept, p) + } + } + room.queue = kept + } + h.mu.Unlock() + for _, p := range dropped { + releaseChatAttachments(p.files) + } + if h.queueStore != nil { + if err := h.queueStore.ClearChatQueuedPrompts(ctx, sessionID); err != nil { + log.Printf("chat queue clear session=%s: %v", sessionID, err) + } + } + if len(dropped) > 0 { + h.fanoutQueue(sessionID, protocol.ChatQueueFrame{}) + } +} + +// resumeQueued restores every session's durable queue after a restart: each +// session gets a room (with no client attached) whose runner delivers the +// prompts in order once the harness is idle. +func (h *chatHub) resumeQueued(ctx context.Context, broker chatBroker) { + if h.queueStore == nil || broker == nil { + return + } + sessions, err := h.queueStore.ChatQueuedSessions(ctx) + if err != nil { + log.Printf("chat queue recovery: %v", err) + return + } + for _, sessionID := range sessions { + rows, err := h.queueStore.ChatQueuedPrompts(ctx, sessionID) + if err != nil { + log.Printf("chat queue recovery session=%s: %v", sessionID, err) + continue + } + var prompts []*chatQueuedPrompt + for _, row := range rows { + p, err := promptFromRow(row) + if err != nil { + releaseChatAttachments(p.files) + log.Printf("chat queue recovery session=%s: %v", sessionID, err) + _, _ = h.queueStore.DeleteChatQueuedPrompt(ctx, sessionID, row.OperationID) + continue + } + prompts = append(prompts, p) + } + if len(prompts) == 0 { + continue + } + h.mu.Lock() + room := h.roomLocked(sessionID) + if room.broker == nil { + room.broker = broker + } + room.notBefore = time.Now().Add(chatQueueRecoverySettle) + for _, p := range prompts { + if !room.holdsOp(p.operationID) { + room.queue = append(room.queue, p) + h.rememberOpLocked(sessionID, p.operationID) + } + } + h.mu.Unlock() + go func(room *chatRoom) { + // Attach the stream first so a turn the harness is still running is + // observed before the settle window lets the runner deliver. + if err := h.ensureObserving(room, broker); err != nil && room.ctx.Err() == nil { + log.Printf("chat queue recovery observe session=%s: %v", room.sessionID, err) + } + h.mu.Lock() + signalWake(room.wake) + h.mu.Unlock() + }(room) + } +} + +func (r *chatRoom) holdsOp(operationID string) bool { + for _, p := range r.queue { + if p.operationID == operationID { + return true + } + } + return false +} + +// duplicateOp reports whether operationID is a prompt this session already +// accepted (queued reports it is still waiting in the queue), so a client retry +// is acknowledged again instead of delivered twice. +func (h *chatHub) duplicateOp(sessionID, operationID string) (duplicate, queued bool) { + if operationID == "" { + return false, false + } + h.mu.Lock() + defer h.mu.Unlock() + if room := h.rooms[sessionID]; room != nil { + for _, p := range room.queue { + if p.operationID == operationID { + return true, p.acked + } + } + } + _, seen := h.recentOps[sessionID+"\x00"+operationID] + return seen, false +} + +func (h *chatHub) rememberOp(sessionID, operationID string) { + h.mu.Lock() + defer h.mu.Unlock() + h.rememberOpLocked(sessionID, operationID) +} + +func (h *chatHub) rememberOpLocked(sessionID, operationID string) { + if operationID == "" { + return + } + key := sessionID + "\x00" + operationID + if h.recentOps == nil { + h.recentOps = map[string]struct{}{} + } + if _, ok := h.recentOps[key]; ok { + return + } + h.recentOps[key] = struct{}{} + h.recentOpOrder = append(h.recentOpOrder, key) + for len(h.recentOpOrder) > chatRecentOpsLimit { + delete(h.recentOps, h.recentOpOrder[0]) + h.recentOpOrder = h.recentOpOrder[1:] + } +} + +// forgetOp lets a prompt that never reached the harness be retried. +func (h *chatHub) forgetOp(sessionID, operationID string) { + h.mu.Lock() + defer h.mu.Unlock() + delete(h.recentOps, sessionID+"\x00"+operationID) +} + +// handleChatUnqueue removes one queued prompt and replies with the queue. +func handleChatUnqueue(ctx context.Context, request protocol.Message, hub *chatHub, write func(protocol.Message) error) { + body := request.ChatUnqueue + if body == nil || !validChatSessionID(body.SessionID) || body.OperationID == "" { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "a valid session id and operation id are required"}) + return + } + found, err := hub.unqueue(ctx, body.SessionID, body.OperationID) + if err != nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) + return + } + if !found { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "that message is no longer queued; it was already sent"}) + return + } + frame := hub.queueFrame(body.SessionID) + frame.Removed = body.OperationID + _ = write(protocol.Message{Type: protocol.ChatUnqueue, ID: request.ID, ChatQueue: &frame}) +} diff --git a/internal/server/chat_queue_test.go b/internal/server/chat_queue_test.go new file mode 100644 index 00000000..7d83f76d --- /dev/null +++ b/internal/server/chat_queue_test.go @@ -0,0 +1,263 @@ +package server + +import ( + "context" + "net/http/httptest" + "path/filepath" + "slices" + "sync" + "testing" + "time" + + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" + + "git.a-wan.net/alan/pipelines/internal/opencode" + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// steeringChat is a channelChat whose harness takes input mid-turn, the way +// Claude Code's stream-json stdin and Codex turn/steer do. +type steeringChat struct { + *channelChat + steerMu sync.Mutex + steered []string +} + +func (c *steeringChat) SteerPrompt(_ context.Context, _, text, _ string, _ []opencode.PromptFile) (bool, error) { + c.steerMu.Lock() + defer c.steerMu.Unlock() + c.steered = append(c.steered, text) + return true, nil +} + +type queueServer struct { + ctx context.Context + cancel context.CancelFunc + srv *httptest.Server +} + +func startQueueServer(t *testing.T, broker chatBroker, store *pipeline.Store) *queueServer { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: broker}, store)) + s := &queueServer{ctx: ctx, cancel: cancel, srv: srv} + t.Cleanup(s.stop) + return s +} + +func (s *queueServer) stop() { + s.srv.CloseClientConnections() + s.srv.Close() + s.cancel() +} + +func openQueueStore(t *testing.T, path string) *pipeline.Store { + t.Helper() + store, err := pipeline.Open(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { store.Close() }) + return store +} + +func queuedOps(t *testing.T, store *pipeline.Store) []string { + t.Helper() + rows, err := store.ChatQueuedPrompts(context.Background(), "ses_1") + if err != nil { + t.Fatal(err) + } + var ops []string + for _, row := range rows { + ops = append(ops, row.OperationID) + } + return ops +} + +func sendPrompt(t *testing.T, ctx context.Context, conn *websocket.Conn, id, text, operation string) protocol.Message { + t.Helper() + writeChat(t, ctx, conn, protocol.Message{Type: protocol.ChatPrompt, ID: id, ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: text, OperationID: operation, Client: "Phone"}}) + return readUntil(t, ctx, conn, protocol.ChatPrompt) +} + +func waitFor(t *testing.T, what string, check func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if check() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +func endTurn(broker *channelChat) { + broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepEnded}}) +} + +// A prompt sent while a turn runs on a harness with mid-turn input is handed +// to that turn at once: it is acknowledged as sent, never queued or stored. +func TestChatPromptDuringTurnIsSteeredImmediately(t *testing.T) { + store := openQueueStore(t, filepath.Join(t.TempDir(), "queue.db")) + broker := &steeringChat{channelChat: newChannelChat()} + server := startQueueServer(t, broker, store) + conn := dialChatConn(t, server.ctx, server.srv) + if ack := sendPrompt(t, server.ctx, conn, "p1", "one", "op-1"); ack.ChatQueued { + t.Fatalf("first prompt queued: %+v", ack) + } + waitFor(t, "first prompt", func() bool { return broker.promptCount() == 1 }) + if ack := sendPrompt(t, server.ctx, conn, "p2", "two", "op-2"); ack.ChatQueued { + t.Fatalf("mid-turn prompt was queued instead of forwarded: %+v", ack) + } + broker.steerMu.Lock() + steered := slices.Clone(broker.steered) + broker.steerMu.Unlock() + if !slices.Equal(steered, []string{"two"}) || broker.promptCount() != 1 { + t.Fatalf("steered=%v prompts=%v", steered, broker.promptTexts()) + } + if ops := queuedOps(t, store); len(ops) != 0 { + t.Fatalf("steered prompt was stored: %v", ops) + } + // A retry of the same operation is acknowledged, never forwarded again. + if ack := sendPrompt(t, server.ctx, conn, "p2-retry", "two", "op-2"); ack.ChatQueued { + t.Fatalf("retry ack = %+v", ack) + } + broker.steerMu.Lock() + defer broker.steerMu.Unlock() + if len(broker.steered) != 1 { + t.Fatalf("retry steered again: %v", broker.steered) + } +} + +// Without mid-turn input the prompt is queued on the server: it is stored +// once (a retry of the same operation id is not a second entry), outlives the +// last client leaving, and is delivered when the turn ends with nobody +// watching. +func TestChatQueuedPromptSurvivesDetachAndDeliversUnwatched(t *testing.T) { + store := openQueueStore(t, filepath.Join(t.TempDir(), "queue.db")) + broker := newChannelChat() + server := startQueueServer(t, broker, store) + conn := dialChatConn(t, server.ctx, server.srv) + sendPrompt(t, server.ctx, conn, "p1", "one", "op-1") + if ack := sendPrompt(t, server.ctx, conn, "p2", "two", "op-2"); !ack.ChatQueued { + t.Fatalf("second prompt not queued: %+v", ack) + } + if ack := sendPrompt(t, server.ctx, conn, "p2-retry", "two", "op-2"); !ack.ChatQueued { + t.Fatalf("retry of a queued prompt not acknowledged as queued: %+v", ack) + } + if ops := queuedOps(t, store); !slices.Equal(ops, []string{"op-2"}) { + t.Fatalf("stored queue = %v, want op-2 once", ops) + } + conn.CloseNow() + time.Sleep(100 * time.Millisecond) // let the server notice the disconnect + endTurn(broker) + waitFor(t, "queued prompt delivered with no client", func() bool { + return slices.Equal(broker.promptTexts(), []string{"one", "two"}) + }) + waitFor(t, "delivered row deleted", func() bool { return len(queuedOps(t, store)) == 0 }) +} + +// The durable queue survives a server restart: the reopened store's prompts +// are delivered once the harness is idle, with no client connected. +func TestChatQueuedPromptSurvivesRestart(t *testing.T) { + previous := chatQueueRecoverySettle + chatQueueRecoverySettle = 10 * time.Millisecond + t.Cleanup(func() { chatQueueRecoverySettle = previous }) + path := filepath.Join(t.TempDir(), "queue.db") + first := openQueueStore(t, path) + broker := newChannelChat() + server := startQueueServer(t, broker, first) + conn := dialChatConn(t, server.ctx, server.srv) + sendPrompt(t, server.ctx, conn, "p1", "one", "op-1") + if ack := sendPrompt(t, server.ctx, conn, "p2", "two", "op-2"); !ack.ChatQueued { + t.Fatalf("second prompt not queued: %+v", ack) + } + server.stop() + first.Close() + + reopened := openQueueStore(t, path) + if ops := queuedOps(t, reopened); !slices.Equal(ops, []string{"op-2"}) { + t.Fatalf("queue after restart = %v", ops) + } + restarted := newChannelChat() + startQueueServer(t, restarted, reopened) + waitFor(t, "recovered prompt delivered", func() bool { + return slices.Equal(restarted.promptTexts(), []string{"two"}) + }) + waitFor(t, "recovered row deleted", func() bool { return len(queuedOps(t, reopened)) == 0 }) +} + +// A client that attaches later (a reload) is told what the server still holds +// in chat_subscribe, can remove a queued prompt, and a removed prompt is never +// delivered. +func TestChatSubscribeListsQueuedPromptsAndUnqueueRemoves(t *testing.T) { + store := openQueueStore(t, filepath.Join(t.TempDir(), "queue.db")) + broker := newChannelChat() + server := startQueueServer(t, broker, store) + phone := dialChatConn(t, server.ctx, server.srv) + sendPrompt(t, server.ctx, phone, "p1", "one", "op-1") + sendPrompt(t, server.ctx, phone, "p2", "two", "op-2") + sendPrompt(t, server.ctx, phone, "p3", "three", "op-3") + phone.CloseNow() + + reloaded := dialChatConn(t, server.ctx, server.srv) + writeChat(t, server.ctx, reloaded, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}) + ack := readUntil(t, server.ctx, reloaded, protocol.ChatSubscribe) + if ack.ChatQueue == nil || len(ack.ChatQueue.Prompts) != 2 { + t.Fatalf("subscribe queue = %+v", ack.ChatQueue) + } + got := ack.ChatQueue.Prompts[0] + if got.OperationID != "op-2" || got.Text != "two" || got.Client != "Phone" || got.MessageID == "" || ack.ChatQueue.Prompts[1].OperationID != "op-3" { + t.Fatalf("queued prompts = %+v", ack.ChatQueue.Prompts) + } + writeChat(t, server.ctx, reloaded, protocol.Message{Type: protocol.ChatUnqueue, ID: "rm", ChatUnqueue: &protocol.ChatUnqueueRequest{SessionID: "ses_1", OperationID: "op-2"}}) + removed := readUntil(t, server.ctx, reloaded, protocol.ChatUnqueue) + if removed.ChatQueue == nil || removed.ChatQueue.Removed != "op-2" || len(removed.ChatQueue.Prompts) != 1 { + t.Fatalf("unqueue reply = %+v", removed.ChatQueue) + } + if ops := queuedOps(t, store); !slices.Equal(ops, []string{"op-3"}) { + t.Fatalf("stored queue after unqueue = %v", ops) + } + endTurn(broker) + // Delivery re-publishes the user turn and reports it sent to the client + // that attached after it was queued. + sawUser := false + for { + var message protocol.Message + if err := wsjson.Read(server.ctx, reloaded, &message); err != nil { + t.Fatalf("no delivery reported: %v", err) + } + if event := message.ChatEvent; event != nil && event.Kind == protocol.ChatEventUser && event.OperationID == "op-3" { + sawUser = true + } + if message.Type == protocol.ChatQueue && message.ChatQueue.Delivered != "" { + if message.ChatQueue.Delivered != "op-3" || len(message.ChatQueue.Prompts) != 0 || !sawUser { + t.Fatalf("delivery frame = %+v (user turn seen: %v)", message.ChatQueue, sawUser) + } + break + } + } + if texts := broker.promptTexts(); !slices.Equal(texts, []string{"one", "three"}) { + t.Fatalf("delivered prompts = %v, want the removed one skipped", texts) + } + if ops := queuedOps(t, store); len(ops) != 0 { + t.Fatalf("stored queue after delivery = %v", ops) + } + // A prompt already delivered can no longer be removed. + writeChat(t, server.ctx, reloaded, protocol.Message{Type: protocol.ChatUnqueue, ID: "rm-late", ChatUnqueue: &protocol.ChatUnqueueRequest{SessionID: "ses_1", OperationID: "op-3"}}) + for { + var message protocol.Message + if err := wsjson.Read(server.ctx, reloaded, &message); err != nil { + t.Fatal(err) + } + if message.ID == "rm-late" { + if message.Type != protocol.Error { + t.Fatalf("late unqueue = %+v", message) + } + break + } + } +} diff --git a/internal/server/chat_room.go b/internal/server/chat_room.go index 844e0687..09e0148d 100644 --- a/internal/server/chat_room.go +++ b/internal/server/chat_room.go @@ -101,14 +101,25 @@ func (s *chatSub) close() { // turn runner does not send a second (success) acknowledgement for the same // request id; write is kept either way so a later broker failure is still // reported to the client that sent it. +// +// durable marks a prompt written to the chat_prompt_queue store: it outlives +// the connection that sent it (write may be nil after a restart) and its row is +// deleted once the harness accepted it. messageID, client and parts let the +// server re-publish the user turn when it is finally delivered, so a client +// that attached after it was queued still sees it enter the transcript. type chatQueuedPrompt struct { text string files []opencode.PromptFile operationID string nativeID string + messageID string + client string + parts []protocol.ChatPart requestID string write func(protocol.Message) error acked bool + durable bool + queuedAt time.Time } type chatObservation struct { @@ -153,6 +164,9 @@ type chatRoom struct { wake chan struct{} turnDone chan struct{} nativeTurnID string + // notBefore holds back a recovered queue until the restarted stream had a + // chance to report a turn the harness is still running. + notBefore time.Time } func newChatRoom(base context.Context, sessionID string) *chatRoom { @@ -295,8 +309,11 @@ func (h *chatHub) attach(sessionID, clientID, label string, broker chatBroker, w } // detach drops one subscription. The room (and its broker stream and queue) is -// released once its last client leaves, so a session with no watchers holds no -// goroutine. +// released once its last client leaves and it has no prompt left to deliver, +// so a session with no watchers holds no goroutine. A room holding queued +// prompts stays (with its stream) until its runner has delivered them: the +// queue belongs to the session, not to the clients watching it. A running turn +// with nothing queued behind it needs no room; the harness finishes it alone. func (h *chatHub) detach(sessionID string, sub *chatSub) { h.mu.Lock() room := h.rooms[sessionID] @@ -305,19 +322,27 @@ func (h *chatHub) detach(sessionID string, sub *chatSub) { return } delete(room.subs, sub.clientID) - if len(room.subs) == 0 { - h.clearCompactionLocked(room) - delete(h.rooms, sessionID) - room.cancel() - // ReleaseShells is a local identity-cache operation (no broker I/O). - // Complete it before a reconnect can acquire this session's new room. - if releaser, ok := room.broker.(chatShellReleaser); ok { - releaser.ReleaseShells(sessionID) - } + if len(room.subs) == 0 && len(room.queue) == 0 { + h.releaseRoomLocked(room) } h.mu.Unlock() } +// releaseRoomLocked destroys an unwatched, idle room. Callers hold h.mu. +func (h *chatHub) releaseRoomLocked(room *chatRoom) { + if h.rooms[room.sessionID] != room { + return + } + h.clearCompactionLocked(room) + delete(h.rooms, room.sessionID) + room.cancel() + // ReleaseShells is a local identity-cache operation (no broker I/O). + // Complete it before a reconnect can acquire this session's new room. + if releaser, ok := room.broker.(chatShellReleaser); ok { + releaser.ReleaseShells(room.sessionID) + } +} + // detachClient drops every chat subscription one connection holds; the read loop // calls it when the connection ends so no subscriber goroutine outlives it. func (h *chatHub) detachClient(clientID string) { @@ -530,6 +555,8 @@ func (h *chatHub) finishTurnLocked(room *chatRoom) { close(room.turnDone) room.turnDone = nil } + // A queue held behind a turn this room only observed advances now. + signalWake(room.wake) } // turnInFlight reports whether sessionID has a turn in flight, so chat_close @@ -576,32 +603,6 @@ func (h *chatHub) flushBuffered(sessionID string) { } } -// submitPrompt accepts one prompt: it starts the turn runner immediately when no -// turn is in flight, or queues it (bounded) when one is. It returns whether the -// prompt was queued or refused. -func (h *chatHub) submitPrompt(sessionID string, p *chatQueuedPrompt) (queued bool, refused bool) { - h.mu.Lock() - room := h.roomLocked(sessionID) - if room.running || len(room.queue) > 0 { - if len(room.queue) >= chatRoomQueueLimit { - h.mu.Unlock() - return false, true - } - // The client is acknowledged as queued by the handler, so the turn runner - // must not acknowledge the same request again once it runs. The write is - // still kept so a later broker failure reaches the client that sent it. - p.acked = true - room.queue = append(room.queue, p) - signalWake(room.wake) - h.mu.Unlock() - return true, false - } - room.queue = append(room.queue, p) - signalWake(room.wake) - h.mu.Unlock() - return false, false -} - // turnRunning reports whether the session has a turn in flight. A client that // subscribes after a turn started (for example the operator's browser while the // CLI harness is working) needs this to show Stop and the live activity line; @@ -707,30 +708,68 @@ func (h *chatHub) runRoom(room *chatRoom) { return } h.runTurn(room, p, done) + // The last client may have left while this turn ran; an unwatched room + // stays only while it still has prompts to deliver. + h.mu.Lock() + if len(room.subs) == 0 && !room.running && len(room.queue) == 0 { + h.releaseRoomLocked(room) + } + h.mu.Unlock() } } // nextPrompt pops the next queued prompt, blocking until one arrives or the room -// is released. It marks the room running and creates the turn's completion -// channel so a stale signal from a previous turn can never end the next one. +// is released. A queued prompt also waits while the harness is still running a +// turn this room did not start (one announced by the stream, or one a harness +// reports in flight after a reload or restart). It marks the room running and +// creates the turn's completion channel so a stale signal from a previous turn +// can never end the next one. func (h *chatHub) nextPrompt(room *chatRoom) (*chatQueuedPrompt, chan struct{}, bool) { for { h.mu.Lock() - if len(room.queue) > 0 { - p := room.queue[0] - room.queue = room.queue[1:] - room.running = true - room.nativeTurnID = "" - room.turnDone = make(chan struct{}) - done := room.turnDone + if room.ctx.Err() != nil { h.mu.Unlock() - return p, done, true + return nil, nil, false + } + var wait time.Duration + if len(room.queue) > 0 { + switch { + case room.observedTurn: + wait = chatQueueBusyPoll + case time.Until(room.notBefore) > 0: + wait = time.Until(room.notBefore) + default: + broker := room.broker + h.mu.Unlock() + busy := brokerTurnRunning(broker, room.sessionID) + h.mu.Lock() + if busy { + wait = chatQueueBusyPoll + } else if len(room.queue) > 0 && !room.observedTurn && room.ctx.Err() == nil { + p := room.queue[0] + room.queue = room.queue[1:] + room.running = true + room.nativeTurnID = "" + room.turnDone = make(chan struct{}) + done := room.turnDone + h.mu.Unlock() + return p, done, true + } else { + h.mu.Unlock() + continue + } + } } h.mu.Unlock() + var timer <-chan time.Time + if wait > 0 { + timer = time.After(wait) + } select { case <-room.ctx.Done(): return nil, nil, false case <-room.wake: + case <-timer: } } } @@ -751,6 +790,7 @@ func (h *chatHub) runTurn(room *chatRoom, p *chatQueuedPrompt, done chan struct{ if broker == nil { h.failPrompt(room, p, errors.New("chat is not configured on this server")) + h.endRoomTurn(room) return } if first { @@ -760,6 +800,7 @@ func (h *chatHub) runTurn(room *chatRoom, p *chatQueuedPrompt, done chan struct{ return } } + h.publishDelivery(room, p) var promptErr error if withFiles, ok := broker.(chatPromptWithFiles); ok && len(p.files) > 0 { promptErr = withFiles.PromptWithMessageIDFiles(room.ctx, room.sessionID, p.text, p.nativeID, p.files) @@ -781,6 +822,13 @@ func (h *chatHub) runTurn(room *chatRoom, p *chatQueuedPrompt, done chan struct{ if p.write != nil && !p.acked { _ = p.write(protocol.Message{Type: protocol.ChatPrompt, ID: p.requestID}) } + if p.acked { + // The harness has it now: drop the durable row and tell every client the + // queued prompt was sent. + h.rememberOp(room.sessionID, p.operationID) + h.forgetQueued(room.sessionID, p) + h.fanoutQueue(room.sessionID, protocol.ChatQueueFrame{Delivered: p.operationID}) + } h.flushBuffered(room.sessionID) // Start the approval poller once, after the first ack, so a pending request // is surfaced after the prompt is acknowledged and shared with every client. @@ -804,13 +852,51 @@ func (h *chatHub) endRoomTurn(room *chatRoom) { // that never reached the agent is visible on the device that sent it rather than // leaving its composer waiting. It is reported for a queued prompt too, whose // success ack was already sent but whose delivery may still fail. +// +// A queued prompt's durable row is dropped (the harness refused it, so it is +// not retried blindly) and every attached client is told through chat_queue, +// including a client that attached after the sender left. func (h *chatHub) failPrompt(room *chatRoom, p *chatQueuedPrompt, err error) { - if p.write == nil || err == nil { + if err == nil { + return + } + h.forgetOp(room.sessionID, p.operationID) + if p.acked { + h.forgetQueued(room.sessionID, p) + h.fanoutQueue(room.sessionID, protocol.ChatQueueFrame{Failed: p.operationID, Error: err.Error()}) + } + if p.write == nil { return } _ = p.write(protocol.Message{Type: protocol.Error, ID: p.requestID, Error: err.Error()}) } +// publishDelivery re-publishes a queued prompt's user turn as it is handed to +// the harness, so a client that attached after it was queued (a reload, or the +// sender's other device) sees it enter the transcript before the reply. Clients +// that already hold it recognise the same message and operation ids. +func (h *chatHub) publishDelivery(room *chatRoom, p *chatQueuedPrompt) { + if !p.acked || p.messageID == "" { + return + } + frame := protocol.ChatEventFrame{ + SessionID: room.sessionID, + MessageID: p.messageID, + Kind: protocol.ChatEventUser, + Part: &protocol.ChatPart{Type: protocol.ChatPartText, ID: "usr", Text: p.text}, + Attachments: p.parts, + Client: p.client, + OperationID: p.operationID, + } + h.mu.Lock() + turn := h.turnLocked(room.sessionID) + if _, retained := turn.messages[p.messageID]; !retained { + h.applyLocked(frame) + } + h.mu.Unlock() + h.fanout(room.sessionID, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}, "") +} + // roomBroker returns the broker bound to the room. func (h *chatHub) roomBroker(room *chatRoom) chatBroker { h.mu.Lock() diff --git a/internal/server/chat_stream.go b/internal/server/chat_stream.go index 502e5150..43363c3c 100644 --- a/internal/server/chat_stream.go +++ b/internal/server/chat_stream.go @@ -59,6 +59,18 @@ func handleChatPrompt(ctx context.Context, request protocol.Message, broker chat return } } + operationID := prompt.OperationID + if operationID == "" { + // Every server-held prompt is addressable (chat_unqueue) and unique in + // its session's durable queue, so mint an id for a client that sent none. + operationID = "srv-" + strings.TrimPrefix(chatClientToken(), "conn-") + } else if duplicate, queued := hub.duplicateOp(prompt.SessionID, operationID); duplicate { + // A retry of a prompt this session already accepted: acknowledge it the + // same way again instead of delivering it a second time. + hub.attach(prompt.SessionID, clientID, sanitizeChatClientLabel(prompt.Client), broker, write) + _ = write(protocol.Message{Type: protocol.ChatPrompt, ID: request.ID, ChatQueued: queued}) + return + } attachmentFiles, attachmentParts, err := prepareChatAttachments(prompt.SessionID, prompt.Attachments) if err != nil { _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) @@ -85,29 +97,43 @@ func handleChatPrompt(ctx context.Context, request protocol.Message, broker chat Part: &protocol.ChatPart{Type: protocol.ChatPartText, ID: "usr", Text: prompt.Text}, Attachments: attachmentParts, Client: sub.label, - OperationID: prompt.OperationID, + OperationID: operationID, } if userFrame.MessageID == "" { userFrame.MessageID = hub.nextUserMessageID() } - queued, refused := hub.submitPrompt(sessionID, &chatQueuedPrompt{ + queuedPrompt := &chatQueuedPrompt{ text: prompt.Text, files: attachmentFiles, - operationID: prompt.OperationID, + operationID: operationID, // Only the caller's real OpenCode message id is forwarded. The hub's own // `usr_` display id is not a valid OpenCode `msg_` id, so a client that // omitted native_message_id lets OpenCode generate one instead. nativeID: prompt.NativeMessageID, + messageID: userFrame.MessageID, + client: sub.label, + parts: attachmentParts, requestID: request.ID, write: write, - }) - if refused { - _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "session " + sessionID + " already has the maximum number of prompts queued"}) + } + // Claim the operation before admission so a concurrent retry of the same + // prompt is acknowledged rather than delivered twice. + hub.rememberOp(sessionID, operationID) + admission, admitErr := hub.admitPrompt(ctx, sessionID, broker, queuedPrompt) + if admission == chatAdmitRefused { + hub.forgetOp(sessionID, operationID) + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: admitErr.Error()}) return } accepted = true + queued := admission == chatAdmitQueued // Publish the user turn to every other client; the sender already has it. hub.broadcastUser(userFrame, clientID) + if admission == chatAdmitSteered { + // The harness took the message into its running turn: it is sent, not + // queued, and the turn already in flight carries its reply. + _ = write(protocol.Message{Type: protocol.ChatPrompt, ID: request.ID}) + } if queued { // The client that sent it is told the prompt is waiting for the in-flight // turn, so it shows queued/steering instead of blocking or silently @@ -200,7 +226,10 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH } // Acknowledge before the stream is allowed to fan out: otherwise an immediately // available retained shell can overtake the subscribe reply on this connection. - _ = write(protocol.Message{Type: protocol.ChatSubscribe, ID: request.ID}) + // The reply carries the prompts the server still holds for the session, so + // a reloaded client draws them as Queued without any device storage. + queue := hub.queueFrame(query.SessionID) + _ = write(protocol.Message{Type: protocol.ChatSubscribe, ID: request.ID, ChatQueue: &queue}) if err := hub.ensureObserving(sub.room, broker); err != nil { sub.close() _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not observe session " + query.SessionID}) diff --git a/internal/server/websocket.go b/internal/server/websocket.go index a8194cfb..eecffe18 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -118,6 +118,14 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin settingsStore = stores[0] } hub.settings = settingsStore + if len(stores) > 0 { + // Prompts queued behind a running turn are durable and owned by their + // session; resume delivering any a previous process left queued. + hub.queueStore = stores[0] + if options.Chat != nil { + go hub.resumeQueued(ctx, options.Chat) + } + } acks := newAttentionAckHub() var ackStore attentionAckStore var notes *appNotifier