From 798e958cb7b0fcd1c45f66533f7ea500e1c85e2f Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 01:47:52 +0100 Subject: [PATCH 1/3] chat: fold an agent turn into one reply, split reasoning from answer, keep tool name/input The phone's chat transcript rendered one agent reply as ~36 bubbles, each with its own thinking block, and a finished tool call lost its name and its input. - Turn fold (android): ChatFormat.kt gains the pure ChatTurn / TranscriptRow / transcriptRows fold - each run of consecutive assistant messages between two of the operator's messages is one agent turn - and ChatTranscript renders one bubble per turn: the turn's single collapsible thinking block (all its reasoning, in order, growing live while it streams), then its tool-call chips, then one answer block of just the turn's text parts. AssistantParts/MessagePart are replaced by AgentTurnItem/UserMessageItem; ChatConversationPane counts the folded rows. - Delta part ids (server + client): internal/protocol/chat.go adds ChatDeltaPartID(messageID, kind, ordinal), mirrored as ChatProtocol.deltaPartId in data/protocol/Chat.kt, and NormalizeChatEvent keys a text or reasoning delta on it. OpenCode numbers a message's text and reasoning parts independently from 0, so the old "%s#%d" gave the first text delta and first reasoning part the same id: the answer was appended into the thinking block, and the server's retained state (chat_hub.go) re-collided on resume. ChatStore's upsert now routes a delta through the kind-aware deltaPartId fold, so it stays robust even against a peer that still shares one id. - Tool fields (server + client): NormalizeChatEvent takes a call's arguments from session.tool.input.ended's text as well as session.tool.called's input object (toolInput), and chat_hub.go's new mergeChatToolPart folds each tool frame onto the retained part so the progress/success frames that omit name and input never blank them. ChatFormat.kt's mergeToolPart mirrors that on the client, so the chip and panel header keep the real tool name and the panel shows INPUT beside OUTPUT. Tests: chat_v2_test.go and chat_test.go add the text/reasoning id-collision and the tool name+input retention cases; ChatReasoningTurnTest, ChatTranscriptTest and ChatRunningWorkTest cover the turn fold, the collision and the tool lifecycle. Checks: Go build, vet, gofmt and the package tests pass (go build ./... && go vet ./...; gofmt -l . empty; go test ./internal/server ./internal/opencode ./internal/protocol ./internal/stage ./internal/pipeline; go test ./...). Android passes offline with --no-daemon (assembleDebug and the full testDebugUnitTest suite, 660 tests). The on-phone observation was not performed: the device lock was held by another run and adb was not available here. Documentation inventory checked: no packaged OpenCode skill reference in internal/author/skill/references/ and no packaged template documents the app's chat transcript, this delta part-id derivation or the tool-field retention, and no internal/author/ managed file inventory entry names any touched file, so none of them needs updating. The shared android/README.md behaviour statements are made stale by this change and are recorded instead in docs/notes/chat-transcript-turn-fold.md for the docs-fold writer, per the repo rule that an item never edits a shared document itself. --- .../net/awan/pipelines/data/protocol/Chat.kt | 10 + .../pipelines/ui/chat/ChatConversationPane.kt | 11 +- .../net/awan/pipelines/ui/chat/ChatFormat.kt | 133 +++++++++++++ .../net/awan/pipelines/ui/chat/ChatStore.kt | 63 ++++--- .../awan/pipelines/ui/chat/ChatTranscript.kt | 178 +++++++----------- .../ui/chat/ChatReasoningTurnTest.kt | 72 +++++++ .../pipelines/ui/chat/ChatRunningWorkTest.kt | 45 +++++ .../pipelines/ui/chat/ChatTranscriptTest.kt | 39 ++++ docs/notes/chat-transcript-turn-fold.md | 79 ++++++++ internal/opencode/chat.go | 32 +++- internal/opencode/chat_v2_test.go | 145 ++++++++++++++ internal/protocol/chat.go | 24 ++- internal/server/chat_hub.go | 81 ++++++-- internal/server/chat_test.go | 50 ++++- 14 files changed, 801 insertions(+), 161 deletions(-) create mode 100644 docs/notes/chat-transcript-turn-fold.md diff --git a/android/app/src/main/java/net/awan/pipelines/data/protocol/Chat.kt b/android/app/src/main/java/net/awan/pipelines/data/protocol/Chat.kt index 04fbbd94..b4d84acd 100644 --- a/android/app/src/main/java/net/awan/pipelines/data/protocol/Chat.kt +++ b/android/app/src/main/java/net/awan/pipelines/data/protocol/Chat.kt @@ -277,6 +277,16 @@ object ChatProtocol { else -> false } + /** + * The stable part id an incremental text/reasoning delta stream addresses, + * mirroring `ChatDeltaPartID` in `internal/protocol/chat.go`. OpenCode numbers + * one message's text and reasoning parts independently from 0, so the kind is + * part of the id: a message's first text delta and its first reasoning delta + * are distinct parts, and answer text can never be appended into the thinking + * block. + */ + fun deltaPartId(messageId: String, kind: String, ordinal: Int): String = "$messageId#$kind#$ordinal" + /** * The `chat_prompt` frame the composer sends. [files] are the attached files' * URIs, in the order they were attached, and are omitted when empty (the diff --git a/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatConversationPane.kt b/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatConversationPane.kt index e446ccc5..9789c597 100644 --- a/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatConversationPane.kt +++ b/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatConversationPane.kt @@ -102,10 +102,12 @@ internal fun ChatConversationPane( // A turn blocked on an approval is not generating, so the running indicator // (and its ticking clock) stands down while the card is unanswered. val showRunning = thread.running && !awaiting - // The count of the transcript's items: the messages, the local command lines, - // the running indicator while a turn is in flight and a pending approval card, - // which is the last item the top bar's approval action jumps to. - val itemCount = thread.messages.size + thread.notices.size + + // The count of the transcript's items: its rows (the operator's messages and + // the folded agent replies), the local command lines, the running indicator + // while a turn is in flight and a pending approval card, which is the last + // item the top bar's approval action jumps to. + val rows = transcriptRows(thread.messages) + val itemCount = rows.size + thread.notices.size + (if (showRunning) 1 else 0) + (if (approval != null) 1 else 0) // The position the operator left this conversation at, read once: a // conversation with none of its own opens at its newest message instead (see @@ -219,6 +221,7 @@ internal fun ChatConversationPane( ) else -> ChatTranscript( thread = thread, + rows = rows, listState = listState, savedScroll = savedScroll, itemCount = itemCount, diff --git a/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatFormat.kt b/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatFormat.kt index efa83de4..1df0166c 100644 --- a/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatFormat.kt +++ b/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatFormat.kt @@ -180,6 +180,53 @@ internal fun runningToolStamps( return stamps } +/** + * The id a text or reasoning delta must be applied to for [kind] within [parts]. + * The broker gives the two kinds distinct ids (see `ChatDeltaPartID`), but a peer + * that still shares one id between them must never have answer text appended into + * a reasoning part (or the reverse): when another kind already holds the bare id, + * the delta is moved onto a suffixed id of its own so the two kinds stay in + * separate parts. Once a part of this kind exists under either id, that part is + * the one addressed, so a stream keeps growing the part it started. + */ +internal fun deltaPartId(parts: List, kind: String, id: String): String { + if (parts.any { it.type == kind && it.id == id }) return id + val suffixed = "$id#$kind" + if (parts.any { it.type == kind && it.id == suffixed }) return suffixed + return if (parts.any { it.id == id }) suffixed else id +} + +/** + * Folds one tool frame's part onto the part retained from earlier frames. A tool + * call is reported across several frames that each carry only part of it (the name + * in `input.started`, the arguments in `input.ended`'s text or `called`'s input, + * then `progress`/`success` carrying neither), so a field already learned is never + * blanked by a later frame that omits it. A non-empty incoming field wins; + * otherwise the retained value is kept. + */ +internal fun mergeToolPart(previous: ChatPart?, incoming: ChatPart): ChatPart { + if (previous == null) return incoming + val priorState = previous.state + val incomingState = incoming.state + return incoming.copy( + type = incoming.type.ifBlank { previous.type }, + name = incoming.name.ifBlank { previous.name }, + executed = incoming.executed || previous.executed, + time = incoming.time ?: previous.time, + state = when { + incomingState == null -> priorState + priorState == null -> incomingState + else -> incomingState.copy( + status = incomingState.status.ifBlank { priorState.status }, + input = incomingState.input ?: priorState.input, + content = incomingState.content.ifEmpty { priorState.content }, + metadata = incomingState.metadata ?: priorState.metadata, + error = incomingState.error ?: priorState.error, + ) + }, + ) +} + /** * The status a session row reports while a turn is in flight: it names the tool * call it is parked on (or how many, when several run at once) so a working chat @@ -378,6 +425,92 @@ internal fun turnReasoning(parts: List): ReasoningTurn = .map { it.text }, ) +/** + * One agent reply: everything the agent produced between two of the operator's + * messages, however many model steps that took. OpenCode reports a turn as one + * assistant message per step, so this is the single fold the transcript uses to + * render a reply as one bubble — the turn's thinking block, then its tool-call + * chips, then its answer — instead of one bubble (and one thinking block) per + * step. + */ +internal data class ChatTurn(val key: String, val parts: List) { + /** The turn's reasoning, folded into the one block the transcript shows first. */ + val reasoning: ReasoningTurn get() = turnReasoning(parts) + + /** The turn's tool calls, in the order the agent made them. */ + val tools: List get() = parts.filter { it.type == ChatProtocol.PART_TOOL } + + /** + * The turn's answer: just its text parts, joined in order. Reasoning feeds the + * folded thinking block and a tool call its chip, so the answer block never + * carries thinking or a command. + */ + val answer: String + get() = parts + .filter { + it.type != ChatProtocol.PART_TOOL && + (partRendering(it) == PartRendering.MARKDOWN || partRendering(it) == PartRendering.PLAIN) + } + .joinToString("\n\n") { it.text } + + /** The turn's own timing, or null when its parts carry none. */ + val elapsedMillis: Long? get() = turnElapsedMillis(parts) + + /** Whether the turn has anything to draw; an empty turn renders no bubble. */ + val visible: Boolean get() = reasoning.count > 0 || tools.isNotEmpty() || answer.isNotBlank() +} + +/** + * One item of the transcript: an operator message, or one folded agent reply. The + * key is stable as a reply grows — an agent turn is keyed by its first message id + * — so the list is not rebuilt and an expanded thinking block keeps its state as + * new steps stream in. + */ +internal sealed interface TranscriptRow { + val key: String + + data class User(val message: ChatMessage) : TranscriptRow { + override val key: String get() = message.id + } + + data class Agent(val turn: ChatTurn) : TranscriptRow { + override val key: String get() = turn.key + } +} + +/** + * The transcript's rows for [messages]: each run of consecutive assistant + * messages folds into one [TranscriptRow.Agent], and each of the operator's own + * messages is its own [TranscriptRow.User]. This is the one fold that makes a + * reply a single bubble, so the transcript and its tests cannot disagree about + * where one agent turn ends and the next begins. + */ +internal fun transcriptRows(messages: List): List { + val rows = ArrayList(messages.size) + var turnKey: String? = null + var parts = ArrayList() + fun flush() { + val key = turnKey + if (key != null) { + val turn = ChatTurn(key, parts) + if (turn.visible) rows.add(TranscriptRow.Agent(turn)) + } + turnKey = null + parts = ArrayList() + } + for (message in messages) { + if (message.role == ChatProtocol.ROLE_ASSISTANT) { + if (turnKey == null) turnKey = message.id + parts.addAll(message.parts) + } else { + flush() + rows.add(TranscriptRow.User(message)) + } + } + flush() + return rows +} + /** The live label of a single-step turn's reasoning block while it streams. */ private const val REASONING_STREAMING_LABEL = "Thinking…" diff --git a/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatStore.kt b/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatStore.kt index d38d2d6b..7b109b34 100644 --- a/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatStore.kt +++ b/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatStore.kt @@ -938,45 +938,56 @@ class ChatStore( } private fun applyDelta(frame: ChatEventFrame) { - val partId = frame.part?.id.orEmpty() - if (partId.isEmpty() || frame.delta.isEmpty()) return - upsertPart(frame, partId) { existing -> - existing?.copy(text = existing.text + frame.delta) - ?: ChatPart(type = frame.kind, id = partId, text = frame.delta) + val rawId = frame.part?.id.orEmpty() + if (rawId.isEmpty() || frame.delta.isEmpty()) return + val kind = frame.kind + upsertParts(frame) { parts -> + val id = deltaPartId(parts, kind, rawId) + val index = parts.indexOfFirst { it.id == id } + if (index < 0) { + parts + ChatPart(type = kind, id = id, text = frame.delta) + } else { + parts.toMutableList().also { it[index] = it[index].copy(text = it[index].text + frame.delta) } + } } } private fun applyTool(frame: ChatEventFrame) { - val part = frame.part ?: return - if (part.id.isEmpty()) return - upsertPart(frame, part.id) { part } + val incoming = frame.part ?: return + if (incoming.id.isEmpty()) return + upsertParts(frame) { parts -> + val index = parts.indexOfFirst { it.id == incoming.id && it.type == ChatProtocol.PART_TOOL } + if (index < 0) { + parts + mergeToolPart(null, incoming) + } else { + parts.toMutableList().also { it[index] = mergeToolPart(it[index], incoming) } + } + } } /** - * Finds or creates the assistant message [frame] targets and the part [partId] - * inside it, replacing it via [upsert] (the old part when it exists, else - * null), then marks the thread running. The text/reasoning delta and the tool - * upsert share this so the two cannot drift. + * Finds or creates the assistant message [frame] targets and replaces its + * parts with [upsert]'s result, then marks the thread running. The text and + * reasoning delta fold and the tool upsert share this so the two cannot drift. */ - private fun upsertPart(frame: ChatEventFrame, partId: String, upsert: (ChatPart?) -> ChatPart) { + private fun upsertParts(frame: ChatEventFrame, upsert: (List) -> List) { update(frame.sessionId) { thread -> val messages = thread.messages.toMutableList() val index = messages.indexOfLast { it.id == frame.messageId } - val message = if (index >= 0) { - messages[index] + val resolvedIndex = if (index >= 0) { + index } else { - ChatMessage( - id = frame.messageId, - sessionId = frame.sessionId, - role = ChatProtocol.ROLE_ASSISTANT, - ).also { messages.add(it) } + messages.add( + ChatMessage( + id = frame.messageId, + sessionId = frame.sessionId, + role = ChatProtocol.ROLE_ASSISTANT, + ), + ) + messages.lastIndex } - val resolvedIndex = if (index >= 0) index else messages.lastIndex - val parts = message.parts.toMutableList() - val partIndex = parts.indexOfFirst { it.id == partId } - val updated = upsert(if (partIndex >= 0) parts[partIndex] else null) - if (partIndex >= 0) parts[partIndex] = updated else parts.add(updated) - messages[resolvedIndex] = message.copy(parts = parts) + val message = messages[resolvedIndex] + messages[resolvedIndex] = message.copy(parts = upsert(message.parts)) val stamped = runningToolStamps(thread.toolStartedAt, messages, now()) setRunning(thread.copy(messages = messages, toolStartedAt = stamped), true) } diff --git a/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatTranscript.kt b/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatTranscript.kt index 9b63b83d..0434bb23 100644 --- a/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatTranscript.kt +++ b/android/app/src/main/java/net/awan/pipelines/ui/chat/ChatTranscript.kt @@ -85,6 +85,7 @@ private const val CHAT_ELAPSED_TICK_MILLIS = 1_000L @Composable internal fun ChatTranscript( thread: ChatThread, + rows: List, listState: LazyListState, savedScroll: ChatScrollPosition?, itemCount: Int, @@ -177,22 +178,26 @@ internal fun ChatTranscript( } Box(Modifier.fillMaxSize()) { // Only the newest turn can be streaming, so its reasoning wears the live - // label; a settled turn's reasoning is labelled as settled. - val streamingMessageId = thread.messages.lastOrNull()?.id?.takeIf { thread.running } + // label; a settled turn's reasoning is labelled as settled. The turn is + // the transcript's last row only while it is the turn in flight. + val streamingTurnKey = (rows.lastOrNull() as? TranscriptRow.Agent)?.key?.takeIf { thread.running } LazyColumn( state = listState, modifier = Modifier.fillMaxSize(), contentPadding = PaddingValues(horizontal = CHAT_GUTTER, vertical = CHAT_ITEM_GAP), verticalArrangement = Arrangement.spacedBy(CHAT_ITEM_GAP, Alignment.Bottom), ) { - items(thread.messages, key = { it.id }) { message -> - MessageItem( - message = message, - reasoning = reasoning, - streaming = message.id == streamingMessageId, - runningSince = thread.runningSince, - onToggleReasoning = onToggleReasoning, - ) + items(rows, key = { it.key }) { row -> + when (row) { + is TranscriptRow.User -> UserMessageItem(row.message) + is TranscriptRow.Agent -> AgentTurnItem( + turn = row.turn, + reasoning = reasoning, + streaming = row.key == streamingTurnKey, + runningSince = thread.runningSince, + onToggleReasoning = onToggleReasoning, + ) + } } // A slash command reports as its own line, after the turns it acted // between and visually distinct from any message to the agent. @@ -398,8 +403,45 @@ private fun rememberElapsedNow(runningSince: Long): Long { } @Composable -private fun MessageItem( - message: ChatMessage, +private fun UserMessageItem(message: ChatMessage) { + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.End, + ) { + Surface( + shape = RoundedCornerShape(CHAT_GUTTER), + color = MaterialTheme.colorScheme.secondaryContainer, + modifier = Modifier.widthIn(max = CHAT_BUBBLE_MAX_WIDTH_DP), + ) { + SelectionContainer { + Column(Modifier.padding(CHAT_GUTTER)) { + // A user turn another device sent is attributed to it, so the + // transcript says who sent the message. Our own messages carry + // no label. + if (message.client.isNotBlank()) { + Text( + text = message.client, + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + UserParts(message.parts) + } + } + } + } +} + +/** + * One agent reply as a single bubble: the turn's one collapsible thinking block + * first, then its tool-call chips in order, then its answer text. A turn's + * reasoning and answer are never mixed — the fold in [ChatTurn] keeps them + * separate, so the answer is never drawn inside the thinking block — and the + * whole reply is one item however many model steps produced it. + */ +@Composable +private fun AgentTurnItem( + turn: ChatTurn, reasoning: ReasoningExpansion, streaming: Boolean, runningSince: Long, @@ -414,46 +456,29 @@ private fun MessageItem( val now = rememberElapsedNow(runningSince) if (runningSince < now) now - runningSince else 0L } else { - turnElapsedMillis(message.parts) + turn.elapsedMillis } - val isUser = message.role == ChatProtocol.ROLE_USER Row( modifier = Modifier.fillMaxWidth(), - horizontalArrangement = if (isUser) Arrangement.End else Arrangement.Start, + horizontalArrangement = Arrangement.Start, ) { Surface( shape = RoundedCornerShape(CHAT_GUTTER), - color = if (isUser) { - MaterialTheme.colorScheme.secondaryContainer - } else { - MaterialTheme.colorScheme.surfaceContainer - }, + color = MaterialTheme.colorScheme.surfaceContainer, modifier = Modifier.widthIn(max = CHAT_BUBBLE_MAX_WIDTH_DP), ) { SelectionContainer { Column(Modifier.padding(CHAT_GUTTER)) { - if (isUser) { - // A user turn another device sent is attributed to it, so - // the transcript says who sent the message. Our own - // messages carry no label. - if (message.client.isNotBlank()) { - Text( - text = message.client, - style = MaterialTheme.typography.labelSmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } - UserParts(message.parts) - } else { - AssistantParts( - messageId = message.id, - parts = message.parts, - reasoning = reasoning, - streaming = streaming, - elapsedMillis = elapsedMillis, - onToggleReasoning = onToggleReasoning, - ) - } + val turnKey = reasoningTurnKey(turn.key) + ReasoningBlock( + turn = turn.reasoning, + expanded = reasoning.isExpanded(turnKey), + streaming = streaming, + elapsedMillis = elapsedMillis, + onToggle = { onToggleReasoning(turnKey) }, + ) + if (turn.tools.isNotEmpty()) ToolCalls(turn.tools) + if (turn.answer.isNotBlank()) MarkdownText(turn.answer) } } } @@ -462,7 +487,7 @@ private fun MessageItem( /** * A user turn: its text, then the files that rode with it. The attachments are - * the same [ChatProtocol.PART_FILE] parts [MessageItem] already renders, so the + * the same [ChatProtocol.PART_FILE] parts the user bubble already renders, so the * sent message shows its file in the transcript through the one display path * rather than a second attachment list under the bubble. */ @@ -478,73 +503,6 @@ private fun UserParts(parts: List) { } } -/** - * An assistant turn's parts in order, with each run of consecutive tool calls - * rendered as one wrapping chip group (mirroring the reference transcript's - * `_ToolCalls`, research §1.3): the calls of a turn sit together on as few lines - * as they fit, instead of each taking a row, so the transcript stays scannable. - * - * The turn's reasoning is folded into exactly one collapsible block, anchored - * where the turn's first reasoning part sits, so a turn that reasons several - * times is one entry rather than a stack of boxes and a streaming turn grows the - * block it has instead of gaining another. - */ -@Composable -private fun AssistantParts( - messageId: String, - parts: List, - reasoning: ReasoningExpansion, - streaming: Boolean, - elapsedMillis: Long?, - onToggleReasoning: (String) -> Unit, -) { - val turn = turnReasoning(parts) - val turnKey = reasoningTurnKey(messageId) - var index = 0 - var reasoningShown = false - while (index < parts.size) { - val part = parts[index] - when { - part.type == ChatProtocol.PART_TOOL -> { - var end = index - while (end < parts.size && parts[end].type == ChatProtocol.PART_TOOL) end++ - ToolCalls(parts.subList(index, end)) - index = end - } - partRendering(part) == PartRendering.REASONING -> { - if (!reasoningShown) { - reasoningShown = true - ReasoningBlock( - turn = turn, - expanded = reasoning.isExpanded(turnKey), - streaming = streaming, - elapsedMillis = elapsedMillis, - onToggle = { onToggleReasoning(turnKey) }, - ) - } - index++ - } - else -> { - MessagePart(part) - index++ - } - } - } -} - -@Composable -private fun MessagePart(part: ChatPart) { - when (partRendering(part)) { - PartRendering.MARKDOWN -> MarkdownText(part.text) - // Reasoning is folded into the turn's one block by [AssistantParts]; a - // reasoning part never reaches here, but rendering nothing keeps this - // exhaustive and safe if one ever did. - PartRendering.REASONING -> Unit - PartRendering.PLAIN -> Text(part.text, style = MaterialTheme.typography.bodyMedium) - PartRendering.HIDDEN -> Unit - } -} - /** * A turn's reasoning: one compact, muted line that names the block — its step * count when it reasoned more than once, and its elapsed time when the turn's own diff --git a/android/app/src/test/java/net/awan/pipelines/ui/chat/ChatReasoningTurnTest.kt b/android/app/src/test/java/net/awan/pipelines/ui/chat/ChatReasoningTurnTest.kt index f1fd4d4d..806aee88 100644 --- a/android/app/src/test/java/net/awan/pipelines/ui/chat/ChatReasoningTurnTest.kt +++ b/android/app/src/test/java/net/awan/pipelines/ui/chat/ChatReasoningTurnTest.kt @@ -1,6 +1,7 @@ package net.awan.pipelines.ui.chat import net.awan.pipelines.data.protocol.ChatEventFrame +import net.awan.pipelines.data.protocol.ChatMessage import net.awan.pipelines.data.protocol.ChatPart import net.awan.pipelines.data.protocol.ChatProtocol import net.awan.pipelines.data.protocol.ChatSession @@ -117,6 +118,77 @@ class ChatReasoningTurnTest { assertFalse(reasoning.isExpanded(reasoningTurnKey("m2"))) } + @Test + fun aTextDeltaAndAReasoningDeltaInOneMessageNeverShareAPart() { + val store = ChatStore() + store.open(ChatSession(id = "s", title = "s")) + store.history("s", emptyList()) + + // The broker numbers a message's text and reasoning independently from 0, + // so a peer that still keyed both off the bare ordinal addresses the same + // id for the two kinds: the client must keep them apart anyway, or the + // answer is appended into the thinking block. + store.event(reasoningDelta("s", "m1", "m1#0", "thinking")) + store.event(textDelta("s", "m1", "m1#0", "answer")) + + val message = store.value.thread("s")!!.messages.single() + val reasoning = message.parts.single { it.type == ChatProtocol.PART_REASONING } + val answer = message.parts.single { it.type == ChatProtocol.PART_TEXT } + assertEquals("thinking", reasoning.text) + assertEquals("answer", answer.text) + + // A later text delta on the same id keeps growing the answer part and + // never touches the reasoning part. + store.event(textDelta("s", "m1", "m1#0", "!")) + val grown = store.value.thread("s")!!.messages.single() + assertEquals("answer!", grown.parts.single { it.type == ChatProtocol.PART_TEXT }.text) + assertEquals("thinking", grown.parts.single { it.type == ChatProtocol.PART_REASONING }.text) + + // The broker's own derivation gives the two kinds distinct ids, and the + // client's mirror produces the same ids. + assertEquals("m1#text#0", ChatProtocol.deltaPartId("m1", ChatProtocol.PART_TEXT, 0)) + assertEquals("m1#reasoning#0", ChatProtocol.deltaPartId("m1", ChatProtocol.PART_REASONING, 0)) + assertFalse( + ChatProtocol.deltaPartId("m1", ChatProtocol.PART_TEXT, 0) == + ChatProtocol.deltaPartId("m1", ChatProtocol.PART_REASONING, 0), + ) + } + + @Test + fun agentStepsBetweenTwoUserMessagesFoldIntoOneTurn() { + // The step-by-step reply a live agent produces: several assistant messages, + // then the final one carrying the answer beside its reasoning. They are one + // turn — one bubble, one thinking block, one answer — and the operator's + // next message starts a new one. + val messages = listOf( + ChatMessage(id = "u1", role = ChatProtocol.ROLE_USER, parts = listOf(text("t", "do it"))), + ChatMessage(id = "m0", role = ChatProtocol.ROLE_ASSISTANT, parts = listOf(reasoning("r0", "thought 0"), ChatPart(type = ChatProtocol.PART_TOOL, id = "c0", name = "shell"))), + ChatMessage(id = "m1", role = ChatProtocol.ROLE_ASSISTANT, parts = listOf(reasoning("r1", "thought 1"), ChatPart(type = ChatProtocol.PART_TOOL, id = "c1", name = "read"))), + ChatMessage(id = "m2", role = ChatProtocol.ROLE_ASSISTANT, parts = listOf(reasoning("r2", "thought 2"), text("t2", "the answer"))), + ChatMessage(id = "u2", role = ChatProtocol.ROLE_USER, parts = listOf(text("t", "again"))), + ) + val rows = transcriptRows(messages) + assertEquals(listOf("u1", "m0", "u2"), rows.map { it.key }) + + val turn = (rows[1] as TranscriptRow.Agent).turn + // The turn is keyed by its first message id, so its expansion state and the + // list key survive the later steps. + assertEquals("m0", turn.key) + assertEquals(3, turn.reasoning.count) + assertEquals("thought 0\n\nthought 1\n\nthought 2", turn.reasoning.text) + assertEquals(listOf("c0", "c1"), turn.tools.map { it.id }) + assertEquals("the answer", turn.answer) + } + + private fun textDelta(sessionId: String, messageId: String, partId: String, delta: String) = + ChatEventFrame( + sessionId = sessionId, + messageId = messageId, + kind = ChatProtocol.PART_TEXT, + delta = delta, + part = ChatPart(type = ChatProtocol.PART_TEXT, id = partId), + ) + private fun reasoningDelta(sessionId: String, messageId: String, partId: String, delta: String) = ChatEventFrame( sessionId = sessionId, diff --git a/android/app/src/test/java/net/awan/pipelines/ui/chat/ChatRunningWorkTest.kt b/android/app/src/test/java/net/awan/pipelines/ui/chat/ChatRunningWorkTest.kt index 3e82282d..f524c022 100644 --- a/android/app/src/test/java/net/awan/pipelines/ui/chat/ChatRunningWorkTest.kt +++ b/android/app/src/test/java/net/awan/pipelines/ui/chat/ChatRunningWorkTest.kt @@ -8,6 +8,7 @@ import net.awan.pipelines.data.protocol.ChatPart import net.awan.pipelines.data.protocol.ChatProtocol import net.awan.pipelines.data.protocol.ChatSession import net.awan.pipelines.data.protocol.ChatTime +import net.awan.pipelines.data.protocol.ChatToolContent import net.awan.pipelines.data.protocol.ChatToolState import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse @@ -160,6 +161,50 @@ class ChatRunningWorkTest { assertEquals(4_000L, store.value.thread("s")!!.toolElapsedMillis(call, clock)) } + @Test + fun aToolCallKeepsItsNameAndInputForItsWholeLifecycle() { + val store = ChatStore() + store.open(ChatSession(id = "s", title = "s")) + store.history("s", emptyList()) + + // The verified frame sequence: the name arrives only in input.started, + // the arguments in called's input, and progress/success carry neither. + store.event(toolFrame("s", "m1", ChatPart(type = ChatProtocol.PART_TOOL, id = "c1", name = "shell", state = ChatToolState(status = ChatProtocol.TOOL_STREAMING)))) + store.event(toolFrame("s", "m1", ChatPart(type = ChatProtocol.PART_TOOL, id = "c1", state = ChatToolState(status = ChatProtocol.TOOL_RUNNING, input = input("command" to "ls"))))) + store.event(toolFrame("s", "m1", ChatPart(type = ChatProtocol.PART_TOOL, id = "c1", state = ChatToolState(status = ChatProtocol.TOOL_RUNNING, metadata = JsonObject(mapOf("exit" to JsonPrimitive(0))))))) + store.event(toolFrame("s", "m1", ChatPart(type = ChatProtocol.PART_TOOL, id = "c1", state = ChatToolState(status = ChatProtocol.TOOL_COMPLETED, content = listOf(ChatToolContent(type = "text", text = "ok")))))) + + val part = store.value.thread("s")!!.messages.single().parts.single() + // Nothing blanked the name or the input the earlier frames taught it. + assertEquals("shell", toolLabel(part)) + assertEquals("ls", toolDetail(part)) + assertEquals("ok", toolOutputText(part.state)) + assertEquals(ChatProtocol.TOOL_COMPLETED, part.state?.status) + assertEquals(JsonPrimitive(0), part.state?.metadata?.get("exit")) + } + + @Test + fun aStreamingArgumentStringIsShownWhileTheCallRuns() { + val store = ChatStore() + store.open(ChatSession(id = "s", title = "s")) + store.event( + toolFrame( + "s", "m1", + ChatPart( + type = ChatProtocol.PART_TOOL, + id = "c1", + name = "edit", + state = ChatToolState(status = ChatProtocol.TOOL_STREAMING, input = JsonPrimitive("{\"filePath\"")), + ), + ), + ) + val call = store.value.thread("s")!!.messages.single().parts.single() + // A partial-JSON argument string is kept and shown verbatim; it has no + // object key to describe the call yet, so the detail stays absent. + assertEquals("{\"filePath\"", prettyToolInput(call.state?.input)) + assertNull(toolDetail(call)) + } + @Test fun toolDetailPrefersDescriptionThenCommandAndIgnoresWhatItCannotRead() { // A description wins over a command when both are present. diff --git a/android/app/src/test/java/net/awan/pipelines/ui/chat/ChatTranscriptTest.kt b/android/app/src/test/java/net/awan/pipelines/ui/chat/ChatTranscriptTest.kt index ce071d77..0a7c3d61 100644 --- a/android/app/src/test/java/net/awan/pipelines/ui/chat/ChatTranscriptTest.kt +++ b/android/app/src/test/java/net/awan/pipelines/ui/chat/ChatTranscriptTest.kt @@ -1,5 +1,6 @@ package net.awan.pipelines.ui.chat +import net.awan.pipelines.data.protocol.ChatMessage import net.awan.pipelines.data.protocol.ChatPart import net.awan.pipelines.data.protocol.ChatProtocol import org.junit.Assert.assertEquals @@ -148,4 +149,42 @@ class ChatTranscriptTest { // ...and a single open chat leaves nothing to switch to. assertEquals(emptyList(), openChatsBeside(listOf(a), "a")) } + + @Test + fun eachConversationalRunOfAgentStepsIsOneRowAndItsTextPartsJoinInOrder() { + // Two user messages with two agent steps between the first pair: the agent + // steps are one turn (one bubble), not one row each, and their text parts + // are one answer joined in order. + val messages = listOf( + ChatMessage(id = "u1", role = ChatProtocol.ROLE_USER, parts = listOf(ChatPart(type = ChatProtocol.PART_TEXT, text = "one"))), + ChatMessage(id = "a1", role = ChatProtocol.ROLE_ASSISTANT, parts = listOf(ChatPart(type = ChatProtocol.PART_TEXT, id = "p1", text = "first"))), + ChatMessage(id = "a2", role = ChatProtocol.ROLE_ASSISTANT, parts = listOf(ChatPart(type = ChatProtocol.PART_TOOL, id = "c1", name = "shell"), ChatPart(type = ChatProtocol.PART_TEXT, id = "p2", text = "second"))), + ChatMessage(id = "u2", role = ChatProtocol.ROLE_USER, parts = listOf(ChatPart(type = ChatProtocol.PART_TEXT, text = "two"))), + ChatMessage(id = "a3", role = ChatProtocol.ROLE_ASSISTANT, parts = listOf(ChatPart(type = ChatProtocol.PART_TEXT, id = "p3", text = "third"))), + ) + val rows = transcriptRows(messages) + assertEquals(listOf("u1", "a1", "u2", "a3"), rows.map { it.key }) + assertEquals( + listOf("u1", "u2"), + rows.filterIsInstance().map { it.key }, + ) + val first = (rows[1] as TranscriptRow.Agent).turn + assertEquals("first\n\nsecond", first.answer) + assertEquals(listOf("c1"), first.tools.map { it.id }) + assertEquals("third", (rows[3] as TranscriptRow.Agent).turn.answer) + } + + @Test + fun anAgentMessageWithNothingToDrawAddsNoRow() { + // An assistant message whose parts are all empty (or reasoning without + // text) has nothing to draw, so it folds away instead of leaving a blank + // bubble; a following user message is still its own row. + val rows = transcriptRows( + listOf( + ChatMessage(id = "a1", role = ChatProtocol.ROLE_ASSISTANT, parts = listOf(ChatPart(type = ChatProtocol.PART_TEXT, id = "p1"))), + ChatMessage(id = "u1", role = ChatProtocol.ROLE_USER, parts = listOf(ChatPart(type = ChatProtocol.PART_TEXT, text = "hi"))), + ), + ) + assertEquals(listOf("u1"), rows.map { it.key }) + } } diff --git a/docs/notes/chat-transcript-turn-fold.md b/docs/notes/chat-transcript-turn-fold.md new file mode 100644 index 00000000..02bc9ecc --- /dev/null +++ b/docs/notes/chat-transcript-turn-fold.md @@ -0,0 +1,79 @@ +# Chat transcript: one agent reply is one bubble + +## What this item changed + +`internal/` and `android/`, the brokered chat transcript. Three defects measured +against the operator's own OpenCode (127.0.0.1:49374): + +- **Server, delta part ids.** `internal/protocol/chat.go` adds + `ChatDeltaPartID(messageID, kind, ordinal)` (mirrored as + `ChatProtocol.deltaPartId` in `android/.../data/protocol/Chat.kt`), and + `internal/opencode/chat.go`'s `NormalizeChatEvent` keys a text or reasoning + delta on it. OpenCode numbers one message's text and reasoning parts + independently from 0, so the old `"%s#%d"` gave a message's first text delta and + its first reasoning part the same id — the client appended the answer into the + thinking block and the server's retained state re-collided on resume. +- **Server, tool fields.** `NormalizeChatEvent` now takes a tool call's arguments + from `session.tool.input.ended`'s `text` as well as `session.tool.called`'s + `input` object (`toolInput`), and `internal/server/chat_hub.go`'s new + `mergeChatToolPart` folds each tool frame onto the retained part so a later + `progress`/`success` frame that omits the name or the input never blanks it. +- **Android, transcript.** `ui/chat/ChatFormat.kt` adds the pure `ChatTurn` / + `TranscriptRow` / `transcriptRows` fold (each run of assistant messages between + two of the operator's messages is one agent turn), the kind-aware + `deltaPartId` net and `mergeToolPart`; `ChatStore` folds deltas and tool frames + through them; `ChatTranscript` renders one bubble per turn — one thinking block + over all the turn's reasoning, then its tool chips, then one answer block of + its text parts — replacing `AssistantParts`/`MessagePart` with + `AgentTurnItem`/`UserMessageItem`. + +Tests: `internal/opencode/chat_v2_test.go` and `internal/server/chat_test.go` add +the id-collision and tool name+input retention cases; `ChatReasoningTurnTest`, +`ChatTranscriptTest` and `ChatRunningWorkTest` cover the turn fold, the +text/reasoning collision and the tool lifecycle on the client. + +## Shared documents this change makes stale + +`android/README.md` describes the surfaces this item changed. The note is written +here rather than edited in place; the `docs-fold` writer should update it. + +- `android/README.md:6021` — the **Reasoning collapsed by default** bullet reads + "`ReasoningBlock` is now one compact, muted line per part". It should say one + block per *agent turn*: the transcript folds every consecutive assistant message + between two of the operator's messages into one `ChatTurn`, so all the turn's + reasoning is one collapsible block (labelled with its step count when it + reasoned more than once) and the reply is one bubble, not one per model step. +- `android/README.md:4669` — "`ChatScreen.MessagePart` now calls + `MarkdownText(part.text)` for `PART_TEXT`; the assistant message stays one + `SelectionContainer`". `MessagePart` no longer exists; the answer is + `MarkdownText(turn.answer)` inside `AgentTurnItem`, where `turn.answer` is the + turn's text parts joined in order. The one-`SelectionContainer`-per-bubble fact + is still true. +- `android/README.md:6040` — the test paragraph's "`ChatTranscriptTest`, 7 cases" + is now 9 (the fold tests); `ChatReasoningTurnTest` and `ChatRunningWorkTest` + gained the collision and tool-lifecycle cases. +- `android/README.md:4786` — the **Grouped chips** bullet says "a run of + consecutive `tool` parts in one assistant turn is now one `FlowRow`". The chips + still group in one `FlowRow`, but the calls shown are now the whole folded + turn's (`ChatTurn.tools`), across every step of the reply, not one assistant + message's parts. + +Dates, counts and capture names in those dated records are past evidence and stay +as they are; only the behaviour statements above are now stale. + +## Not stale + +- `docs/cli.md`'s and `internal/author/skill/references/operations.md`'s brokered + chat sections describe the `chat_*` surface and its retained + per-session/per-message state and resync; that remains true. Neither documents + the delta part-id derivation or the tool-field retention, so neither gains a + wrong statement — the derivation now lives in the `internal/protocol/chat.go` + comment. The docs-fold writer may add one sentence to the reference's + retained-state paragraph if it documents part ids there, but nothing there is + now false. +- No packaged OpenCode skill reference documents the app's chat transcript, and no + packaged template or `internal/author/` managed file names it. +- `android/AGENTS.md` and `internal/server/AGENTS.md` list the files this item + touched (`ui/chat/`, `internal/server/chat_*.go`, `internal/protocol/`, + `internal/opencode/chat.go`); no file moved and no build/test step changed, so + both maps stay correct. diff --git a/internal/opencode/chat.go b/internal/opencode/chat.go index 41a7b25c..eda1ec3a 100644 --- a/internal/opencode/chat.go +++ b/internal/opencode/chat.go @@ -388,6 +388,7 @@ type chatEvent struct { Delta string `json:"delta"` ID string `json:"id"` Name string `json:"name"` + Text string `json:"text"` Input json.RawMessage `json:"input"` Content []protocol.ChatToolContent `json:"content"` Metadata map[string]any `json:"metadata"` @@ -451,11 +452,11 @@ func NormalizeChatEvent(payload []byte, sessionID string, lastMessageID *string) case outer.Type == "session.text.delta": frame.Kind = protocol.ChatEventText frame.Delta = data.Delta - frame.Part = &protocol.ChatPart{Type: protocol.ChatPartText, ID: fmt.Sprintf("%s#%d", messageID, data.Ordinal)} + frame.Part = &protocol.ChatPart{Type: protocol.ChatPartText, ID: protocol.ChatDeltaPartID(messageID, protocol.ChatPartText, data.Ordinal)} case outer.Type == "session.reasoning.delta": frame.Kind = protocol.ChatEventReasoning frame.Delta = data.Delta - frame.Part = &protocol.ChatPart{Type: protocol.ChatPartReasoning, ID: fmt.Sprintf("%s#%d", messageID, data.Ordinal)} + frame.Part = &protocol.ChatPart{Type: protocol.ChatPartReasoning, ID: protocol.ChatDeltaPartID(messageID, protocol.ChatPartReasoning, data.Ordinal)} case outer.Type == "session.step.started" || outer.Type == "session.step.streamed" || outer.Type == "session.step.ended": frame.Kind = protocol.ChatEventStep frame.Step = &protocol.ChatStep{Status: strings.TrimPrefix(outer.Type, "session.step."), Index: stepIndex(data)} @@ -465,7 +466,7 @@ func NormalizeChatEvent(payload []byte, sessionID string, lastMessageID *string) return protocol.ChatEventFrame{}, false, nil } frame.Kind = protocol.ChatEventTool - state := &protocol.ChatToolState{Status: status, Input: data.Input, Content: data.Content, Metadata: data.Metadata, Error: data.Error} + state := &protocol.ChatToolState{Status: status, Input: toolInput(data), Content: data.Content, Metadata: data.Metadata, Error: data.Error} frame.Part = &protocol.ChatPart{Type: protocol.ChatPartTool, ID: data.ID, Name: data.Name, State: state} case outer.Type == "session.usage.updated": frame.Kind = protocol.ChatEventUsage @@ -480,6 +481,31 @@ func NormalizeChatEvent(payload []byte, sessionID string, lastMessageID *string) return frame, true, nil } +// toolInput picks a tool call's arguments from whichever frame carries them: +// session.tool.called carries the parsed `input` object, session.tool.input.ended +// carries the same arguments as a `text` JSON string, and progress/success carry +// neither. Text that already parses as JSON is kept as that value (so an object +// pretty-prints); a partial string is carried as a JSON string so the frame stays +// valid and the client can show it while streaming. It returns nil when the frame +// carries no arguments, so the consumer retains what it already learned. +func toolInput(data chatEvent) json.RawMessage { + if len(data.Input) > 0 && string(data.Input) != "null" { + return data.Input + } + text := strings.TrimSpace(data.Text) + if text == "" { + return nil + } + if json.Valid([]byte(text)) { + return json.RawMessage(text) + } + encoded, err := json.Marshal(text) + if err != nil { + return nil + } + return encoded +} + // toolStatus maps the verified tool event types onto the ToolState lifecycle. func toolStatus(eventType string) (string, bool) { switch eventType { diff --git a/internal/opencode/chat_v2_test.go b/internal/opencode/chat_v2_test.go index cd5dbb7a..4e76d7d3 100644 --- a/internal/opencode/chat_v2_test.go +++ b/internal/opencode/chat_v2_test.go @@ -2,6 +2,8 @@ package opencode import ( "context" + "encoding/json" + "fmt" "net/http" "net/http/httptest" "path/filepath" @@ -121,6 +123,149 @@ func TestChatStreamRealV2Shapes(t *testing.T) { } } +// TestChatDeltaPartIDsNeverCollide pins the broker's part-id namespace: OpenCode +// numbers one message's text and reasoning parts independently from 0, so a +// message's first text delta and its first reasoning delta share an ordinal. The +// two must still land on distinct part ids — otherwise the client appends the +// answer into the thinking block and the server's retained state re-collides on a +// resume. The id is the shared ChatDeltaPartID derivation, so it is deterministic +// across a reconnect. +func TestChatDeltaPartIDsNeverCollide(t *testing.T) { + payloads := map[string]string{ + "text": `{"type":"session.text.delta","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","ordinal":0,"delta":"answer"}}`, + "reasoning": `{"type":"session.reasoning.delta","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","ordinal":0,"delta":"think"}}`, + } + ids := map[string]string{} + for kind, payload := range payloads { + last := "" + frame, ok, err := NormalizeChatEvent([]byte(payload), "ses_1", &last) + if err != nil || !ok { + t.Fatalf("%s delta: ok=%v err=%v", kind, ok, err) + } + if frame.Part == nil { + t.Fatalf("%s delta carried no part", kind) + } + if frame.Part.Type != kind { + t.Fatalf("%s delta part type = %q", kind, frame.Part.Type) + } + ids[kind] = frame.Part.ID + } + if ids["text"] == ids["reasoning"] { + t.Fatalf("text and reasoning share part id %q", ids["text"]) + } + if ids["text"] != protocol.ChatDeltaPartID("msg_1", protocol.ChatPartText, 0) { + t.Fatalf("text part id = %q, want the stable derivation", ids["text"]) + } + if ids["reasoning"] != protocol.ChatDeltaPartID("msg_1", protocol.ChatPartReasoning, 0) { + t.Fatalf("reasoning part id = %q, want the stable derivation", ids["reasoning"]) + } +} + +// TestChatToolLifecycleCarriesNameAndInput pins what the broker learns from each +// verified tool frame: input.started carries the name, input.ended carries the +// arguments as its `text` string, called carries the parsed input object, and +// progress/success carry neither — so those frames must surface no name and no +// input, leaving the retention to the consumer instead of blanking the field. +func TestChatToolLifecycleCarriesNameAndInput(t *testing.T) { + frames := []struct { + name string + payload string + wantStatus string + wantName string + wantInput string // the JSON encoding expected, "" for absent + }{ + { + name: "input.started names the call", + payload: `{"type":"session.tool.input.started","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","name":"shell"}}`, + wantStatus: protocol.ChatToolStreaming, + wantName: "shell", + }, + { + name: "input.ended carries the arguments as text", + payload: `{"type":"session.tool.input.ended","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","text":"{\"command\":\"ls\"}"}}`, + wantStatus: protocol.ChatToolStreaming, + wantInput: `{"command":"ls"}`, + }, + { + name: "called carries the parsed input object", + payload: `{"type":"session.tool.called","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","input":{"command":"ls"}}}`, + wantStatus: protocol.ChatToolRunning, + wantInput: `{"command":"ls"}`, + }, + { + name: "progress carries metadata only", + payload: `{"type":"session.tool.progress","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","metadata":{"exit":0}}}`, + wantStatus: protocol.ChatToolRunning, + }, + { + name: "success carries the output and neither name nor input", + payload: `{"type":"session.tool.success","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","content":[{"type":"text","text":"ok"}],"metadata":{"exit":0},"executed":true}}`, + wantStatus: protocol.ChatToolCompleted, + }, + } + for _, tc := range frames { + t.Run(tc.name, func(t *testing.T) { + last := "" + frame, ok, err := NormalizeChatEvent([]byte(tc.payload), "ses_1", &last) + if err != nil || !ok { + t.Fatalf("ok=%v err=%v", ok, err) + } + if frame.Part == nil || frame.Part.State == nil { + t.Fatalf("tool frame = %+v", frame) + } + if frame.Part.State.Status != tc.wantStatus { + t.Fatalf("status = %q, want %q", frame.Part.State.Status, tc.wantStatus) + } + if frame.Part.Name != tc.wantName { + t.Fatalf("name = %q, want %q", frame.Part.Name, tc.wantName) + } + if tc.wantInput == "" { + if len(frame.Part.State.Input) != 0 { + t.Fatalf("input = %s, want absent", frame.Part.State.Input) + } + return + } + if !json.Valid(frame.Part.State.Input) { + t.Fatalf("input %s is not valid JSON", frame.Part.State.Input) + } + var want, got any + if err := json.Unmarshal([]byte(tc.wantInput), &want); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(frame.Part.State.Input, &got); err != nil { + t.Fatalf("input %s: %v", frame.Part.State.Input, err) + } + if fmt.Sprintf("%v", want) != fmt.Sprintf("%v", got) { + t.Fatalf("input = %s, want %s", frame.Part.State.Input, tc.wantInput) + } + }) + } +} + +// TestChatToolInputFromPartialText pins the streaming case: input.ended's text +// that is not yet a complete JSON value must still surface, carried as a JSON +// string so the frame stays valid and the client can show the partial arguments. +func TestChatToolInputFromPartialText(t *testing.T) { + last := "" + frame, ok, err := NormalizeChatEvent([]byte(`{"type":"session.tool.input.ended","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","text":"{\"command\":\"l"}}`), "ses_1", &last) + if err != nil || !ok { + t.Fatalf("ok=%v err=%v", ok, err) + } + if frame.Part == nil || frame.Part.State == nil { + t.Fatalf("frame = %+v", frame) + } + if !json.Valid(frame.Part.State.Input) { + t.Fatalf("partial input %s is not valid JSON", frame.Part.State.Input) + } + var got string + if err := json.Unmarshal(frame.Part.State.Input, &got); err != nil { + t.Fatalf("partial input is not a JSON string: %s (%v)", frame.Part.State.Input, err) + } + if got != `{"command":"l` { + t.Fatalf("partial input text = %q", got) + } +} + // TestOpenStreamRejectsUnknownBackend pins that the shared construction path // validates the backend before any process is started, so OpenStream never // launches a serve for an unknown backend. diff --git a/internal/protocol/chat.go b/internal/protocol/chat.go index 9db719f7..2fdeaaf5 100644 --- a/internal/protocol/chat.go +++ b/internal/protocol/chat.go @@ -1,6 +1,9 @@ package protocol -import "encoding/json" +import ( + "encoding/json" + "fmt" +) // Chat requests and responses are brokered by the Pipelines server (route B in // the OpenCode chat research): the phone keeps talking only to /connect and the @@ -213,8 +216,21 @@ type ChatMessage struct { Parts []ChatPart `json:"parts"` } +// ChatDeltaPartID is the stable part id an incremental text or reasoning delta +// stream addresses. OpenCode numbers one message's text and reasoning parts +// independently from 0, so the kind is part of the id: for a single message its +// first text delta and its first reasoning delta are distinct ids and a delta can +// never be appended into the other kind's part. The derivation is deterministic, +// so a reconnect or a resume re-addresses the same part, and it is mirrored in +// the client's ChatProtocol.deltaPartId (data/protocol/Chat.kt). +func ChatDeltaPartID(messageID, kind string, ordinal int) string { + return fmt.Sprintf("%s#%s#%d", messageID, kind, ordinal) +} + // ChatPart is one element of a ChatMessage. Type selects the meaning: text and -// reasoning use Text and ID, tool uses ID, Name, Executed, State and Time. +// reasoning use Text and ID, tool uses ID, Name, Executed, State and Time. A text +// or reasoning part's ID is ChatDeltaPartID(message id, kind, ordinal), so the two +// kinds never share an id within a message. type ChatPart struct { Type string `json:"type"` ID string `json:"id,omitempty"` @@ -227,6 +243,10 @@ type ChatPart struct { // ChatToolState is the lifecycle of a tool call. Input is a raw JSON value: a // partial-JSON string while streaming, an object once running or completed. +// OpenCode spreads one call across several frames (name in input.started, the +// arguments in input.ended's text or called's input, then progress/success +// carrying neither), so Input is absent on the frames that do not carry it and +// every consumer retains the value it already learned instead of blanking it. // Content is the tool's textual/file result, Metadata its provider stats, // Error the structured failure for status "error". type ChatToolState struct { diff --git a/internal/server/chat_hub.go b/internal/server/chat_hub.go index b04105b7..fdc14b3b 100644 --- a/internal/server/chat_hub.go +++ b/internal/server/chat_hub.go @@ -108,10 +108,12 @@ func (h *chatHub) clock() time.Time { } // apply folds one pushed frame into the retained state. Text and reasoning -// deltas append to their message's part; a tool frame upserts the full part -// (accounting for its whole size) so the latest lifecycle status, input, -// content, metadata, error and time are retained under the frame's message id. -// Step and usage frames update the session's latest progress. +// deltas append to their message's part; a tool frame upserts the part (merging +// the fields it carries onto what is retained, and accounting for the whole +// size) so the latest lifecycle status, name, input, content, metadata, error and +// time are retained under the frame's message id and a later frame that omits a +// field never blanks it. Step and usage frames update the session's latest +// progress. func (h *chatHub) apply(frame protocol.ChatEventFrame) { if frame.SessionID == "" { return @@ -181,29 +183,82 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) { if frame.Part == nil || frame.Part.ID == "" || turn.overflow { return } - // The tool part is retained whole, so size the complete part (input, - // content, metadata, error and time) and charge the difference against - // the cap on every upsert, not just the first-seen name. - incoming := chatPartBytes(frame.Part) + // The tool part is retained whole, so size the complete, merged part + // (input, content, metadata, error and time) and charge the difference + // against the cap on every upsert, not just the first-seen name. part, ok := message.parts[frame.Part.ID] if !ok { - if !turn.retain(incoming) { + merged := mergeChatToolPart(nil, frame.Part) + if !turn.retain(chatPartBytes(&merged)) { return } - part = &chatPart{} message.order = append(message.order, frame.Part.ID) - message.parts[frame.Part.ID] = part - } else if previous := chatPartBytes(&part.part); incoming > previous { + message.parts[frame.Part.ID] = &chatPart{part: merged} + return + } + merged := mergeChatToolPart(&part.part, frame.Part) + incoming := chatPartBytes(&merged) + previous := chatPartBytes(&part.part) + if incoming > previous { if !turn.retain(incoming - previous) { return } } else { turn.shrink(previous - incoming) } - part.part = *frame.Part + part.part = merged } } +// mergeChatToolPart folds one tool frame onto the part retained from the frames +// before it. A tool call is reported across several frames that each carry only +// part of it — input.started the name, input.ended the arguments as text, called +// the parsed input, progress metadata, and success the output with no name and no +// input — so a field the broker already learned is never blanked by a later frame +// that omits it. A non-empty incoming field wins; otherwise the retained value is +// kept. +func mergeChatToolPart(previous, incoming *protocol.ChatPart) protocol.ChatPart { + if previous == nil { + return *incoming + } + merged := *incoming + if merged.Type == "" { + merged.Type = previous.Type + } + if merged.Name == "" { + merged.Name = previous.Name + } + merged.Executed = merged.Executed || previous.Executed + if merged.Time == nil { + merged.Time = previous.Time + } + switch { + case incoming.State == nil: + merged.State = previous.State + case previous.State == nil: + // The incoming state stands on its own; merged already carries it. + default: + state := *incoming.State + if state.Status == "" { + state.Status = previous.State.Status + } + if len(state.Input) == 0 { + state.Input = previous.State.Input + } + if len(state.Content) == 0 { + state.Content = previous.State.Content + } + if state.Metadata == nil { + state.Metadata = previous.State.Metadata + } + if state.Error == nil { + state.Error = previous.State.Error + } + merged.State = &state + } + return merged +} + // chatPartBytes estimates the retained size of a part so the per-session cap // accounts for its whole state instead of only the first-seen name. It walks the // fields rather than re-marshalling, so the cost matches the bytes it stores. diff --git a/internal/server/chat_test.go b/internal/server/chat_test.go index 1dd43b36..bf19de7f 100644 --- a/internal/server/chat_test.go +++ b/internal/server/chat_test.go @@ -267,8 +267,8 @@ func chatFrame(kind, delta string, part *protocol.ChatPart, step *protocol.ChatS func scriptedChat() *fakeChat { return &fakeChat{script: []protocol.ChatEventFrame{ - chatFrame(protocol.ChatEventText, "hello ", &protocol.ChatPart{Type: protocol.ChatPartText, ID: "msg_1#0"}, nil), - chatFrame(protocol.ChatEventText, "world", &protocol.ChatPart{Type: protocol.ChatPartText, ID: "msg_1#0"}, nil), + chatFrame(protocol.ChatEventText, "hello ", &protocol.ChatPart{Type: protocol.ChatPartText, ID: protocol.ChatDeltaPartID("msg_1", protocol.ChatPartText, 0)}, nil), + chatFrame(protocol.ChatEventText, "world", &protocol.ChatPart{Type: protocol.ChatPartText, ID: protocol.ChatDeltaPartID("msg_1", protocol.ChatPartText, 0)}, nil), chatFrame(protocol.ChatEventTool, "", &protocol.ChatPart{Type: protocol.ChatPartTool, ID: "call_1", Name: "shell", State: &protocol.ChatToolState{Status: protocol.ChatToolCompleted, Input: []byte(`{"command":"ls"}`)}}, nil), chatFrame(protocol.ChatEventStep, "", nil, &protocol.ChatStep{Status: protocol.ChatStepEnded, Index: 1}), }} @@ -451,7 +451,7 @@ func TestChatMessagesResumesRetainedState(t *testing.T) { if message.ID != "msg_1" || message.Role != protocol.ChatRoleAssistant || len(message.Parts) != 2 { t.Fatalf("resumed message = %+v", message) } - if message.Parts[0].Type != protocol.ChatPartText || message.Parts[0].ID != "msg_1#0" || message.Parts[0].Text != "hello world" { + if message.Parts[0].Type != protocol.ChatPartText || message.Parts[0].ID != protocol.ChatDeltaPartID("msg_1", protocol.ChatPartText, 0) || message.Parts[0].Text != "hello world" { t.Fatalf("resumed text part = %+v", message.Parts[0]) } if message.Parts[1].Type != protocol.ChatPartTool || message.Parts[1].State == nil || message.Parts[1].State.Status != protocol.ChatToolCompleted { @@ -895,6 +895,50 @@ func TestChatHubChargesToolStateBytes(t *testing.T) { } } +// TestChatHubRetainsToolNameAndInputAcrossLifecycle pins that a tool call keeps +// its name and its input for its whole lifecycle in the retained state a resume +// serves: the name arrives only in input.started, the arguments in input.ended's +// text or called's input, and then progress/success carry neither, so no later +// frame may blank a field the hub already learned. +func TestChatHubRetainsToolNameAndInputAcrossLifecycle(t *testing.T) { + hub := newChatHub() + apply := func(part *protocol.ChatPart) { + hub.apply(protocol.ChatEventFrame{SessionID: "ses_1", MessageID: "msg_1", Kind: protocol.ChatEventTool, Part: part}) + } + // input.started: the name, no arguments yet. + apply(&protocol.ChatPart{Type: protocol.ChatPartTool, ID: "call_1", Name: "shell", State: &protocol.ChatToolState{Status: protocol.ChatToolStreaming}}) + // input.ended/called: the arguments, no name. + apply(&protocol.ChatPart{Type: protocol.ChatPartTool, ID: "call_1", State: &protocol.ChatToolState{Status: protocol.ChatToolRunning, Input: json.RawMessage(`{"command":"ls"}`)}}) + // progress: metadata only. + apply(&protocol.ChatPart{Type: protocol.ChatPartTool, ID: "call_1", State: &protocol.ChatToolState{Status: protocol.ChatToolRunning, Metadata: map[string]any{"exit": 0}}}) + // success: the output, no name and no input. + apply(&protocol.ChatPart{Type: protocol.ChatPartTool, ID: "call_1", State: &protocol.ChatToolState{Status: protocol.ChatToolCompleted, Content: []protocol.ChatToolContent{{Type: "text", Text: "ok"}}}}) + + messages := hub.snapshot("ses_1") + if len(messages) != 1 || len(messages[0].Parts) != 1 { + t.Fatalf("snapshot = %+v", messages) + } + part := messages[0].Parts[0] + if part.Name != "shell" { + t.Fatalf("tool name was blanked by a later frame: %+v", part) + } + if part.State == nil || part.State.Status != protocol.ChatToolCompleted { + t.Fatalf("tool state = %+v", part.State) + } + var input struct { + Command string `json:"command"` + } + if err := json.Unmarshal(part.State.Input, &input); err != nil || input.Command != "ls" { + t.Fatalf("tool input was blanked by a later frame: %s (err %v)", part.State.Input, err) + } + if len(part.State.Content) != 1 || part.State.Content[0].Text != "ok" { + t.Fatalf("tool content = %+v", part.State.Content) + } + if part.State.Metadata["exit"] == nil { + t.Fatalf("tool metadata was blanked by a later frame: %+v", part.State.Metadata) + } +} + // TestChatPromptSubscribesBeforePrompt pins the subscribe-before-prompt order: // the firehose has no replay cursor, so the subscription must be attached before // the prompt is posted or deltas emitted in between are lost. From 01d5dad9b9e0a7102f4571fea8f3221bc15be30a Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 01:50:55 +0100 Subject: [PATCH 2/3] docs: describe the agent-driven workflow, drop the scripted driver model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite the workflow documentation to describe how the system actually works: the agent (the session) is the brain and advances the queue by hand through the `pipelines` CLI. There is no orchestration script, no driver, no polling loop and no lane machinery, and none may be written. - references/work.md: rewritten end to end. The queue-driver script (drivers/workd.sh), the claim-file ledger with atomic renames, the PAUSED marker and the poll loop are gone. State is now the server (authoritative for run state), the queue file (authoritative for intent) and the conversation. The agent reads the queue, picks the item, creates its worktree/branch and launches it with `pipelines run --json --yes`; progress is event-driven via a background `pipelines wait [...]` (never polled); on completion the agent reconciles with `pipelines inspect --json`, verifies it landed and starts the next item. - references/queue.md: the landing gate runs the command directly (`git merge-base --is-ancestor main`, exit 0 = landed) instead of a `land-check.sh` helper; the never-delete-branch / never-remove-worktree rules are kept, and the waiting section states the agent advances the queue by hand. - references/orchestration.md: intro and "Running and watching" state the agent-driven, event-driven model; "Pipeline choice" names the current set (Android, TUI, Server, Generic, Plan and land, UI sweep, Probe) by area and records the retired names. - references/debugging.md: "Duplicated work after a driver restart" became "…after a session restart", resuming from the queue file not a cursor file. - SKILL.md: the routing row no longer says "queue driver". - AGENTS.md: the maintenance workflow names the current pipeline set instead of the retired `Pipelines maintenance - plan implement review`. - internal/tui/AGENTS.md: the restart line no longer names lanes. - docs/notes/agent-driven-work-model.md: the docs-fold note recording that docs/cli.md needs its "Scripted runs"/`--wait` framing aligned to the event-driven model, and that the dated evidence docs stay as they are. Checked before committing: no packaged template directory exists in this repo, so none needed updating; internal/author/install.go's managed file inventory needs no change (no packaged reference was added or removed — the 13 listed files all match internal/author/skill/references/); no stale `queue driver`/`workd.sh`/ `claim-file` references remain anywhere under internal/author. Genuine device tooling (android/scripts/capture_png.sh-style) and server-side polling prose were left alone. Checks: `go test ./internal/author -count=1` ok; `go build ./...` clean; `go vet ./...` clean; `gofmt -l .` clean. --- AGENTS.md | 18 +- docs/notes/agent-driven-work-model.md | 56 ++++ internal/author/skill/SKILL.md | 2 +- internal/author/skill/references/debugging.md | 19 +- .../author/skill/references/orchestration.md | 40 ++- internal/author/skill/references/queue.md | 8 +- internal/author/skill/references/work.md | 294 ++++++------------ internal/tui/AGENTS.md | 2 +- 8 files changed, 207 insertions(+), 232 deletions(-) create mode 100644 docs/notes/agent-driven-work-model.md diff --git a/AGENTS.md b/AGENTS.md index cc40fd17..4f100447 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,10 +31,14 @@ the single `docs-fold` writer rather than editing them per item: leave a ## Maintenance workflow -Top-level agents use the configured `Pipelines maintenance - plan implement review` -maintenance workflow for code changes. Inspect the retained plan/implement/review -outputs and diffs, execute the checks independently, and iterate on concrete -failures rather than trusting stage summaries. An executing stage performs its own -role directly and never recursively launches a workflow. If the workflow is -unavailable, ask instead of silently bypassing it. Never hardcode a disposable -server URL as a project default; the configured endpoint is authoritative. +Top-level agents advance work through the `pipelines` CLI; **the agent is the brain** +and there is no orchestration script, driver, polling loop or lane machinery, and none +may be written. Choose the pipeline by the area the work is in — `Android`, `TUI`, +`Server`, `Generic`, `Plan and land`, `UI sweep`, `Probe` — each reading the project's +own map (`android/AGENTS.md`, `internal/tui/AGENTS.md`, `internal/server/AGENTS.md`) +for how that technology is built, tested, run and captured. Inspect the retained +plan/implement/review outputs and diffs, execute the checks independently, and iterate +on concrete failures rather than trusting stage summaries. An executing stage performs +its own role directly and never recursively launches a workflow. If a pipeline is +unavailable, ask instead of silently bypassing it. Never hardcode a disposable server +URL as a project default; the configured endpoint is authoritative. diff --git a/docs/notes/agent-driven-work-model.md b/docs/notes/agent-driven-work-model.md new file mode 100644 index 00000000..90b412df --- /dev/null +++ b/docs/notes/agent-driven-work-model.md @@ -0,0 +1,56 @@ +# Work documentation now describes the agent-driven model + +## What this item changed + +The workflow documentation was rewritten to describe how the system actually works: +the agent (the session) is the brain and advances the queue by hand through the +`pipelines` CLI. There is **no orchestration script, no driver, no polling loop and no +lane machinery, and none may be written.** + +- `internal/author/skill/references/work.md` — rewritten end to end. The queue-driver + script (`drivers/workd.sh`), the claim-file ledger with atomic renames, the `PAUSED` + marker and the poll loop are gone. The ledger is now the per-project markdown + `queue.md` (authoritative for intent) plus `notes.md`; run state is read from the + server (`pipelines inspect --json`); the agent launches each run itself with + `pipelines run --json --yes` from the item's worktree and is woken + by a background `pipelines wait`. +- `internal/author/skill/references/queue.md` — the landing gate now runs the command + directly (`git merge-base --is-ancestor main`, exit 0 = landed) instead of a + `land-check.sh` helper; the waiting section states the agent advances the queue by + hand and there is no driver. +- `internal/author/skill/references/orchestration.md` — the intro and "Running and + watching" state the agent-driven, event-driven model; "Pipeline choice" now names the + current set (`Android`, `TUI`, `Server`, `Generic`, `Plan and land`, `UI sweep`, + `Probe`), chosen by area, and records the retired names. +- `internal/author/skill/references/debugging.md` — "Duplicated work after a driver + restart" became "…after a session restart", resuming from the queue file rather than a + cursor file. +- `internal/author/skill/SKILL.md` — the routing table row no longer says "queue driver". +- `AGENTS.md` — the maintenance workflow section names the current pipeline set instead + of the retired `Pipelines maintenance - plan implement review`. +- `internal/tui/AGENTS.md` — the "do not restart … the lanes …" line no longer names + lanes. + +`delivery.md`, `operations.md` and `verification.md` were swept and needed no change: +they contain no driver, poll loop, cursor file or lane guidance. Genuine device tooling +(`android/scripts/capture_png.sh`) is left alone. + +## Shared documents this change makes stale + +No README or topic doc still teaches the scripted driver model. One topic doc needs a +wording alignment for the event-driven model; the `docs-fold` writer should apply it +when it next folds notes: + +- `docs/cli.md` — its "Scripted runs" section frames runs as scripted automation, and + its `--wait` paragraph is the only follow-a-run guidance. It should state that the + agent (not a script) advances work, and that `pipelines wait [run-id ...]` + run as a **background task** is the event-driven wait. The `wait` command's own flag + text is already recorded separately in `docs/notes/wait-command.md`. + +## Do not change + +- The dated evidence documents (`docs/pipeline-performance-review.md`, + `docs/chat-verification.md`, `docs/app-integration-verification.md`) describe the + scripted era as it was and stay as they are. +- `internal/author/install.go`'s managed file inventory needs no change: no packaged + reference file was added or removed. diff --git a/internal/author/skill/SKILL.md b/internal/author/skill/SKILL.md index 68ec01d2..93942ffc 100644 --- a/internal/author/skill/SKILL.md +++ b/internal/author/skill/SKILL.md @@ -59,7 +59,7 @@ and `references/format.md` (in parallel when possible); for notifications also r | Task | Packaged reference | | --- | --- | | Briefs, work items, pipeline choice, running and watching | `references/orchestration.md` | -| Across projects: work ledger, start/pause/resume/status, queue driver | `references/work.md` | +| Across projects: work ledger, start/pause/resume/status, the queue | `references/work.md` | | The work queue, parallel items and landing | `references/queue.md` | | Diagnose a wedged, blocked or failed run | `references/debugging.md` | | Publish artefacts, LAN URLs, install boundaries | `references/delivery.md` | diff --git a/internal/author/skill/references/debugging.md b/internal/author/skill/references/debugging.md index 5f8c0da0..54e82afd 100644 --- a/internal/author/skill/references/debugging.md +++ b/internal/author/skill/references/debugging.md @@ -130,13 +130,16 @@ Each playbook is *symptom → how to confirm → what to do*. approval. Put the capture helper in the repository so every UI run reuses one path. See the example in `references/delivery.md`. -## Duplicated work after a driver restart +## Duplicated work after a session restart -- **Symptom** Items already completed run again after a queue driver - restarts: the same brief is re-invoked, side effects repeat, or a commit is made +- **Symptom** Items already completed run again after the session that was advancing the + queue restarts: the same brief is re-invoked, side effects repeat, or a commit is made twice. -- **Confirm** Compare the queue's items against their recorded state. A list alone - has no resume point. -- **Do** Record each item's state in the queue itself (`references/queue.md`) and resume from it. Record an item - only once it has landed, and skip recorded items on startup. Never relaunch a - duplicate of a run that is still active — reattach with its request key instead. +- **Confirm** Compare the queue's items against their recorded state and the server's run + history (`inspect RUN_ID --json`). A list alone has no resume point. +- **Do** Record each item's state in the queue itself (`references/queue.md`) and resume + from it: read the queue top to bottom, skip items already marked done, and settle the + ones marked running by inspecting their run id. Record an item only once it has landed. + Never relaunch a duplicate of a run that is still active — reattach with its request key + instead. Do not build a driver or cursor file to remember position; the queue file is + the resume point. diff --git a/internal/author/skill/references/orchestration.md b/internal/author/skill/references/orchestration.md index c36ddce7..aadfbc80 100644 --- a/internal/author/skill/references/orchestration.md +++ b/internal/author/skill/references/orchestration.md @@ -5,6 +5,11 @@ briefs and a set of managed runs, not the change itself. The implementer stage m the change; the reviewer stages check it; the committer stage commits it. Route all project work through Pipelines runs. +The agent advances this **by hand through the `pipelines` CLI**: it launches each run, +is woken when it finishes, reconciles against the server, and starts the next item. +There is no orchestration script, driver, polling loop or lane machinery, and none may +be written. + ## Stance - **Reconnoitre first.** Read enough of the repository, its notes and recent history @@ -141,6 +146,24 @@ pipelines run "Docs fold" ## Pipeline choice +Choose by the **area the work is in**; each area pipeline reads the project's own map +for its build, test, run and capture commands: + +- **`Android`** — `android/`. +- **`TUI`** — `internal/tui/`. +- **`Server`** — `internal/server/`, `internal/stage/`, `internal/pipeline/` and + `internal/protocol/`. +- **`Generic`** — only when a change genuinely spans areas. +- **`Plan and land`** — when the change needs a plan before implementation + (contract-level work). +- **`UI sweep`** — a full-screen review pass. +- **`Probe`** — a harness smoke check. + +The retired names (`Pipelines quick`/`maintenance`/`critical`, `Focused tweak`, `Quick +tweak`, `Plan, implement, review`, `Harness probe`) stay in the registry because +definitions are immutable, but nothing should start them. If a pipeline is unavailable, +ask instead of silently bypassing it. + - Use a pipeline that keeps **implementer + reviewer + ui_reviewer + committer** for anything that matters. The planner is the only stage a leaner variant may drop. - **Never remove a review stage to go faster.** The reviewer stages are the @@ -162,9 +185,10 @@ pipelines run "Docs fold" - A run cannot approve its own work. When an item genuinely needs an operator gate before it continues or commits, insert a `type=human` decision stage (`references/format.md`); never invent a model stage that pretends to approve. -- Choose per item: a logic, server or format change usually wants the planner - (a plan/implement/review pipeline); a small, fully understood tweak may use the - leaner implementer-first variant. Both keep all review stages. +- Pick the pipeline whose area matches the change (above); `Generic` only when a change + genuinely spans areas. A logic, server or format change usually wants the planner + (`Plan and land`); a small, fully understood tweak may use the leaner area pipeline. + Both keep all review stages. ## Running and watching @@ -179,9 +203,13 @@ pipelines run "Docs fold" and are recorded on the run; `retry` reuses them. See `references/harnesses.md` for how a stage's model and its repair path both use the effective model. - Watch to a **terminal state** (`succeeded`, `failed`, `canceled`, `interrupted`). - Poll `CLI_PREFIX inspect RUN_ID --json` (or use `--wait`) until terminal. A reached - client wait limit is not a run failure: the run stays active; keep waiting or - reattach with the original request key, never start a duplicate run. + Progress is **event-driven, never polled**: `CLI_PREFIX wait RUN_ID […]` blocks until + the runs are terminal and is run as a **background task**, so the session is woken + exactly when the batch finishes; `CLI_PREFIX inspect RUN_ID --json` then reads the + retained outcome. A reached client wait limit is not a run failure: the run stays + active; keep waiting or reattach with the original request key, never start a + duplicate run. Do not build a polling loop, a cursor file or a driver script to + answer "is it done yet". - Read the **outcomes**, not just the status: the option each stage returned, the evidence it recorded, and any run or step diagnostic. Interpret diagnostics by category (`references/debugging.md`). diff --git a/internal/author/skill/references/queue.md b/internal/author/skill/references/queue.md index c4b51d0b..246d4f7f 100644 --- a/internal/author/skill/references/queue.md +++ b/internal/author/skill/references/queue.md @@ -49,8 +49,9 @@ Serialize only when: ## The landing gate, and what must never happen -- After every item, and **before any cleanup or before starting the next item**, run - `land-check.sh `. It must report the branch as an **ancestor of main**. +- After every item, and **before any cleanup or before starting the next item**, verify + the branch is an **ancestor of main** directly, with no helper script: + `git merge-base --is-ancestor main` (exit 0 = landed). - If it refuses: do not remove the worktree, do not touch the branch, do not start the next item. Report the landing failure and the commits that are not in main. - **Never delete a branch.** Deleting a branch whose commits are not yet in main leaves them @@ -60,6 +61,9 @@ Serialize only when: ## Waiting without a scripting harness +- The agent advances the queue **by hand**: read it, create the item's worktree, and launch + `pipelines run --json --yes` from that worktree. Nothing else advances it — + there is no driver, and none may be written. - `pipelines wait […​]` blocks until the runs finish and exits; run it as a **background task** so the session is woken exactly when the batch is done. A run that failed is still a finished run: it exits 0 and says what happened. diff --git a/internal/author/skill/references/work.md b/internal/author/skill/references/work.md index 03dfeb69..5308c137 100644 --- a/internal/author/skill/references/work.md +++ b/internal/author/skill/references/work.md @@ -1,256 +1,136 @@ # Project work ledger and sessions Work in a project is not a chat. It is a durable ledger: what is queued, what is -running, what has landed, and what is blocked. A session — this conversation — starts, -pauses, resumes and reports that ledger. Nothing important lives only in the -conversation: running the skill again, in any project on this machine, reads the -ledger and continues. This is the multi-project lifecycle on top of +running, what has landed, and what is blocked. A session — this conversation — reads +the ledger, launches the runs, is woken when they finish, and reports. + +**The agent is the brain.** It advances the queue by hand through the `pipelines` +CLI: it reads the queue, picks the item, launches the run(s) itself, is woken when they +finish, reconciles against the server, and starts the next item. There is **no +orchestration script, no driver, no polling loop and no lane machinery, and none may be +written.** Where a check is needed, run the command directly. + +State lives in exactly three places: + +- the **server** is authoritative for run state — `pipelines inspect --json`; +- the **queue file** is authoritative for intent — what is queued, in what order, and + what is finished; +- the **conversation** holds this session's working memory. + +Never keep progress in a script's own files. A session that dies and comes back reads +the queue file and continues. This is the multi-project lifecycle on top of `references/orchestration.md` (work items, briefs, pipeline choice) and -`references/queue.md` (parallel items). +`references/queue.md` (parallel items, one worktree per item, landing). ## Where the ledger lives -Keep state **outside repositories**, under one work root, so it never collides with a +Keep state **outside repositories**, under one work root, so it never collides with an item's worktree or a commit: ${XDG_STATE_HOME:-$HOME/.local/state}/pipelines/work/ - registry.json every project this machine knows + registry.json every project this machine knows projects// - state.json project, root, endpoint, pipeline, workspace - notes.md recon, decisions, open questions, blockers - queue/.brief pending briefs, read in filename order - active/.brief, .claim claimed: the brief plus its run id and key - done/ landed: run id, status, finished - failed/ and .brief failed twice: run id, status, diagnostic - log/driver.log - drivers/workd.sh this project's queue driver - PAUSED marker: stop dispatching new items + queue.md the project's markdown queue: items, briefs, state + notes.md recon, decisions, open questions, blockers - `` is `-<8 hex of sha256(absolute project root)>`. **Resolve a - worktree to its main repository** so every item shares the project's one ledger. + worktree to its main repository** so every item shares the project's one queue. - `PIPELINES_WORK_DIR` overrides the work root. `registry.json` maps projects to roots; add a project the first time it is worked on and keep it current. -- `state.json` holds `project`, `root`, `endpoint` (the CLI's `--url`), `pipeline` - (the default pipeline) and `workspace` (the default run directory). -- The ledger is the operator's record: keep `notes.md` truthful about decisions taken - and left open, never narrate it as a diary. +- `queue.md` is the resume point (`references/queue.md`): read it top to bottom and + take the first unfinished item. `notes.md` is the operator's record: keep it truthful + about decisions taken and left open, never narrate it as a diary. +- The CLI's configured endpoint and the area map's default pipeline come from the CLI + and the project's own map, not from a script. Do not write a per-project state file + for a script to read. ## Operations Map one operator sentence to one operation. Do the operation, then answer with the -status block — nothing else. +status block — nothing else. **You launch the runs yourself**; nothing else does it. | Operator says | Do | | --- | --- | -| start / work on X | Reconnoitre, register the project, write small briefs into `queue/`, then run the driver. | -| status / how is it going | Print the status block. | -| queue "…" / add … | Reconnoitre, split into small items, append briefs; the driver picks them up. | -| pause / stop for now | Write `PAUSED`. The item in flight finishes; no new one starts. | -| resume / continue | Reconcile, remove `PAUSED`, run the driver. | -| drop item N | Move its brief to `failed/` with a reason. Never silently delete work. | -| done / stop everything | Pause and report what landed and what remains. | +| start / work on X | Reconnoitre, register the project, write the queue file, then launch the first item's run yourself. | +| status / how is it going | Read the server and the queue, then print the status block. | +| queue "…" / add … | Reconnoitre, split into small items, append them to the queue file, launch what is ready. | +| pause / stop for now | Launch nothing new. The runs already in flight finish; reconcile them when they do. | +| resume / continue | Reconcile against the server, then launch the next item. | +| drop item N | Mark its item failed in the queue file with a reason. Never silently delete work. | +| done / stop everything | Stop launching, wait for what is in flight, and report what landed and what remains. | **Reconcile before acting, always.** The server is authoritative for run state; the -ledger is authoritative for intent. On every invocation: read `registry.json` and the -project's ledger; for each `active/` claim, `CLI_PREFIX inspect --json` and -settle it (running → keep waiting; succeeded → `done/`; failed/interrupted → -`failed/`); only then do the requested operation. Never trust a claim file for a -status, and never start a second run for an item whose request key is already -admitted — that key is what makes a redispatch attach instead of duplicate. +queue file is authoritative for intent. On every invocation: read the project's +`queue.md`; for each item marked running, `pipelines inspect --json` and settle +it (running → keep waiting; succeeded → verify it landed and mark it done; +failed/interrupted → mark it failed with the diagnostic). Only then do the requested +operation. Never trust your memory of a run's status, and never start a second run for +an item whose request key is already admitted — reuse the original `--request-key` so a +redispatch attaches instead of duplicating. Status block (these lines, no prose around them): pipelines — active queue: 4 pending · 1 running (run 185 · reviewer · 6m) · 12 done · 2 failed - next: 0005-graph-arrows on "Focused tweak - implement review commit" + next: 0005-graph-arrows on "TUI" blocked: none ## The brief, and per-item directives -Write every brief exactly as `references/orchestration.md` describes. A brief file in -`queue/` may open with up to three directive lines, stripped before the prompt is sent: +Write every brief exactly as `references/orchestration.md` describes. An item in +`queue.md` may open its brief with up to three directive lines, stripped before the +prompt is sent: - # pipeline: Focused tweak - implement review commit + # pipeline: TUI # workspace: /path/to/item-worktree # device: emulator-5556 Goal: … -`pipeline` overrides the default for that item; `workspace` routes an item to its own -checkout (a driver only claims items for its own workspace); `device` is for the -brief's own text. Number briefs `0001-…`, `0002-…` so order is explicit. +`pipeline` overrides the default for that item; `workspace` names the item's own +worktree (create it with `git worktree add -b main`, `references/queue.md`); +`device` is for the brief's own text. Number items `0001-…`, `0002-…` so order is +explicit. -## The driver +## Advancing the queue by hand -A queue is advanced by a small driver, so work continues while the operator sleeps. -Write the driver once per project to `drivers/workd.sh` and launch it; it is -restart-safe (it resumes from `done/`, reattaches to `active/` runs by id, and never -starts a duplicate), and it stops on the first item that fails twice. Run **one -item at a time per worktree**; several items in their own worktrees advance in -parallel because an item is claimed by an atomic rename. +There is no driver. You advance the queue yourself, through the CLI: -```sh -#!/bin/sh -# workd.sh - advance one project's Pipelines work ledger. -# -# sh workd.sh [PROJECT_ROOT] [WORKSPACE] -# -# PROJECT_ROOT identifies the ledger (default: the current directory). -# WORKSPACE is the checkout this driver may run in (default: the ledger's -# workspace). One item per worktree lets several items advance against one shared -# queue: an item is claimed by an atomic rename, so it runs exactly once. -# -# Restart-safe. It resumes from done/, reattaches to active/ runs by id and -# never starts a duplicate. PIPELINES_WORK_POLL sets the poll gap in seconds -# (default 20); PIPELINES_CLI uses another client binary. -set -u +1. **Read the queue top to bottom** and take the first unfinished item. +2. **Choose the pipeline by area** (`references/orchestration.md`): `Android`, `TUI`, + `Server`, `Generic`, `Plan and land`, `UI sweep`, `Probe`. Each area pipeline reads + the project's own map for its build, test, run and capture commands. +3. **Create the item's worktree and branch**: `git worktree add -b main`. + One worktree per item is what lets independent items run at once without the + workspace guard becoming a bottleneck (`references/queue.md`). +4. **Launch the run from that worktree**: + `pipelines run --json --yes`. Keep the returned run id. +5. **Be woken when it finishes — do not poll.** `pipelines wait […]` blocks + until the runs are terminal and is run as a **background task**, so the session is + woken exactly when the batch finishes — or arm a timer and check on the tick. No + polling loop, no cursor file, no driver script, ever. +6. **Reconcile.** `pipelines inspect --json`; read the outcomes and evidence; + verify the branch landed with `git merge-base --is-ancestor main` (exit 0 = + landed); mark the item in `queue.md`; then start the next item — or several + independent items at once. -CLI=${PIPELINES_CLI:-pipelines} -POLL=${PIPELINES_WORK_POLL:-20} -ROOT=$(cd "${1:-$(pwd)}" && pwd) -WORK_ROOT=${PIPELINES_WORK_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/pipelines/work} -SLUG=$(basename "$ROOT")-$(printf '%s' "$ROOT" | shasum -a 256 | cut -c1-8) -D=$WORK_ROOT/projects/$SLUG -LOG=$D/log/driver.log -mkdir -p "$D/queue" "$D/active" "$D/done" "$D/failed" "$D/log" 2>/dev/null +**Parallelism is your judgement, bounded by real conflicts only**: two items editing +the same files at once, the one capture device, or the provider's spend cap. Launch as +many independent runs as the machine can carry without overload; serialise only for a +real conflict and say which limit is binding. Never present serialisation as the +operator's decision. -say() { printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" | tee -a "$LOG"; } -jfield() { python3 -c 'import json,sys -try: d=json.load(open(sys.argv[1])) -except Exception: d={} -print(d.get(sys.argv[2]) or "")' "$1" "$2"; } -dstatus() { "$CLI" inspect "$1" --json 2>/dev/null | python3 -c 'import json,sys -try: d=json.load(sys.stdin) -except Exception: print("unknown"); raise SystemExit -print(d.get("status") or "unknown")'; } -item_field() { sed -n "s/^# *$2 *: *//p" "$1" 2>/dev/null | head -1; } # from a brief -claim_field() { sed -n "s/^$2=//p" "$1" 2>/dev/null | head -1; } # from a claim file -prompt_of() { python3 -c ' -import re,sys -lines=open(sys.argv[1],encoding="utf-8",errors="replace").read().splitlines() -while lines and re.match(r"^#\s*(pipeline|workspace|device)\s*:", lines[0]): - lines.pop(0) -while lines and lines[0].strip()=="": - lines.pop(0) -print("\n".join(lines))' "$1"; } - -[ -f "$D/state.json" ] || { echo "no ledger: $D/state.json" >&2; exit 2; } -PIPELINE=$(jfield "$D/state.json" pipeline) -ENDPOINT=$(jfield "$D/state.json" endpoint) -WS=${2:-$(jfield "$D/state.json" workspace)} -[ -n "$WS" ] || WS=$ROOT -[ -n "$PIPELINE" ] || { echo "state.json has no pipeline" >&2; exit 2; } - -wait_terminal() { # $1 run id -> terminal status (or "unknown" if unreadable) - bad=0 - while :; do - st=$(dstatus "$1") - case "$st" in - succeeded|failed|canceled|interrupted) echo "$st"; return 0;; - unknown) bad=$((bad+1)); [ "$bad" -ge 5 ] && { echo "unknown"; return 0; };; - *) bad=0;; - esac - sleep "$POLL" - done -} - -dispatch() { # $1 brief file $2 pipeline $3 key -> run id (empty on failure) - if [ -n "$ENDPOINT" ]; then - out=$("$CLI" run "$2" "$(prompt_of "$1")" --json --yes --workspace "$WS" --request-key "$3" --url "$ENDPOINT" 2>&1) - else - out=$("$CLI" run "$2" "$(prompt_of "$1")" --json --yes --workspace "$WS" --request-key "$3" 2>&1) - fi - printf '%s' "$out" | python3 -c 'import json,sys -try: d=json.load(sys.stdin) -except Exception: print("") -else: print((d.get("run_detail") or {}).get("id") or "")' -} - -record() { # $1 id $2 status $3 run id - where=done; [ "$2" = succeeded ] || where=failed - printf 'run_id=%s\nstatus=%s\nfinished=%s\n' "$3" "$2" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "$D/$where/$1" - rm -f "$D/active/$1.brief" "$D/active/$1.claim" -} - -run_item() { # $1 id $2 brief $3 pipeline $4 key -> 0 succeeded / 1 failed - rid=$(dispatch "$2" "$3" "$4") - if [ -z "$rid" ]; then printf 'status=dispatch-failed\n' > "$D/active/$1.claim"; say "dispatch failed: $1"; return 1; fi - printf 'run_id=%s\n' "$rid" > "$D/active/$1.claim" - st=$(wait_terminal "$rid") - if [ "$st" = succeeded ]; then record "$1" succeeded "$rid"; say "done $1 (run $rid)"; return 0; fi - printf 'run_id=%s\nstatus=%s\n' "$rid" "$st" > "$D/active/$1.claim" - say "failed $1 (run $rid, $st)" - return 1 -} - -reconcile() { # settle anything the previous process left claimed - for a in "$D"/active/*; do - [ -e "$a" ] || continue - id=$(basename "$a"); id=${id%.brief}; id=${id%.claim} - rid=$(claim_field "$D/active/$id.claim" run_id) - if [ -z "$rid" ]; then - say "requeueing undispatched $id" - [ -e "$D/active/$id.brief" ] && mv "$D/active/$id.brief" "$D/queue/$id.brief" - rm -f "$D/active/$id.claim" - continue - fi - st=$(wait_terminal "$rid") - record "$id" "$st" "$rid" - say "reconciled $id as $st (run $rid)" - done -} - -reconcile - -while :; do - if [ -e "$D/PAUSED" ]; then sleep "$POLL"; continue; fi - next="" - for q in $(ls -1 "$D"/queue/*.brief 2>/dev/null | sort); do - id=$(basename "$q" .brief) - [ -e "$D/done/$id" ] && continue - [ -e "$D/failed/$id" ] && continue - iws=$(item_field "$q" workspace) - [ -n "$iws" ] && [ "$iws" != "$WS" ] && continue - next=$q; break - done - if [ -z "$next" ]; then say "idle (no claimable items for $WS)"; exit 0; fi - id=$(basename "$next" .brief) - mv "$next" "$D/active/$id.brief" 2>/dev/null || continue - ipipe=$(item_field "$D/active/$id.brief" pipeline) - [ -n "$ipipe" ] || ipipe=$PIPELINE - key="work-$SLUG-$id" - if run_item "$id" "$D/active/$id.brief" "$ipipe" "$key"; then continue; fi - if [ "$(claim_field "$D/active/$id.claim" status)" = "dispatch-failed" ]; then - say "STOP: could not dispatch $id" - mv "$D/active/$id.brief" "$D/failed/$id.brief" 2>/dev/null - exit 1 - fi - say "retrying $id once" - if run_item "$id" "$D/active/$id.brief" "$ipipe" "$key-r1"; then continue; fi - rid=$(claim_field "$D/active/$id.claim" run_id) - st=$(claim_field "$D/active/$id.claim" status) - printf 'run_id=%s\nstatus=%s\nattempts=2\n' "$rid" "${st:-failed}" > "$D/failed/$id" - mv "$D/active/$id.brief" "$D/failed/$id.brief" 2>/dev/null - rm -f "$D/active/$id.claim" - say "STOP: $id failed twice; recorded under failed/" - exit 1 -done -``` - -Launch it detached and keep its log: - - nohup sh /projects//drivers/workd.sh >/dev/null 2>&1 & - -The driver only *advances* a prepared queue; it never writes briefs. Preparing a good -queue and settling a failure is the agent's work. +**Landing stays the pipeline's job**: the committer stage merges and lands its own +work. There is no merger, no resolver and nothing to hand to a script. You verify the +landing gate (`references/queue.md`); you do not perform the merge. ## Sleeping and waking -- Before the operator sleeps: reconcile, make sure the queue is full enough to cover - the night, and confirm the driver is running. A drained queue is fine; it idles. -- Configure a **failure notification** (`references/notifications.md`) so a stopped - driver is not silent. On an overnight failure the driver stops after one retry; that - is by design — do not add retry loops to make it look busy. +- Before the operator sleeps: reconcile, make sure the queue has enough items to cover + the night, and launch what is ready with a background `pipelines wait`. +- Configure a **failure notification** (`references/notifications.md`) so a failed run + is not silent. A run that fails is recorded and the next item starts; do not add + retry loops to make it look busy. - On waking, reconcile first, then answer with the status block: what landed, what failed and why, what is next. @@ -262,7 +142,7 @@ The operator wants the result, not the shift report. two sentences; never answer a narrow question with a sectioned report. - Answer the question that was asked first. Put any extra fact that changes a decision in a single clause, not in a new section. -- Answer with the status block for status; do not narrate polling or re-print the +- Answer with the status block for status; do not narrate waiting or re-print the queue when nothing changed. - Report a run by its outcome and evidence (`references/verification.md`), not the steps you took to watch it. diff --git a/internal/tui/AGENTS.md b/internal/tui/AGENTS.md index 09886d5b..06f7275a 100644 --- a/internal/tui/AGENTS.md +++ b/internal/tui/AGENTS.md @@ -42,5 +42,5 @@ say why is a review finding. you checked. - Terminal width matters: verify at a narrow width as well as a wide one, and never assume colour support (`plain-nocolor-*` exists for a reason). -- Do not restart the deployed server, the lanes or ntfy to test a TUI change: run the binary against a +- Do not restart the deployed server or ntfy to test a TUI change: run the binary against a scratch server or use the fixtures. From cff6f4dfacf07244c572b2f864786b01586fe4c6 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 02:23:58 +0100 Subject: [PATCH 3/3] server: write a subscribe snapshot in full before its ack handleRunEvents seeded a new subscription by pushing every matching run through the subscription channel (runEventSubscriberBuffer = 32) and only then acked. sendMessage is a non-blocking send that closes the subscription when the channel is full, so a snapshot larger than the buffer dropped the subscription it was seeding: the remaining snapshot frames hit the closed-done early return, the ack was never sent, and no later delta could arrive. The connection stayed open, so the client could not tell it was no longer subscribed. This was the app's only subscription - AppViewModel subscribes with a blank workspace, matching every run - so the app's live state froze. The subscription now starts in a snapshot phase. The subscribing connection itself writes the snapshot frames via writeSnapshot (one direct write per frame, outside the bounded channel, so the drop policy cannot fire on them) and the ack, and ready then moves any deltas held meanwhile into the channel in order and starts the writer goroutine. sendMessage holds deltas in a bounded pending queue while snapshotting instead of closing, so the drop-a-slow-subscriber policy still applies, but only to deltas. Ordering is snapshot, ack, then deltas; a write failure or a listRuns error closes the subscription instead of leaving it registered. Numbers reported by this run: the probe against the deployed server saw blank-scope frames=2 ack=0 (scoped frames=1 ack=0) because the snapshot was ~252 frames into a 32-frame buffer; the scratch-server probe with the fix saw 40 snapshot frames, ack=1 and a delta after a run transition. Guarded now by TestRunEventSubscribeSnapshotLargerThanBuffer (seeds 40 finished runs, asserts every snapshot frame, the ack strictly after them, and a terminal delta for a run started after subscribing), which fails against the previous runevents.go. Checked for staleness: the packaged templates and the packaged authoring skill references internal/author/skill/references/ carry no run-event subscription contract (their "snapshot" mentions are the workspace-change snapshots in operations.md), and README carries none either, so none of them needs updating. No UI surface changed. Checks run before committing: gofmt -l . prints nothing, go build ./..., go vet ./..., go test ./... (all packages ok), go test -race ./internal/server (ok). --- internal/server/run_event_websocket_test.go | 90 +++++++++++++++ internal/server/runevents.go | 120 +++++++++++++++++--- 2 files changed, 194 insertions(+), 16 deletions(-) diff --git a/internal/server/run_event_websocket_test.go b/internal/server/run_event_websocket_test.go index b6ba611e..f2ec8df8 100644 --- a/internal/server/run_event_websocket_test.go +++ b/internal/server/run_event_websocket_test.go @@ -2,6 +2,7 @@ package server import ( "context" + "fmt" "net/http/httptest" "path/filepath" "strings" @@ -212,6 +213,95 @@ func TestRunEventSubscribeSendsSnapshot(t *testing.T) { } } +// TestRunEventSubscribeSnapshotLargerThanBuffer proves a subscribe whose +// snapshot holds more runs than the subscription's bounded buffer delivers the +// snapshot in full, then the ack, then a live delta for a run started +// afterwards. Before the snapshot fix, the subscription wrote its own snapshot +// through the bounded channel, so sendMessage's drop-a-slow-subscriber policy +// fired on the snapshot itself: the subscription closed part-way through, the +// client saw at most runEventSubscriberBuffer frames, and it never received the +// ack or a single later delta. +func TestRunEventSubscribeSnapshotLargerThanBuffer(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, handler, ids := eventStore(t, ctx, "one") + handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + const seeded = runEventSubscriberBuffer + 8 + // Seed more finished runs than the buffer holds, each in its own workspace so + // no two runs share a tree, then wait for every one to settle. A blank-scope + // subscribe sees them all. + seededIDs := make([]int64, 0, seeded) + for i := 0; i < seeded; i++ { + run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: fmt.Sprintf("snapshot-%d", i), Workspace: eventWorkspace(t), Input: map[string]any{"prompt": "test"}}) + if err != nil { + t.Fatal(err) + } + seededIDs = append(seededIDs, run.ID) + } + handler.runner.wg.Wait() + + client := dialRunEvents(t, ctx, handler) + client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: ""}) + snapshotCtx, cancelSnapshot := context.WithTimeout(ctx, 8*time.Second) + defer cancelSnapshot() + seen := map[int64]bool{} + var ack *protocol.Message + for ack == nil || len(seen) < seeded { + var message protocol.Message + if err := wsjson.Read(snapshotCtx, client.conn, &message); err != nil { + t.Fatalf("snapshot stopped after %d of %d frames (ack=%v): %v", len(seen), seeded, ack != nil, err) + } + switch message.Type { + case protocol.RunEvent: + if message.RunEvent != nil { + seen[message.RunEvent.RunID] = true + } + case protocol.Subscribe: + got := message + ack = &got + // The snapshot precedes the ack: when the ack is read every snapshot + // frame must already have been, or a frame was reordered after it. + if len(seen) != seeded { + t.Fatalf("ack arrived after %d of %d snapshot frames", len(seen), seeded) + } + } + } + if ack.ID != "snap" { + t.Fatalf("subscribe ack = %+v", *ack) + } + for _, id := range seededIDs { + if !seen[id] { + t.Fatalf("snapshot omitted run %d: got %d of %d frames", id, len(seen), seeded) + } + } + + // The subscription must outlive its own snapshot: a run started after it must + // still deliver a delta, through to its terminal frame. + late := eventWorkspace(t) + client.send(protocol.Message{Type: protocol.RunPipeline, ID: "late", PipelineID: ids[0], Version: 1, RequestKey: "snapshot-late", Workspace: late, Input: map[string]any{"prompt": "test"}}) + deltaCtx, cancelDelta := context.WithTimeout(ctx, 8*time.Second) + defer cancelDelta() + var terminal *protocol.RunEventFrame + for terminal == nil { + var message protocol.Message + if err := wsjson.Read(deltaCtx, client.conn, &message); err != nil { + t.Fatalf("no terminal delta after the snapshot: %v", err) + } + if message.Type != protocol.RunEvent || message.RunEvent == nil || message.RunEvent.Workspace != late { + continue + } + switch message.RunEvent.Status { + case "succeeded", "failed", "canceled": + terminal = message.RunEvent + } + } + if terminal.RunID == 0 || terminal.Outcome != "success" { + t.Fatalf("late terminal frame = %+v", terminal) + } +} + // TestMergeRunEventFrameKeepsIdentity proves a coalesced frame keeps the run's // identity from an earlier transition when a later delta omits it, so the merged // frame a client actually receives still carries what its row needs. diff --git a/internal/server/runevents.go b/internal/server/runevents.go index b17ef92c..e35d683d 100644 --- a/internal/server/runevents.go +++ b/internal/server/runevents.go @@ -54,9 +54,16 @@ type runCoalescer struct { } // runSubscription is one connection's watch over a workspace (and optionally -// explicit run ids). Frames are handed to a dedicated writer goroutine so a slow +// explicit run ids). Deltas are handed to a dedicated writer goroutine so a slow // or broken subscriber never stalls the producer, and its buffered channel // bounds how far behind it may fall. +// +// A subscription starts in its snapshot phase: the subscribing connection +// writes the snapshot and the ack itself (writeSnapshot) and only then starts +// the writer goroutine (ready). mu guards that phase, so the drop-a-slow- +// subscriber policy can never fire on the snapshot or lose its ack: while +// snapshotting a delta is held in pending instead of the bounded channel, and +// nothing writes to the channel until ready has released the held frames. type runSubscription struct { hub *runEventHub clientID string @@ -66,6 +73,10 @@ type runSubscription struct { done chan struct{} once sync.Once write func(protocol.Message) error + + mu sync.Mutex + snapshotting bool + pending []protocol.Message } func newRunEventHub() *runEventHub { @@ -85,8 +96,11 @@ func (h *runEventHub) clock() time.Time { return time.Now() } -// subscribe registers one connection's watch and starts its writer goroutine. It -// returns nil when the connection already holds the per-client subscription cap. +// subscribe registers one connection's watch and returns it in its snapshot +// phase: the caller delivers the snapshot and the ack with writeSnapshot and +// then calls ready, which releases the deltas emitted meanwhile and starts the +// writer goroutine. It returns nil when the connection already holds the +// per-client subscription cap. func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, write func(protocol.Message) error) *runSubscription { if h == nil || write == nil { return nil @@ -97,12 +111,13 @@ func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, writ return nil } sub := &runSubscription{ - hub: h, - clientID: clientID, - workspace: workspace, - ch: make(chan protocol.Message, runEventSubscriberBuffer), - done: make(chan struct{}), - write: write, + hub: h, + clientID: clientID, + workspace: workspace, + ch: make(chan protocol.Message, runEventSubscriberBuffer), + done: make(chan struct{}), + write: write, + snapshotting: true, } for _, id := range runIDs { if id <= 0 { @@ -119,7 +134,8 @@ func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, writ } h.byClient[clientID][sub] = struct{}{} h.mu.Unlock() - go sub.run() + // The writer goroutine starts only in ready, after the snapshot and the ack + // are on the wire, so a delta can never overtake them. return sub } @@ -320,6 +336,47 @@ func (s *runSubscription) run() { } } +// writeSnapshot writes one frame of the subscription's snapshot directly on the +// subscribing connection. It runs only during the snapshot phase, before the +// writer goroutine exists, so it cannot reorder against a delta and is never +// subject to the bounded channel's drop policy. It returns false when the frame +// could not be written, having closed the subscription; the caller must then +// stop writing the snapshot. +func (s *runSubscription) writeSnapshot(msg protocol.Message) bool { + select { + case <-s.done: + return false + default: + } + if err := s.write(msg); err != nil { + s.close() + return false + } + return true +} + +// ready ends the snapshot phase: the deltas held while the snapshot was written +// are moved into the channel in order and the writer goroutine starts. The +// channel is empty until then (sendMessage queues into pending) and pending is +// bounded by the channel's capacity, so the move cannot block. A subscription +// closed during its snapshot starts no goroutine at all. +func (s *runSubscription) ready() { + s.mu.Lock() + select { + case <-s.done: + s.mu.Unlock() + return + default: + } + for _, msg := range s.pending { + s.ch <- msg + } + s.pending = nil + s.snapshotting = false + s.mu.Unlock() + go s.run() +} + func (s *runSubscription) send(frame protocol.RunEventFrame) { s.sendMessage(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) } @@ -330,6 +387,23 @@ func (s *runSubscription) sendMessage(msg protocol.Message) { return default: } + s.mu.Lock() + if s.snapshotting { + // The snapshot is still being written on the subscribing connection and + // no writer goroutine is draining yet, so closing here would truncate a + // snapshot in flight and swallow its ack. Hold the delta instead, bounded + // the same as the channel: the policy can only ever affect deltas, since + // the snapshot and the ack are written directly by the subscriber. + if len(s.pending) >= runEventSubscriberBuffer { + s.mu.Unlock() + s.close() + return + } + s.pending = append(s.pending, msg) + s.mu.Unlock() + return + } + s.mu.Unlock() select { case s.ch <- msg: default: @@ -395,9 +469,12 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) { // handleRunEvents dispatches one subscribe/unsubscribe request and reports // whether it was a run-event type (so the caller never falls through to the -// generic unknown-command response). Subscribe registers the watch, sends the -// current state of the watched runs through the subscription, then acks; the -// snapshot therefore precedes every later delta. +// generic unknown-command response). Subscribe registers the watch, writes the +// current state of the watched runs as its snapshot, acks, and only then lets +// deltas flow: the snapshot and the ack go straight out on the subscribing +// connection, so a snapshot larger than the subscription's buffer can no longer +// drop the subscription it is seeding, and the ordering is always snapshot, +// ack, deltas. func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner, clientID string, write func(protocol.Message) error) bool { switch request.Type { case protocol.Subscribe: @@ -414,6 +491,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner // workspace-scoped read the polling clients use, instead of a new query. summaries, err := runs.listRuns() if err != nil { + sub.close() _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) return true } @@ -421,7 +499,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner if !sub.matches(summary.Workspace, summary.ID) { continue } - sub.send(protocol.RunEventFrame{ + frame := protocol.RunEventFrame{ RunID: summary.ID, Workspace: summary.Workspace, PipelineID: summary.PipelineID, @@ -434,9 +512,19 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner CompletedSteps: summary.CompletedSteps, CreatedAt: summary.CreatedAt, FinishedAt: summary.FinishedAt, - }) + } + if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) { + // The connection write failed; the subscription is closed and the + // read loop will stop on its next read error, so there is nothing + // left to reply to. + return true + } } - sub.sendMessage(protocol.Message{Type: protocol.Subscribe, ID: request.ID}) + if err := write(protocol.Message{Type: protocol.Subscribe, ID: request.ID}); err != nil { + sub.close() + return true + } + sub.ready() return true case protocol.Unsubscribe: events.unsubscribeClient(clientID)