Merge branch 'main' into verify-gaps

This commit is contained in:
2026-10-18 02:46:48 +01:00
24 changed files with 1202 additions and 409 deletions
+11 -7
View File
@@ -31,10 +31,14 @@ the single `docs-fold` writer rather than editing them per item: leave a
## Maintenance workflow
Top-level agents use the configured `Pipelines maintenance - plan implement review`
maintenance workflow for code changes. Inspect the retained plan/implement/review
outputs and diffs, execute the checks independently, and iterate on concrete
failures rather than trusting stage summaries. An executing stage performs its own
role directly and never recursively launches a workflow. If the workflow is
unavailable, ask instead of silently bypassing it. Never hardcode a disposable
server URL as a project default; the configured endpoint is authoritative.
Top-level agents advance work through the `pipelines` CLI; **the agent is the brain**
and there is no orchestration script, driver, polling loop or lane machinery, and none
may be written. Choose the pipeline by the area the work is in — `Android`, `TUI`,
`Server`, `Generic`, `Plan and land`, `UI sweep`, `Probe` — each reading the project's
own map (`android/AGENTS.md`, `internal/tui/AGENTS.md`, `internal/server/AGENTS.md`)
for how that technology is built, tested, run and captured. Inspect the retained
plan/implement/review outputs and diffs, execute the checks independently, and iterate
on concrete failures rather than trusting stage summaries. An executing stage performs
its own role directly and never recursively launches a workflow. If a pipeline is
unavailable, ask instead of silently bypassing it. Never hardcode a disposable server
URL as a project default; the configured endpoint is authoritative.
@@ -277,6 +277,16 @@ object ChatProtocol {
else -> false
}
/**
* The stable part id an incremental text/reasoning delta stream addresses,
* mirroring `ChatDeltaPartID` in `internal/protocol/chat.go`. OpenCode numbers
* one message's text and reasoning parts independently from 0, so the kind is
* part of the id: a message's first text delta and its first reasoning delta
* are distinct parts, and answer text can never be appended into the thinking
* block.
*/
fun deltaPartId(messageId: String, kind: String, ordinal: Int): String = "$messageId#$kind#$ordinal"
/**
* The `chat_prompt` frame the composer sends. [files] are the attached files'
* URIs, in the order they were attached, and are omitted when empty (the
@@ -102,10 +102,12 @@ internal fun ChatConversationPane(
// A turn blocked on an approval is not generating, so the running indicator
// (and its ticking clock) stands down while the card is unanswered.
val showRunning = thread.running && !awaiting
// The count of the transcript's items: the messages, the local command lines,
// the running indicator while a turn is in flight and a pending approval card,
// which is the last item the top bar's approval action jumps to.
val itemCount = thread.messages.size + thread.notices.size +
// The count of the transcript's items: its rows (the operator's messages and
// the folded agent replies), the local command lines, the running indicator
// while a turn is in flight and a pending approval card, which is the last
// item the top bar's approval action jumps to.
val rows = transcriptRows(thread.messages)
val itemCount = rows.size + thread.notices.size +
(if (showRunning) 1 else 0) + (if (approval != null) 1 else 0)
// The position the operator left this conversation at, read once: a
// conversation with none of its own opens at its newest message instead (see
@@ -219,6 +221,7 @@ internal fun ChatConversationPane(
)
else -> ChatTranscript(
thread = thread,
rows = rows,
listState = listState,
savedScroll = savedScroll,
itemCount = itemCount,
@@ -180,6 +180,53 @@ internal fun runningToolStamps(
return stamps
}
/**
* The id a text or reasoning delta must be applied to for [kind] within [parts].
* The broker gives the two kinds distinct ids (see `ChatDeltaPartID`), but a peer
* that still shares one id between them must never have answer text appended into
* a reasoning part (or the reverse): when another kind already holds the bare id,
* the delta is moved onto a suffixed id of its own so the two kinds stay in
* separate parts. Once a part of this kind exists under either id, that part is
* the one addressed, so a stream keeps growing the part it started.
*/
internal fun deltaPartId(parts: List<ChatPart>, kind: String, id: String): String {
if (parts.any { it.type == kind && it.id == id }) return id
val suffixed = "$id#$kind"
if (parts.any { it.type == kind && it.id == suffixed }) return suffixed
return if (parts.any { it.id == id }) suffixed else id
}
/**
* Folds one tool frame's part onto the part retained from earlier frames. A tool
* call is reported across several frames that each carry only part of it (the name
* in `input.started`, the arguments in `input.ended`'s text or `called`'s input,
* then `progress`/`success` carrying neither), so a field already learned is never
* blanked by a later frame that omits it. A non-empty incoming field wins;
* otherwise the retained value is kept.
*/
internal fun mergeToolPart(previous: ChatPart?, incoming: ChatPart): ChatPart {
if (previous == null) return incoming
val priorState = previous.state
val incomingState = incoming.state
return incoming.copy(
type = incoming.type.ifBlank { previous.type },
name = incoming.name.ifBlank { previous.name },
executed = incoming.executed || previous.executed,
time = incoming.time ?: previous.time,
state = when {
incomingState == null -> priorState
priorState == null -> incomingState
else -> incomingState.copy(
status = incomingState.status.ifBlank { priorState.status },
input = incomingState.input ?: priorState.input,
content = incomingState.content.ifEmpty { priorState.content },
metadata = incomingState.metadata ?: priorState.metadata,
error = incomingState.error ?: priorState.error,
)
},
)
}
/**
* The status a session row reports while a turn is in flight: it names the tool
* call it is parked on (or how many, when several run at once) so a working chat
@@ -378,6 +425,92 @@ internal fun turnReasoning(parts: List<ChatPart>): ReasoningTurn =
.map { it.text },
)
/**
* One agent reply: everything the agent produced between two of the operator's
* messages, however many model steps that took. OpenCode reports a turn as one
* assistant message per step, so this is the single fold the transcript uses to
* render a reply as one bubble — the turn's thinking block, then its tool-call
* chips, then its answer — instead of one bubble (and one thinking block) per
* step.
*/
internal data class ChatTurn(val key: String, val parts: List<ChatPart>) {
/** The turn's reasoning, folded into the one block the transcript shows first. */
val reasoning: ReasoningTurn get() = turnReasoning(parts)
/** The turn's tool calls, in the order the agent made them. */
val tools: List<ChatPart> get() = parts.filter { it.type == ChatProtocol.PART_TOOL }
/**
* The turn's answer: just its text parts, joined in order. Reasoning feeds the
* folded thinking block and a tool call its chip, so the answer block never
* carries thinking or a command.
*/
val answer: String
get() = parts
.filter {
it.type != ChatProtocol.PART_TOOL &&
(partRendering(it) == PartRendering.MARKDOWN || partRendering(it) == PartRendering.PLAIN)
}
.joinToString("\n\n") { it.text }
/** The turn's own timing, or null when its parts carry none. */
val elapsedMillis: Long? get() = turnElapsedMillis(parts)
/** Whether the turn has anything to draw; an empty turn renders no bubble. */
val visible: Boolean get() = reasoning.count > 0 || tools.isNotEmpty() || answer.isNotBlank()
}
/**
* One item of the transcript: an operator message, or one folded agent reply. The
* key is stable as a reply grows — an agent turn is keyed by its first message id
* — so the list is not rebuilt and an expanded thinking block keeps its state as
* new steps stream in.
*/
internal sealed interface TranscriptRow {
val key: String
data class User(val message: ChatMessage) : TranscriptRow {
override val key: String get() = message.id
}
data class Agent(val turn: ChatTurn) : TranscriptRow {
override val key: String get() = turn.key
}
}
/**
* The transcript's rows for [messages]: each run of consecutive assistant
* messages folds into one [TranscriptRow.Agent], and each of the operator's own
* messages is its own [TranscriptRow.User]. This is the one fold that makes a
* reply a single bubble, so the transcript and its tests cannot disagree about
* where one agent turn ends and the next begins.
*/
internal fun transcriptRows(messages: List<ChatMessage>): List<TranscriptRow> {
val rows = ArrayList<TranscriptRow>(messages.size)
var turnKey: String? = null
var parts = ArrayList<ChatPart>()
fun flush() {
val key = turnKey
if (key != null) {
val turn = ChatTurn(key, parts)
if (turn.visible) rows.add(TranscriptRow.Agent(turn))
}
turnKey = null
parts = ArrayList()
}
for (message in messages) {
if (message.role == ChatProtocol.ROLE_ASSISTANT) {
if (turnKey == null) turnKey = message.id
parts.addAll(message.parts)
} else {
flush()
rows.add(TranscriptRow.User(message))
}
}
flush()
return rows
}
/** The live label of a single-step turn's reasoning block while it streams. */
private const val REASONING_STREAMING_LABEL = "Thinking…"
@@ -938,45 +938,56 @@ class ChatStore(
}
private fun applyDelta(frame: ChatEventFrame) {
val partId = frame.part?.id.orEmpty()
if (partId.isEmpty() || frame.delta.isEmpty()) return
upsertPart(frame, partId) { existing ->
existing?.copy(text = existing.text + frame.delta)
?: ChatPart(type = frame.kind, id = partId, text = frame.delta)
val rawId = frame.part?.id.orEmpty()
if (rawId.isEmpty() || frame.delta.isEmpty()) return
val kind = frame.kind
upsertParts(frame) { parts ->
val id = deltaPartId(parts, kind, rawId)
val index = parts.indexOfFirst { it.id == id }
if (index < 0) {
parts + ChatPart(type = kind, id = id, text = frame.delta)
} else {
parts.toMutableList().also { it[index] = it[index].copy(text = it[index].text + frame.delta) }
}
}
}
private fun applyTool(frame: ChatEventFrame) {
val part = frame.part ?: return
if (part.id.isEmpty()) return
upsertPart(frame, part.id) { part }
val incoming = frame.part ?: return
if (incoming.id.isEmpty()) return
upsertParts(frame) { parts ->
val index = parts.indexOfFirst { it.id == incoming.id && it.type == ChatProtocol.PART_TOOL }
if (index < 0) {
parts + mergeToolPart(null, incoming)
} else {
parts.toMutableList().also { it[index] = mergeToolPart(it[index], incoming) }
}
}
}
/**
* Finds or creates the assistant message [frame] targets and the part [partId]
* inside it, replacing it via [upsert] (the old part when it exists, else
* null), then marks the thread running. The text/reasoning delta and the tool
* upsert share this so the two cannot drift.
* Finds or creates the assistant message [frame] targets and replaces its
* parts with [upsert]'s result, then marks the thread running. The text and
* reasoning delta fold and the tool upsert share this so the two cannot drift.
*/
private fun upsertPart(frame: ChatEventFrame, partId: String, upsert: (ChatPart?) -> ChatPart) {
private fun upsertParts(frame: ChatEventFrame, upsert: (List<ChatPart>) -> List<ChatPart>) {
update(frame.sessionId) { thread ->
val messages = thread.messages.toMutableList()
val index = messages.indexOfLast { it.id == frame.messageId }
val message = if (index >= 0) {
messages[index]
val resolvedIndex = if (index >= 0) {
index
} else {
ChatMessage(
id = frame.messageId,
sessionId = frame.sessionId,
role = ChatProtocol.ROLE_ASSISTANT,
).also { messages.add(it) }
messages.add(
ChatMessage(
id = frame.messageId,
sessionId = frame.sessionId,
role = ChatProtocol.ROLE_ASSISTANT,
),
)
messages.lastIndex
}
val resolvedIndex = if (index >= 0) index else messages.lastIndex
val parts = message.parts.toMutableList()
val partIndex = parts.indexOfFirst { it.id == partId }
val updated = upsert(if (partIndex >= 0) parts[partIndex] else null)
if (partIndex >= 0) parts[partIndex] = updated else parts.add(updated)
messages[resolvedIndex] = message.copy(parts = parts)
val message = messages[resolvedIndex]
messages[resolvedIndex] = message.copy(parts = upsert(message.parts))
val stamped = runningToolStamps(thread.toolStartedAt, messages, now())
setRunning(thread.copy(messages = messages, toolStartedAt = stamped), true)
}
@@ -85,6 +85,7 @@ private const val CHAT_ELAPSED_TICK_MILLIS = 1_000L
@Composable
internal fun ChatTranscript(
thread: ChatThread,
rows: List<TranscriptRow>,
listState: LazyListState,
savedScroll: ChatScrollPosition?,
itemCount: Int,
@@ -177,22 +178,26 @@ internal fun ChatTranscript(
}
Box(Modifier.fillMaxSize()) {
// Only the newest turn can be streaming, so its reasoning wears the live
// label; a settled turn's reasoning is labelled as settled.
val streamingMessageId = thread.messages.lastOrNull()?.id?.takeIf { thread.running }
// label; a settled turn's reasoning is labelled as settled. The turn is
// the transcript's last row only while it is the turn in flight.
val streamingTurnKey = (rows.lastOrNull() as? TranscriptRow.Agent)?.key?.takeIf { thread.running }
LazyColumn(
state = listState,
modifier = Modifier.fillMaxSize(),
contentPadding = PaddingValues(horizontal = CHAT_GUTTER, vertical = CHAT_ITEM_GAP),
verticalArrangement = Arrangement.spacedBy(CHAT_ITEM_GAP, Alignment.Bottom),
) {
items(thread.messages, key = { it.id }) { message ->
MessageItem(
message = message,
reasoning = reasoning,
streaming = message.id == streamingMessageId,
runningSince = thread.runningSince,
onToggleReasoning = onToggleReasoning,
)
items(rows, key = { it.key }) { row ->
when (row) {
is TranscriptRow.User -> UserMessageItem(row.message)
is TranscriptRow.Agent -> AgentTurnItem(
turn = row.turn,
reasoning = reasoning,
streaming = row.key == streamingTurnKey,
runningSince = thread.runningSince,
onToggleReasoning = onToggleReasoning,
)
}
}
// A slash command reports as its own line, after the turns it acted
// between and visually distinct from any message to the agent.
@@ -398,8 +403,45 @@ private fun rememberElapsedNow(runningSince: Long): Long {
}
@Composable
private fun MessageItem(
message: ChatMessage,
private fun UserMessageItem(message: ChatMessage) {
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.End,
) {
Surface(
shape = RoundedCornerShape(CHAT_GUTTER),
color = MaterialTheme.colorScheme.secondaryContainer,
modifier = Modifier.widthIn(max = CHAT_BUBBLE_MAX_WIDTH_DP),
) {
SelectionContainer {
Column(Modifier.padding(CHAT_GUTTER)) {
// A user turn another device sent is attributed to it, so the
// transcript says who sent the message. Our own messages carry
// no label.
if (message.client.isNotBlank()) {
Text(
text = message.client,
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
UserParts(message.parts)
}
}
}
}
}
/**
* One agent reply as a single bubble: the turn's one collapsible thinking block
* first, then its tool-call chips in order, then its answer text. A turn's
* reasoning and answer are never mixed — the fold in [ChatTurn] keeps them
* separate, so the answer is never drawn inside the thinking block — and the
* whole reply is one item however many model steps produced it.
*/
@Composable
private fun AgentTurnItem(
turn: ChatTurn,
reasoning: ReasoningExpansion,
streaming: Boolean,
runningSince: Long,
@@ -414,46 +456,29 @@ private fun MessageItem(
val now = rememberElapsedNow(runningSince)
if (runningSince < now) now - runningSince else 0L
} else {
turnElapsedMillis(message.parts)
turn.elapsedMillis
}
val isUser = message.role == ChatProtocol.ROLE_USER
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = if (isUser) Arrangement.End else Arrangement.Start,
horizontalArrangement = Arrangement.Start,
) {
Surface(
shape = RoundedCornerShape(CHAT_GUTTER),
color = if (isUser) {
MaterialTheme.colorScheme.secondaryContainer
} else {
MaterialTheme.colorScheme.surfaceContainer
},
color = MaterialTheme.colorScheme.surfaceContainer,
modifier = Modifier.widthIn(max = CHAT_BUBBLE_MAX_WIDTH_DP),
) {
SelectionContainer {
Column(Modifier.padding(CHAT_GUTTER)) {
if (isUser) {
// A user turn another device sent is attributed to it, so
// the transcript says who sent the message. Our own
// messages carry no label.
if (message.client.isNotBlank()) {
Text(
text = message.client,
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
UserParts(message.parts)
} else {
AssistantParts(
messageId = message.id,
parts = message.parts,
reasoning = reasoning,
streaming = streaming,
elapsedMillis = elapsedMillis,
onToggleReasoning = onToggleReasoning,
)
}
val turnKey = reasoningTurnKey(turn.key)
ReasoningBlock(
turn = turn.reasoning,
expanded = reasoning.isExpanded(turnKey),
streaming = streaming,
elapsedMillis = elapsedMillis,
onToggle = { onToggleReasoning(turnKey) },
)
if (turn.tools.isNotEmpty()) ToolCalls(turn.tools)
if (turn.answer.isNotBlank()) MarkdownText(turn.answer)
}
}
}
@@ -462,7 +487,7 @@ private fun MessageItem(
/**
* A user turn: its text, then the files that rode with it. The attachments are
* the same [ChatProtocol.PART_FILE] parts [MessageItem] already renders, so the
* the same [ChatProtocol.PART_FILE] parts the user bubble already renders, so the
* sent message shows its file in the transcript through the one display path
* rather than a second attachment list under the bubble.
*/
@@ -478,73 +503,6 @@ private fun UserParts(parts: List<ChatPart>) {
}
}
/**
* An assistant turn's parts in order, with each run of consecutive tool calls
* rendered as one wrapping chip group (mirroring the reference transcript's
* `_ToolCalls`, research §1.3): the calls of a turn sit together on as few lines
* as they fit, instead of each taking a row, so the transcript stays scannable.
*
* The turn's reasoning is folded into exactly one collapsible block, anchored
* where the turn's first reasoning part sits, so a turn that reasons several
* times is one entry rather than a stack of boxes and a streaming turn grows the
* block it has instead of gaining another.
*/
@Composable
private fun AssistantParts(
messageId: String,
parts: List<ChatPart>,
reasoning: ReasoningExpansion,
streaming: Boolean,
elapsedMillis: Long?,
onToggleReasoning: (String) -> Unit,
) {
val turn = turnReasoning(parts)
val turnKey = reasoningTurnKey(messageId)
var index = 0
var reasoningShown = false
while (index < parts.size) {
val part = parts[index]
when {
part.type == ChatProtocol.PART_TOOL -> {
var end = index
while (end < parts.size && parts[end].type == ChatProtocol.PART_TOOL) end++
ToolCalls(parts.subList(index, end))
index = end
}
partRendering(part) == PartRendering.REASONING -> {
if (!reasoningShown) {
reasoningShown = true
ReasoningBlock(
turn = turn,
expanded = reasoning.isExpanded(turnKey),
streaming = streaming,
elapsedMillis = elapsedMillis,
onToggle = { onToggleReasoning(turnKey) },
)
}
index++
}
else -> {
MessagePart(part)
index++
}
}
}
}
@Composable
private fun MessagePart(part: ChatPart) {
when (partRendering(part)) {
PartRendering.MARKDOWN -> MarkdownText(part.text)
// Reasoning is folded into the turn's one block by [AssistantParts]; a
// reasoning part never reaches here, but rendering nothing keeps this
// exhaustive and safe if one ever did.
PartRendering.REASONING -> Unit
PartRendering.PLAIN -> Text(part.text, style = MaterialTheme.typography.bodyMedium)
PartRendering.HIDDEN -> Unit
}
}
/**
* A turn's reasoning: one compact, muted line that names the block — its step
* count when it reasoned more than once, and its elapsed time when the turn's own
@@ -1,6 +1,7 @@
package net.awan.pipelines.ui.chat
import net.awan.pipelines.data.protocol.ChatEventFrame
import net.awan.pipelines.data.protocol.ChatMessage
import net.awan.pipelines.data.protocol.ChatPart
import net.awan.pipelines.data.protocol.ChatProtocol
import net.awan.pipelines.data.protocol.ChatSession
@@ -117,6 +118,77 @@ class ChatReasoningTurnTest {
assertFalse(reasoning.isExpanded(reasoningTurnKey("m2")))
}
@Test
fun aTextDeltaAndAReasoningDeltaInOneMessageNeverShareAPart() {
val store = ChatStore()
store.open(ChatSession(id = "s", title = "s"))
store.history("s", emptyList())
// The broker numbers a message's text and reasoning independently from 0,
// so a peer that still keyed both off the bare ordinal addresses the same
// id for the two kinds: the client must keep them apart anyway, or the
// answer is appended into the thinking block.
store.event(reasoningDelta("s", "m1", "m1#0", "thinking"))
store.event(textDelta("s", "m1", "m1#0", "answer"))
val message = store.value.thread("s")!!.messages.single()
val reasoning = message.parts.single { it.type == ChatProtocol.PART_REASONING }
val answer = message.parts.single { it.type == ChatProtocol.PART_TEXT }
assertEquals("thinking", reasoning.text)
assertEquals("answer", answer.text)
// A later text delta on the same id keeps growing the answer part and
// never touches the reasoning part.
store.event(textDelta("s", "m1", "m1#0", "!"))
val grown = store.value.thread("s")!!.messages.single()
assertEquals("answer!", grown.parts.single { it.type == ChatProtocol.PART_TEXT }.text)
assertEquals("thinking", grown.parts.single { it.type == ChatProtocol.PART_REASONING }.text)
// The broker's own derivation gives the two kinds distinct ids, and the
// client's mirror produces the same ids.
assertEquals("m1#text#0", ChatProtocol.deltaPartId("m1", ChatProtocol.PART_TEXT, 0))
assertEquals("m1#reasoning#0", ChatProtocol.deltaPartId("m1", ChatProtocol.PART_REASONING, 0))
assertFalse(
ChatProtocol.deltaPartId("m1", ChatProtocol.PART_TEXT, 0) ==
ChatProtocol.deltaPartId("m1", ChatProtocol.PART_REASONING, 0),
)
}
@Test
fun agentStepsBetweenTwoUserMessagesFoldIntoOneTurn() {
// The step-by-step reply a live agent produces: several assistant messages,
// then the final one carrying the answer beside its reasoning. They are one
// turn — one bubble, one thinking block, one answer — and the operator's
// next message starts a new one.
val messages = listOf(
ChatMessage(id = "u1", role = ChatProtocol.ROLE_USER, parts = listOf(text("t", "do it"))),
ChatMessage(id = "m0", role = ChatProtocol.ROLE_ASSISTANT, parts = listOf(reasoning("r0", "thought 0"), ChatPart(type = ChatProtocol.PART_TOOL, id = "c0", name = "shell"))),
ChatMessage(id = "m1", role = ChatProtocol.ROLE_ASSISTANT, parts = listOf(reasoning("r1", "thought 1"), ChatPart(type = ChatProtocol.PART_TOOL, id = "c1", name = "read"))),
ChatMessage(id = "m2", role = ChatProtocol.ROLE_ASSISTANT, parts = listOf(reasoning("r2", "thought 2"), text("t2", "the answer"))),
ChatMessage(id = "u2", role = ChatProtocol.ROLE_USER, parts = listOf(text("t", "again"))),
)
val rows = transcriptRows(messages)
assertEquals(listOf("u1", "m0", "u2"), rows.map { it.key })
val turn = (rows[1] as TranscriptRow.Agent).turn
// The turn is keyed by its first message id, so its expansion state and the
// list key survive the later steps.
assertEquals("m0", turn.key)
assertEquals(3, turn.reasoning.count)
assertEquals("thought 0\n\nthought 1\n\nthought 2", turn.reasoning.text)
assertEquals(listOf("c0", "c1"), turn.tools.map { it.id })
assertEquals("the answer", turn.answer)
}
private fun textDelta(sessionId: String, messageId: String, partId: String, delta: String) =
ChatEventFrame(
sessionId = sessionId,
messageId = messageId,
kind = ChatProtocol.PART_TEXT,
delta = delta,
part = ChatPart(type = ChatProtocol.PART_TEXT, id = partId),
)
private fun reasoningDelta(sessionId: String, messageId: String, partId: String, delta: String) =
ChatEventFrame(
sessionId = sessionId,
@@ -8,6 +8,7 @@ import net.awan.pipelines.data.protocol.ChatPart
import net.awan.pipelines.data.protocol.ChatProtocol
import net.awan.pipelines.data.protocol.ChatSession
import net.awan.pipelines.data.protocol.ChatTime
import net.awan.pipelines.data.protocol.ChatToolContent
import net.awan.pipelines.data.protocol.ChatToolState
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
@@ -160,6 +161,50 @@ class ChatRunningWorkTest {
assertEquals(4_000L, store.value.thread("s")!!.toolElapsedMillis(call, clock))
}
@Test
fun aToolCallKeepsItsNameAndInputForItsWholeLifecycle() {
val store = ChatStore()
store.open(ChatSession(id = "s", title = "s"))
store.history("s", emptyList())
// The verified frame sequence: the name arrives only in input.started,
// the arguments in called's input, and progress/success carry neither.
store.event(toolFrame("s", "m1", ChatPart(type = ChatProtocol.PART_TOOL, id = "c1", name = "shell", state = ChatToolState(status = ChatProtocol.TOOL_STREAMING))))
store.event(toolFrame("s", "m1", ChatPart(type = ChatProtocol.PART_TOOL, id = "c1", state = ChatToolState(status = ChatProtocol.TOOL_RUNNING, input = input("command" to "ls")))))
store.event(toolFrame("s", "m1", ChatPart(type = ChatProtocol.PART_TOOL, id = "c1", state = ChatToolState(status = ChatProtocol.TOOL_RUNNING, metadata = JsonObject(mapOf("exit" to JsonPrimitive(0)))))))
store.event(toolFrame("s", "m1", ChatPart(type = ChatProtocol.PART_TOOL, id = "c1", state = ChatToolState(status = ChatProtocol.TOOL_COMPLETED, content = listOf(ChatToolContent(type = "text", text = "ok"))))))
val part = store.value.thread("s")!!.messages.single().parts.single()
// Nothing blanked the name or the input the earlier frames taught it.
assertEquals("shell", toolLabel(part))
assertEquals("ls", toolDetail(part))
assertEquals("ok", toolOutputText(part.state))
assertEquals(ChatProtocol.TOOL_COMPLETED, part.state?.status)
assertEquals(JsonPrimitive(0), part.state?.metadata?.get("exit"))
}
@Test
fun aStreamingArgumentStringIsShownWhileTheCallRuns() {
val store = ChatStore()
store.open(ChatSession(id = "s", title = "s"))
store.event(
toolFrame(
"s", "m1",
ChatPart(
type = ChatProtocol.PART_TOOL,
id = "c1",
name = "edit",
state = ChatToolState(status = ChatProtocol.TOOL_STREAMING, input = JsonPrimitive("{\"filePath\"")),
),
),
)
val call = store.value.thread("s")!!.messages.single().parts.single()
// A partial-JSON argument string is kept and shown verbatim; it has no
// object key to describe the call yet, so the detail stays absent.
assertEquals("{\"filePath\"", prettyToolInput(call.state?.input))
assertNull(toolDetail(call))
}
@Test
fun toolDetailPrefersDescriptionThenCommandAndIgnoresWhatItCannotRead() {
// A description wins over a command when both are present.
@@ -1,5 +1,6 @@
package net.awan.pipelines.ui.chat
import net.awan.pipelines.data.protocol.ChatMessage
import net.awan.pipelines.data.protocol.ChatPart
import net.awan.pipelines.data.protocol.ChatProtocol
import org.junit.Assert.assertEquals
@@ -148,4 +149,42 @@ class ChatTranscriptTest {
// ...and a single open chat leaves nothing to switch to.
assertEquals(emptyList<ChatThread>(), openChatsBeside(listOf(a), "a"))
}
@Test
fun eachConversationalRunOfAgentStepsIsOneRowAndItsTextPartsJoinInOrder() {
// Two user messages with two agent steps between the first pair: the agent
// steps are one turn (one bubble), not one row each, and their text parts
// are one answer joined in order.
val messages = listOf(
ChatMessage(id = "u1", role = ChatProtocol.ROLE_USER, parts = listOf(ChatPart(type = ChatProtocol.PART_TEXT, text = "one"))),
ChatMessage(id = "a1", role = ChatProtocol.ROLE_ASSISTANT, parts = listOf(ChatPart(type = ChatProtocol.PART_TEXT, id = "p1", text = "first"))),
ChatMessage(id = "a2", role = ChatProtocol.ROLE_ASSISTANT, parts = listOf(ChatPart(type = ChatProtocol.PART_TOOL, id = "c1", name = "shell"), ChatPart(type = ChatProtocol.PART_TEXT, id = "p2", text = "second"))),
ChatMessage(id = "u2", role = ChatProtocol.ROLE_USER, parts = listOf(ChatPart(type = ChatProtocol.PART_TEXT, text = "two"))),
ChatMessage(id = "a3", role = ChatProtocol.ROLE_ASSISTANT, parts = listOf(ChatPart(type = ChatProtocol.PART_TEXT, id = "p3", text = "third"))),
)
val rows = transcriptRows(messages)
assertEquals(listOf("u1", "a1", "u2", "a3"), rows.map { it.key })
assertEquals(
listOf("u1", "u2"),
rows.filterIsInstance<TranscriptRow.User>().map { it.key },
)
val first = (rows[1] as TranscriptRow.Agent).turn
assertEquals("first\n\nsecond", first.answer)
assertEquals(listOf("c1"), first.tools.map { it.id })
assertEquals("third", (rows[3] as TranscriptRow.Agent).turn.answer)
}
@Test
fun anAgentMessageWithNothingToDrawAddsNoRow() {
// An assistant message whose parts are all empty (or reasoning without
// text) has nothing to draw, so it folds away instead of leaving a blank
// bubble; a following user message is still its own row.
val rows = transcriptRows(
listOf(
ChatMessage(id = "a1", role = ChatProtocol.ROLE_ASSISTANT, parts = listOf(ChatPart(type = ChatProtocol.PART_TEXT, id = "p1"))),
ChatMessage(id = "u1", role = ChatProtocol.ROLE_USER, parts = listOf(ChatPart(type = ChatProtocol.PART_TEXT, text = "hi"))),
),
)
assertEquals(listOf("u1"), rows.map { it.key })
}
}
+56
View File
@@ -0,0 +1,56 @@
# Work documentation now describes the agent-driven model
## What this item changed
The workflow documentation was rewritten to describe how the system actually works:
the agent (the session) is the brain and advances the queue by hand through the
`pipelines` CLI. There is **no orchestration script, no driver, no polling loop and no
lane machinery, and none may be written.**
- `internal/author/skill/references/work.md` — rewritten end to end. The queue-driver
script (`drivers/workd.sh`), the claim-file ledger with atomic renames, the `PAUSED`
marker and the poll loop are gone. The ledger is now the per-project markdown
`queue.md` (authoritative for intent) plus `notes.md`; run state is read from the
server (`pipelines inspect <run-id> --json`); the agent launches each run itself with
`pipelines run <pipeline> <brief> --json --yes` from the item's worktree and is woken
by a background `pipelines wait`.
- `internal/author/skill/references/queue.md` — the landing gate now runs the command
directly (`git merge-base --is-ancestor <branch> main`, exit 0 = landed) instead of a
`land-check.sh` helper; the waiting section states the agent advances the queue by
hand and there is no driver.
- `internal/author/skill/references/orchestration.md` — the intro and "Running and
watching" state the agent-driven, event-driven model; "Pipeline choice" now names the
current set (`Android`, `TUI`, `Server`, `Generic`, `Plan and land`, `UI sweep`,
`Probe`), chosen by area, and records the retired names.
- `internal/author/skill/references/debugging.md` — "Duplicated work after a driver
restart" became "…after a session restart", resuming from the queue file rather than a
cursor file.
- `internal/author/skill/SKILL.md` — the routing table row no longer says "queue driver".
- `AGENTS.md` — the maintenance workflow section names the current pipeline set instead
of the retired `Pipelines maintenance - plan implement review`.
- `internal/tui/AGENTS.md` — the "do not restart … the lanes …" line no longer names
lanes.
`delivery.md`, `operations.md` and `verification.md` were swept and needed no change:
they contain no driver, poll loop, cursor file or lane guidance. Genuine device tooling
(`android/scripts/capture_png.sh`) is left alone.
## Shared documents this change makes stale
No README or topic doc still teaches the scripted driver model. One topic doc needs a
wording alignment for the event-driven model; the `docs-fold` writer should apply it
when it next folds notes:
- `docs/cli.md` — its "Scripted runs" section frames runs as scripted automation, and
its `--wait` paragraph is the only follow-a-run guidance. It should state that the
agent (not a script) advances work, and that `pipelines wait <run-id> [run-id ...]`
run as a **background task** is the event-driven wait. The `wait` command's own flag
text is already recorded separately in `docs/notes/wait-command.md`.
## Do not change
- The dated evidence documents (`docs/pipeline-performance-review.md`,
`docs/chat-verification.md`, `docs/app-integration-verification.md`) describe the
scripted era as it was and stay as they are.
- `internal/author/install.go`'s managed file inventory needs no change: no packaged
reference file was added or removed.
+79
View File
@@ -0,0 +1,79 @@
# Chat transcript: one agent reply is one bubble
## What this item changed
`internal/` and `android/`, the brokered chat transcript. Three defects measured
against the operator's own OpenCode (127.0.0.1:49374):
- **Server, delta part ids.** `internal/protocol/chat.go` adds
`ChatDeltaPartID(messageID, kind, ordinal)` (mirrored as
`ChatProtocol.deltaPartId` in `android/.../data/protocol/Chat.kt`), and
`internal/opencode/chat.go`'s `NormalizeChatEvent` keys a text or reasoning
delta on it. OpenCode numbers one message's text and reasoning parts
independently from 0, so the old `"%s#%d"` gave a message's first text delta and
its first reasoning part the same id — the client appended the answer into the
thinking block and the server's retained state re-collided on resume.
- **Server, tool fields.** `NormalizeChatEvent` now takes a tool call's arguments
from `session.tool.input.ended`'s `text` as well as `session.tool.called`'s
`input` object (`toolInput`), and `internal/server/chat_hub.go`'s new
`mergeChatToolPart` folds each tool frame onto the retained part so a later
`progress`/`success` frame that omits the name or the input never blanks it.
- **Android, transcript.** `ui/chat/ChatFormat.kt` adds the pure `ChatTurn` /
`TranscriptRow` / `transcriptRows` fold (each run of assistant messages between
two of the operator's messages is one agent turn), the kind-aware
`deltaPartId` net and `mergeToolPart`; `ChatStore` folds deltas and tool frames
through them; `ChatTranscript` renders one bubble per turn — one thinking block
over all the turn's reasoning, then its tool chips, then one answer block of
its text parts — replacing `AssistantParts`/`MessagePart` with
`AgentTurnItem`/`UserMessageItem`.
Tests: `internal/opencode/chat_v2_test.go` and `internal/server/chat_test.go` add
the id-collision and tool name+input retention cases; `ChatReasoningTurnTest`,
`ChatTranscriptTest` and `ChatRunningWorkTest` cover the turn fold, the
text/reasoning collision and the tool lifecycle on the client.
## Shared documents this change makes stale
`android/README.md` describes the surfaces this item changed. The note is written
here rather than edited in place; the `docs-fold` writer should update it.
- `android/README.md:6021` — the **Reasoning collapsed by default** bullet reads
"`ReasoningBlock` is now one compact, muted line per part". It should say one
block per *agent turn*: the transcript folds every consecutive assistant message
between two of the operator's messages into one `ChatTurn`, so all the turn's
reasoning is one collapsible block (labelled with its step count when it
reasoned more than once) and the reply is one bubble, not one per model step.
- `android/README.md:4669` — "`ChatScreen.MessagePart` now calls
`MarkdownText(part.text)` for `PART_TEXT`; the assistant message stays one
`SelectionContainer`". `MessagePart` no longer exists; the answer is
`MarkdownText(turn.answer)` inside `AgentTurnItem`, where `turn.answer` is the
turn's text parts joined in order. The one-`SelectionContainer`-per-bubble fact
is still true.
- `android/README.md:6040` — the test paragraph's "`ChatTranscriptTest`, 7 cases"
is now 9 (the fold tests); `ChatReasoningTurnTest` and `ChatRunningWorkTest`
gained the collision and tool-lifecycle cases.
- `android/README.md:4786` — the **Grouped chips** bullet says "a run of
consecutive `tool` parts in one assistant turn is now one `FlowRow`". The chips
still group in one `FlowRow`, but the calls shown are now the whole folded
turn's (`ChatTurn.tools`), across every step of the reply, not one assistant
message's parts.
Dates, counts and capture names in those dated records are past evidence and stay
as they are; only the behaviour statements above are now stale.
## Not stale
- `docs/cli.md`'s and `internal/author/skill/references/operations.md`'s brokered
chat sections describe the `chat_*` surface and its retained
per-session/per-message state and resync; that remains true. Neither documents
the delta part-id derivation or the tool-field retention, so neither gains a
wrong statement — the derivation now lives in the `internal/protocol/chat.go`
comment. The docs-fold writer may add one sentence to the reference's
retained-state paragraph if it documents part ids there, but nothing there is
now false.
- No packaged OpenCode skill reference documents the app's chat transcript, and no
packaged template or `internal/author/` managed file names it.
- `android/AGENTS.md` and `internal/server/AGENTS.md` list the files this item
touched (`ui/chat/`, `internal/server/chat_*.go`, `internal/protocol/`,
`internal/opencode/chat.go`); no file moved and no build/test step changed, so
both maps stay correct.
+1 -1
View File
@@ -59,7 +59,7 @@ and `references/format.md` (in parallel when possible); for notifications also r
| Task | Packaged reference |
| --- | --- |
| Briefs, work items, pipeline choice, running and watching | `references/orchestration.md` |
| Across projects: work ledger, start/pause/resume/status, queue driver | `references/work.md` |
| Across projects: work ledger, start/pause/resume/status, the queue | `references/work.md` |
| The work queue, parallel items and landing | `references/queue.md` |
| Diagnose a wedged, blocked or failed run | `references/debugging.md` |
| Publish artefacts, LAN URLs, install boundaries | `references/delivery.md` |
+11 -8
View File
@@ -130,13 +130,16 @@ Each playbook is *symptom → how to confirm → what to do*.
approval. Put the capture helper in the repository so every UI run reuses one path.
See the example in `references/delivery.md`.
## Duplicated work after a driver restart
## Duplicated work after a session restart
- **Symptom** Items already completed run again after a queue driver
restarts: the same brief is re-invoked, side effects repeat, or a commit is made
- **Symptom** Items already completed run again after the session that was advancing the
queue restarts: the same brief is re-invoked, side effects repeat, or a commit is made
twice.
- **Confirm** Compare the queue's items against their recorded state. A list alone
has no resume point.
- **Do** Record each item's state in the queue itself (`references/queue.md`) and resume from it. Record an item
only once it has landed, and skip recorded items on startup. Never relaunch a
duplicate of a run that is still active — reattach with its request key instead.
- **Confirm** Compare the queue's items against their recorded state and the server's run
history (`inspect RUN_ID --json`). A list alone has no resume point.
- **Do** Record each item's state in the queue itself (`references/queue.md`) and resume
from it: read the queue top to bottom, skip items already marked done, and settle the
ones marked running by inspecting their run id. Record an item only once it has landed.
Never relaunch a duplicate of a run that is still active — reattach with its request key
instead. Do not build a driver or cursor file to remember position; the queue file is
the resume point.
@@ -5,6 +5,11 @@ briefs and a set of managed runs, not the change itself. The implementer stage m
the change; the reviewer stages check it; the committer stage commits it. Route all
project work through Pipelines runs.
The agent advances this **by hand through the `pipelines` CLI**: it launches each run,
is woken when it finishes, reconciles against the server, and starts the next item.
There is no orchestration script, driver, polling loop or lane machinery, and none may
be written.
## Stance
- **Reconnoitre first.** Read enough of the repository, its notes and recent history
@@ -141,6 +146,24 @@ pipelines run "Docs fold"
## Pipeline choice
Choose by the **area the work is in**; each area pipeline reads the project's own map
for its build, test, run and capture commands:
- **`Android`** — `android/`.
- **`TUI`** — `internal/tui/`.
- **`Server`** — `internal/server/`, `internal/stage/`, `internal/pipeline/` and
`internal/protocol/`.
- **`Generic`** — only when a change genuinely spans areas.
- **`Plan and land`** — when the change needs a plan before implementation
(contract-level work).
- **`UI sweep`** — a full-screen review pass.
- **`Probe`** — a harness smoke check.
The retired names (`Pipelines quick`/`maintenance`/`critical`, `Focused tweak`, `Quick
tweak`, `Plan, implement, review`, `Harness probe`) stay in the registry because
definitions are immutable, but nothing should start them. If a pipeline is unavailable,
ask instead of silently bypassing it.
- Use a pipeline that keeps **implementer + reviewer + ui_reviewer + committer** for
anything that matters. The planner is the only stage a leaner variant may drop.
- **Never remove a review stage to go faster.** The reviewer stages are the
@@ -162,9 +185,10 @@ pipelines run "Docs fold"
- A run cannot approve its own work. When an item genuinely needs an operator gate
before it continues or commits, insert a `type=human` decision stage
(`references/format.md`); never invent a model stage that pretends to approve.
- Choose per item: a logic, server or format change usually wants the planner
(a plan/implement/review pipeline); a small, fully understood tweak may use the
leaner implementer-first variant. Both keep all review stages.
- Pick the pipeline whose area matches the change (above); `Generic` only when a change
genuinely spans areas. A logic, server or format change usually wants the planner
(`Plan and land`); a small, fully understood tweak may use the leaner area pipeline.
Both keep all review stages.
## Running and watching
@@ -179,9 +203,13 @@ pipelines run "Docs fold"
and are recorded on the run; `retry` reuses them. See `references/harnesses.md`
for how a stage's model and its repair path both use the effective model.
- Watch to a **terminal state** (`succeeded`, `failed`, `canceled`, `interrupted`).
Poll `CLI_PREFIX inspect RUN_ID --json` (or use `--wait`) until terminal. A reached
client wait limit is not a run failure: the run stays active; keep waiting or
reattach with the original request key, never start a duplicate run.
Progress is **event-driven, never polled**: `CLI_PREFIX wait RUN_ID […]` blocks until
the runs are terminal and is run as a **background task**, so the session is woken
exactly when the batch finishes; `CLI_PREFIX inspect RUN_ID --json` then reads the
retained outcome. A reached client wait limit is not a run failure: the run stays
active; keep waiting or reattach with the original request key, never start a
duplicate run. Do not build a polling loop, a cursor file or a driver script to
answer "is it done yet".
- Read the **outcomes**, not just the status: the option each stage returned, the
evidence it recorded, and any run or step diagnostic. Interpret diagnostics by
category (`references/debugging.md`).
+6 -2
View File
@@ -49,8 +49,9 @@ Serialize only when:
## The landing gate, and what must never happen
- After every item, and **before any cleanup or before starting the next item**, run
`land-check.sh <branch>`. It must report the branch as an **ancestor of main**.
- After every item, and **before any cleanup or before starting the next item**, verify
the branch is an **ancestor of main** directly, with no helper script:
`git merge-base --is-ancestor <branch> main` (exit 0 = landed).
- If it refuses: do not remove the worktree, do not touch the branch, do not start the next item.
Report the landing failure and the commits that are not in main.
- **Never delete a branch.** Deleting a branch whose commits are not yet in main leaves them
@@ -60,6 +61,9 @@ Serialize only when:
## Waiting without a scripting harness
- The agent advances the queue **by hand**: read it, create the item's worktree, and launch
`pipelines run <pipeline> <brief> --json --yes` from that worktree. Nothing else advances it —
there is no driver, and none may be written.
- `pipelines wait <run-id> […​]` blocks until the runs finish and exits; run it as a **background
task** so the session is woken exactly when the batch is done. A run that failed is still a
finished run: it exits 0 and says what happened.
+87 -207
View File
@@ -1,256 +1,136 @@
# Project work ledger and sessions
Work in a project is not a chat. It is a durable ledger: what is queued, what is
running, what has landed, and what is blocked. A session — this conversation — starts,
pauses, resumes and reports that ledger. Nothing important lives only in the
conversation: running the skill again, in any project on this machine, reads the
ledger and continues. This is the multi-project lifecycle on top of
running, what has landed, and what is blocked. A session — this conversation — reads
the ledger, launches the runs, is woken when they finish, and reports.
**The agent is the brain.** It advances the queue by hand through the `pipelines`
CLI: it reads the queue, picks the item, launches the run(s) itself, is woken when they
finish, reconciles against the server, and starts the next item. There is **no
orchestration script, no driver, no polling loop and no lane machinery, and none may be
written.** Where a check is needed, run the command directly.
State lives in exactly three places:
- the **server** is authoritative for run state — `pipelines inspect <run-id> --json`;
- the **queue file** is authoritative for intent — what is queued, in what order, and
what is finished;
- the **conversation** holds this session's working memory.
Never keep progress in a script's own files. A session that dies and comes back reads
the queue file and continues. This is the multi-project lifecycle on top of
`references/orchestration.md` (work items, briefs, pipeline choice) and
`references/queue.md` (parallel items).
`references/queue.md` (parallel items, one worktree per item, landing).
## Where the ledger lives
Keep state **outside repositories**, under one work root, so it never collides with a
Keep state **outside repositories**, under one work root, so it never collides with an
item's worktree or a commit:
${XDG_STATE_HOME:-$HOME/.local/state}/pipelines/work/
registry.json every project this machine knows
registry.json every project this machine knows
projects/<slug>/
state.json project, root, endpoint, pipeline, workspace
notes.md recon, decisions, open questions, blockers
queue/<NNNN-slug>.brief pending briefs, read in filename order
active/<id>.brief, <id>.claim claimed: the brief plus its run id and key
done/<id> landed: run id, status, finished
failed/<id> and <id>.brief failed twice: run id, status, diagnostic
log/driver.log
drivers/workd.sh this project's queue driver
PAUSED marker: stop dispatching new items
queue.md the project's markdown queue: items, briefs, state
notes.md recon, decisions, open questions, blockers
- `<slug>` is `<basename>-<8 hex of sha256(absolute project root)>`. **Resolve a
worktree to its main repository** so every item shares the project's one ledger.
worktree to its main repository** so every item shares the project's one queue.
- `PIPELINES_WORK_DIR` overrides the work root. `registry.json` maps projects to
roots; add a project the first time it is worked on and keep it current.
- `state.json` holds `project`, `root`, `endpoint` (the CLI's `--url`), `pipeline`
(the default pipeline) and `workspace` (the default run directory).
- The ledger is the operator's record: keep `notes.md` truthful about decisions taken
and left open, never narrate it as a diary.
- `queue.md` is the resume point (`references/queue.md`): read it top to bottom and
take the first unfinished item. `notes.md` is the operator's record: keep it truthful
about decisions taken and left open, never narrate it as a diary.
- The CLI's configured endpoint and the area map's default pipeline come from the CLI
and the project's own map, not from a script. Do not write a per-project state file
for a script to read.
## Operations
Map one operator sentence to one operation. Do the operation, then answer with the
status block — nothing else.
status block — nothing else. **You launch the runs yourself**; nothing else does it.
| Operator says | Do |
| --- | --- |
| start / work on X | Reconnoitre, register the project, write small briefs into `queue/`, then run the driver. |
| status / how is it going | Print the status block. |
| queue "…" / add … | Reconnoitre, split into small items, append briefs; the driver picks them up. |
| pause / stop for now | Write `PAUSED`. The item in flight finishes; no new one starts. |
| resume / continue | Reconcile, remove `PAUSED`, run the driver. |
| drop item N | Move its brief to `failed/` with a reason. Never silently delete work. |
| done / stop everything | Pause and report what landed and what remains. |
| start / work on X | Reconnoitre, register the project, write the queue file, then launch the first item's run yourself. |
| status / how is it going | Read the server and the queue, then print the status block. |
| queue "…" / add … | Reconnoitre, split into small items, append them to the queue file, launch what is ready. |
| pause / stop for now | Launch nothing new. The runs already in flight finish; reconcile them when they do. |
| resume / continue | Reconcile against the server, then launch the next item. |
| drop item N | Mark its item failed in the queue file with a reason. Never silently delete work. |
| done / stop everything | Stop launching, wait for what is in flight, and report what landed and what remains. |
**Reconcile before acting, always.** The server is authoritative for run state; the
ledger is authoritative for intent. On every invocation: read `registry.json` and the
project's ledger; for each `active/` claim, `CLI_PREFIX inspect <run id> --json` and
settle it (running → keep waiting; succeeded → `done/`; failed/interrupted →
`failed/`); only then do the requested operation. Never trust a claim file for a
status, and never start a second run for an item whose request key is already
admitted — that key is what makes a redispatch attach instead of duplicate.
queue file is authoritative for intent. On every invocation: read the project's
`queue.md`; for each item marked running, `pipelines inspect <run-id> --json` and settle
it (running → keep waiting; succeeded → verify it landed and mark it done;
failed/interrupted → mark it failed with the diagnostic). Only then do the requested
operation. Never trust your memory of a run's status, and never start a second run for
an item whose request key is already admitted — reuse the original `--request-key` so a
redispatch attaches instead of duplicating.
Status block (these lines, no prose around them):
pipelines — active
queue: 4 pending · 1 running (run 185 · reviewer · 6m) · 12 done · 2 failed
next: 0005-graph-arrows on "Focused tweak - implement review commit"
next: 0005-graph-arrows on "TUI"
blocked: none
## The brief, and per-item directives
Write every brief exactly as `references/orchestration.md` describes. A brief file in
`queue/` may open with up to three directive lines, stripped before the prompt is sent:
Write every brief exactly as `references/orchestration.md` describes. An item in
`queue.md` may open its brief with up to three directive lines, stripped before the
prompt is sent:
# pipeline: Focused tweak - implement review commit
# pipeline: TUI
# workspace: /path/to/item-worktree
# device: emulator-5556
Goal: …
`pipeline` overrides the default for that item; `workspace` routes an item to its own
checkout (a driver only claims items for its own workspace); `device` is for the
brief's own text. Number briefs `0001-…`, `0002-…` so order is explicit.
`pipeline` overrides the default for that item; `workspace` names the item's own
worktree (create it with `git worktree add -b <item> <path> main`, `references/queue.md`);
`device` is for the brief's own text. Number items `0001-…`, `0002-…` so order is
explicit.
## The driver
## Advancing the queue by hand
A queue is advanced by a small driver, so work continues while the operator sleeps.
Write the driver once per project to `drivers/workd.sh` and launch it; it is
restart-safe (it resumes from `done/`, reattaches to `active/` runs by id, and never
starts a duplicate), and it stops on the first item that fails twice. Run **one
item at a time per worktree**; several items in their own worktrees advance in
parallel because an item is claimed by an atomic rename.
There is no driver. You advance the queue yourself, through the CLI:
```sh
#!/bin/sh
# workd.sh - advance one project's Pipelines work ledger.
#
# sh workd.sh [PROJECT_ROOT] [WORKSPACE]
#
# PROJECT_ROOT identifies the ledger (default: the current directory).
# WORKSPACE is the checkout this driver may run in (default: the ledger's
# workspace). One item per worktree lets several items advance against one shared
# queue: an item is claimed by an atomic rename, so it runs exactly once.
#
# Restart-safe. It resumes from done/, reattaches to active/ runs by id and
# never starts a duplicate. PIPELINES_WORK_POLL sets the poll gap in seconds
# (default 20); PIPELINES_CLI uses another client binary.
set -u
1. **Read the queue top to bottom** and take the first unfinished item.
2. **Choose the pipeline by area** (`references/orchestration.md`): `Android`, `TUI`,
`Server`, `Generic`, `Plan and land`, `UI sweep`, `Probe`. Each area pipeline reads
the project's own map for its build, test, run and capture commands.
3. **Create the item's worktree and branch**: `git worktree add -b <item> <path> main`.
One worktree per item is what lets independent items run at once without the
workspace guard becoming a bottleneck (`references/queue.md`).
4. **Launch the run from that worktree**:
`pipelines run <pipeline> <brief> --json --yes`. Keep the returned run id.
5. **Be woken when it finishes — do not poll.** `pipelines wait <run-id> […]` blocks
until the runs are terminal and is run as a **background task**, so the session is
woken exactly when the batch finishes — or arm a timer and check on the tick. No
polling loop, no cursor file, no driver script, ever.
6. **Reconcile.** `pipelines inspect <run-id> --json`; read the outcomes and evidence;
verify the branch landed with `git merge-base --is-ancestor <branch> main` (exit 0 =
landed); mark the item in `queue.md`; then start the next item — or several
independent items at once.
CLI=${PIPELINES_CLI:-pipelines}
POLL=${PIPELINES_WORK_POLL:-20}
ROOT=$(cd "${1:-$(pwd)}" && pwd)
WORK_ROOT=${PIPELINES_WORK_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/pipelines/work}
SLUG=$(basename "$ROOT")-$(printf '%s' "$ROOT" | shasum -a 256 | cut -c1-8)
D=$WORK_ROOT/projects/$SLUG
LOG=$D/log/driver.log
mkdir -p "$D/queue" "$D/active" "$D/done" "$D/failed" "$D/log" 2>/dev/null
**Parallelism is your judgement, bounded by real conflicts only**: two items editing
the same files at once, the one capture device, or the provider's spend cap. Launch as
many independent runs as the machine can carry without overload; serialise only for a
real conflict and say which limit is binding. Never present serialisation as the
operator's decision.
say() { printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" | tee -a "$LOG"; }
jfield() { python3 -c 'import json,sys
try: d=json.load(open(sys.argv[1]))
except Exception: d={}
print(d.get(sys.argv[2]) or "")' "$1" "$2"; }
dstatus() { "$CLI" inspect "$1" --json 2>/dev/null | python3 -c 'import json,sys
try: d=json.load(sys.stdin)
except Exception: print("unknown"); raise SystemExit
print(d.get("status") or "unknown")'; }
item_field() { sed -n "s/^# *$2 *: *//p" "$1" 2>/dev/null | head -1; } # from a brief
claim_field() { sed -n "s/^$2=//p" "$1" 2>/dev/null | head -1; } # from a claim file
prompt_of() { python3 -c '
import re,sys
lines=open(sys.argv[1],encoding="utf-8",errors="replace").read().splitlines()
while lines and re.match(r"^#\s*(pipeline|workspace|device)\s*:", lines[0]):
lines.pop(0)
while lines and lines[0].strip()=="":
lines.pop(0)
print("\n".join(lines))' "$1"; }
[ -f "$D/state.json" ] || { echo "no ledger: $D/state.json" >&2; exit 2; }
PIPELINE=$(jfield "$D/state.json" pipeline)
ENDPOINT=$(jfield "$D/state.json" endpoint)
WS=${2:-$(jfield "$D/state.json" workspace)}
[ -n "$WS" ] || WS=$ROOT
[ -n "$PIPELINE" ] || { echo "state.json has no pipeline" >&2; exit 2; }
wait_terminal() { # $1 run id -> terminal status (or "unknown" if unreadable)
bad=0
while :; do
st=$(dstatus "$1")
case "$st" in
succeeded|failed|canceled|interrupted) echo "$st"; return 0;;
unknown) bad=$((bad+1)); [ "$bad" -ge 5 ] && { echo "unknown"; return 0; };;
*) bad=0;;
esac
sleep "$POLL"
done
}
dispatch() { # $1 brief file $2 pipeline $3 key -> run id (empty on failure)
if [ -n "$ENDPOINT" ]; then
out=$("$CLI" run "$2" "$(prompt_of "$1")" --json --yes --workspace "$WS" --request-key "$3" --url "$ENDPOINT" 2>&1)
else
out=$("$CLI" run "$2" "$(prompt_of "$1")" --json --yes --workspace "$WS" --request-key "$3" 2>&1)
fi
printf '%s' "$out" | python3 -c 'import json,sys
try: d=json.load(sys.stdin)
except Exception: print("")
else: print((d.get("run_detail") or {}).get("id") or "")'
}
record() { # $1 id $2 status $3 run id
where=done; [ "$2" = succeeded ] || where=failed
printf 'run_id=%s\nstatus=%s\nfinished=%s\n' "$3" "$2" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "$D/$where/$1"
rm -f "$D/active/$1.brief" "$D/active/$1.claim"
}
run_item() { # $1 id $2 brief $3 pipeline $4 key -> 0 succeeded / 1 failed
rid=$(dispatch "$2" "$3" "$4")
if [ -z "$rid" ]; then printf 'status=dispatch-failed\n' > "$D/active/$1.claim"; say "dispatch failed: $1"; return 1; fi
printf 'run_id=%s\n' "$rid" > "$D/active/$1.claim"
st=$(wait_terminal "$rid")
if [ "$st" = succeeded ]; then record "$1" succeeded "$rid"; say "done $1 (run $rid)"; return 0; fi
printf 'run_id=%s\nstatus=%s\n' "$rid" "$st" > "$D/active/$1.claim"
say "failed $1 (run $rid, $st)"
return 1
}
reconcile() { # settle anything the previous process left claimed
for a in "$D"/active/*; do
[ -e "$a" ] || continue
id=$(basename "$a"); id=${id%.brief}; id=${id%.claim}
rid=$(claim_field "$D/active/$id.claim" run_id)
if [ -z "$rid" ]; then
say "requeueing undispatched $id"
[ -e "$D/active/$id.brief" ] && mv "$D/active/$id.brief" "$D/queue/$id.brief"
rm -f "$D/active/$id.claim"
continue
fi
st=$(wait_terminal "$rid")
record "$id" "$st" "$rid"
say "reconciled $id as $st (run $rid)"
done
}
reconcile
while :; do
if [ -e "$D/PAUSED" ]; then sleep "$POLL"; continue; fi
next=""
for q in $(ls -1 "$D"/queue/*.brief 2>/dev/null | sort); do
id=$(basename "$q" .brief)
[ -e "$D/done/$id" ] && continue
[ -e "$D/failed/$id" ] && continue
iws=$(item_field "$q" workspace)
[ -n "$iws" ] && [ "$iws" != "$WS" ] && continue
next=$q; break
done
if [ -z "$next" ]; then say "idle (no claimable items for $WS)"; exit 0; fi
id=$(basename "$next" .brief)
mv "$next" "$D/active/$id.brief" 2>/dev/null || continue
ipipe=$(item_field "$D/active/$id.brief" pipeline)
[ -n "$ipipe" ] || ipipe=$PIPELINE
key="work-$SLUG-$id"
if run_item "$id" "$D/active/$id.brief" "$ipipe" "$key"; then continue; fi
if [ "$(claim_field "$D/active/$id.claim" status)" = "dispatch-failed" ]; then
say "STOP: could not dispatch $id"
mv "$D/active/$id.brief" "$D/failed/$id.brief" 2>/dev/null
exit 1
fi
say "retrying $id once"
if run_item "$id" "$D/active/$id.brief" "$ipipe" "$key-r1"; then continue; fi
rid=$(claim_field "$D/active/$id.claim" run_id)
st=$(claim_field "$D/active/$id.claim" status)
printf 'run_id=%s\nstatus=%s\nattempts=2\n' "$rid" "${st:-failed}" > "$D/failed/$id"
mv "$D/active/$id.brief" "$D/failed/$id.brief" 2>/dev/null
rm -f "$D/active/$id.claim"
say "STOP: $id failed twice; recorded under failed/"
exit 1
done
```
Launch it detached and keep its log:
nohup sh <work root>/projects/<slug>/drivers/workd.sh <project root> >/dev/null 2>&1 &
The driver only *advances* a prepared queue; it never writes briefs. Preparing a good
queue and settling a failure is the agent's work.
**Landing stays the pipeline's job**: the committer stage merges and lands its own
work. There is no merger, no resolver and nothing to hand to a script. You verify the
landing gate (`references/queue.md`); you do not perform the merge.
## Sleeping and waking
- Before the operator sleeps: reconcile, make sure the queue is full enough to cover
the night, and confirm the driver is running. A drained queue is fine; it idles.
- Configure a **failure notification** (`references/notifications.md`) so a stopped
driver is not silent. On an overnight failure the driver stops after one retry; that
is by design — do not add retry loops to make it look busy.
- Before the operator sleeps: reconcile, make sure the queue has enough items to cover
the night, and launch what is ready with a background `pipelines wait`.
- Configure a **failure notification** (`references/notifications.md`) so a failed run
is not silent. A run that fails is recorded and the next item starts; do not add
retry loops to make it look busy.
- On waking, reconcile first, then answer with the status block: what landed, what
failed and why, what is next.
@@ -262,7 +142,7 @@ The operator wants the result, not the shift report.
two sentences; never answer a narrow question with a sectioned report.
- Answer the question that was asked first. Put any extra fact that changes a decision in
a single clause, not in a new section.
- Answer with the status block for status; do not narrate polling or re-print the
- Answer with the status block for status; do not narrate waiting or re-print the
queue when nothing changed.
- Report a run by its outcome and evidence (`references/verification.md`), not the
steps you took to watch it.
+29 -3
View File
@@ -388,6 +388,7 @@ type chatEvent struct {
Delta string `json:"delta"`
ID string `json:"id"`
Name string `json:"name"`
Text string `json:"text"`
Input json.RawMessage `json:"input"`
Content []protocol.ChatToolContent `json:"content"`
Metadata map[string]any `json:"metadata"`
@@ -451,11 +452,11 @@ func NormalizeChatEvent(payload []byte, sessionID string, lastMessageID *string)
case outer.Type == "session.text.delta":
frame.Kind = protocol.ChatEventText
frame.Delta = data.Delta
frame.Part = &protocol.ChatPart{Type: protocol.ChatPartText, ID: fmt.Sprintf("%s#%d", messageID, data.Ordinal)}
frame.Part = &protocol.ChatPart{Type: protocol.ChatPartText, ID: protocol.ChatDeltaPartID(messageID, protocol.ChatPartText, data.Ordinal)}
case outer.Type == "session.reasoning.delta":
frame.Kind = protocol.ChatEventReasoning
frame.Delta = data.Delta
frame.Part = &protocol.ChatPart{Type: protocol.ChatPartReasoning, ID: fmt.Sprintf("%s#%d", messageID, data.Ordinal)}
frame.Part = &protocol.ChatPart{Type: protocol.ChatPartReasoning, ID: protocol.ChatDeltaPartID(messageID, protocol.ChatPartReasoning, data.Ordinal)}
case outer.Type == "session.step.started" || outer.Type == "session.step.streamed" || outer.Type == "session.step.ended":
frame.Kind = protocol.ChatEventStep
frame.Step = &protocol.ChatStep{Status: strings.TrimPrefix(outer.Type, "session.step."), Index: stepIndex(data)}
@@ -465,7 +466,7 @@ func NormalizeChatEvent(payload []byte, sessionID string, lastMessageID *string)
return protocol.ChatEventFrame{}, false, nil
}
frame.Kind = protocol.ChatEventTool
state := &protocol.ChatToolState{Status: status, Input: data.Input, Content: data.Content, Metadata: data.Metadata, Error: data.Error}
state := &protocol.ChatToolState{Status: status, Input: toolInput(data), Content: data.Content, Metadata: data.Metadata, Error: data.Error}
frame.Part = &protocol.ChatPart{Type: protocol.ChatPartTool, ID: data.ID, Name: data.Name, State: state}
case outer.Type == "session.usage.updated":
frame.Kind = protocol.ChatEventUsage
@@ -480,6 +481,31 @@ func NormalizeChatEvent(payload []byte, sessionID string, lastMessageID *string)
return frame, true, nil
}
// toolInput picks a tool call's arguments from whichever frame carries them:
// session.tool.called carries the parsed `input` object, session.tool.input.ended
// carries the same arguments as a `text` JSON string, and progress/success carry
// neither. Text that already parses as JSON is kept as that value (so an object
// pretty-prints); a partial string is carried as a JSON string so the frame stays
// valid and the client can show it while streaming. It returns nil when the frame
// carries no arguments, so the consumer retains what it already learned.
func toolInput(data chatEvent) json.RawMessage {
if len(data.Input) > 0 && string(data.Input) != "null" {
return data.Input
}
text := strings.TrimSpace(data.Text)
if text == "" {
return nil
}
if json.Valid([]byte(text)) {
return json.RawMessage(text)
}
encoded, err := json.Marshal(text)
if err != nil {
return nil
}
return encoded
}
// toolStatus maps the verified tool event types onto the ToolState lifecycle.
func toolStatus(eventType string) (string, bool) {
switch eventType {
+145
View File
@@ -2,6 +2,8 @@ package opencode
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"path/filepath"
@@ -121,6 +123,149 @@ func TestChatStreamRealV2Shapes(t *testing.T) {
}
}
// TestChatDeltaPartIDsNeverCollide pins the broker's part-id namespace: OpenCode
// numbers one message's text and reasoning parts independently from 0, so a
// message's first text delta and its first reasoning delta share an ordinal. The
// two must still land on distinct part ids — otherwise the client appends the
// answer into the thinking block and the server's retained state re-collides on a
// resume. The id is the shared ChatDeltaPartID derivation, so it is deterministic
// across a reconnect.
func TestChatDeltaPartIDsNeverCollide(t *testing.T) {
payloads := map[string]string{
"text": `{"type":"session.text.delta","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","ordinal":0,"delta":"answer"}}`,
"reasoning": `{"type":"session.reasoning.delta","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","ordinal":0,"delta":"think"}}`,
}
ids := map[string]string{}
for kind, payload := range payloads {
last := ""
frame, ok, err := NormalizeChatEvent([]byte(payload), "ses_1", &last)
if err != nil || !ok {
t.Fatalf("%s delta: ok=%v err=%v", kind, ok, err)
}
if frame.Part == nil {
t.Fatalf("%s delta carried no part", kind)
}
if frame.Part.Type != kind {
t.Fatalf("%s delta part type = %q", kind, frame.Part.Type)
}
ids[kind] = frame.Part.ID
}
if ids["text"] == ids["reasoning"] {
t.Fatalf("text and reasoning share part id %q", ids["text"])
}
if ids["text"] != protocol.ChatDeltaPartID("msg_1", protocol.ChatPartText, 0) {
t.Fatalf("text part id = %q, want the stable derivation", ids["text"])
}
if ids["reasoning"] != protocol.ChatDeltaPartID("msg_1", protocol.ChatPartReasoning, 0) {
t.Fatalf("reasoning part id = %q, want the stable derivation", ids["reasoning"])
}
}
// TestChatToolLifecycleCarriesNameAndInput pins what the broker learns from each
// verified tool frame: input.started carries the name, input.ended carries the
// arguments as its `text` string, called carries the parsed input object, and
// progress/success carry neither — so those frames must surface no name and no
// input, leaving the retention to the consumer instead of blanking the field.
func TestChatToolLifecycleCarriesNameAndInput(t *testing.T) {
frames := []struct {
name string
payload string
wantStatus string
wantName string
wantInput string // the JSON encoding expected, "" for absent
}{
{
name: "input.started names the call",
payload: `{"type":"session.tool.input.started","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","name":"shell"}}`,
wantStatus: protocol.ChatToolStreaming,
wantName: "shell",
},
{
name: "input.ended carries the arguments as text",
payload: `{"type":"session.tool.input.ended","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","text":"{\"command\":\"ls\"}"}}`,
wantStatus: protocol.ChatToolStreaming,
wantInput: `{"command":"ls"}`,
},
{
name: "called carries the parsed input object",
payload: `{"type":"session.tool.called","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","input":{"command":"ls"}}}`,
wantStatus: protocol.ChatToolRunning,
wantInput: `{"command":"ls"}`,
},
{
name: "progress carries metadata only",
payload: `{"type":"session.tool.progress","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","metadata":{"exit":0}}}`,
wantStatus: protocol.ChatToolRunning,
},
{
name: "success carries the output and neither name nor input",
payload: `{"type":"session.tool.success","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","content":[{"type":"text","text":"ok"}],"metadata":{"exit":0},"executed":true}}`,
wantStatus: protocol.ChatToolCompleted,
},
}
for _, tc := range frames {
t.Run(tc.name, func(t *testing.T) {
last := ""
frame, ok, err := NormalizeChatEvent([]byte(tc.payload), "ses_1", &last)
if err != nil || !ok {
t.Fatalf("ok=%v err=%v", ok, err)
}
if frame.Part == nil || frame.Part.State == nil {
t.Fatalf("tool frame = %+v", frame)
}
if frame.Part.State.Status != tc.wantStatus {
t.Fatalf("status = %q, want %q", frame.Part.State.Status, tc.wantStatus)
}
if frame.Part.Name != tc.wantName {
t.Fatalf("name = %q, want %q", frame.Part.Name, tc.wantName)
}
if tc.wantInput == "" {
if len(frame.Part.State.Input) != 0 {
t.Fatalf("input = %s, want absent", frame.Part.State.Input)
}
return
}
if !json.Valid(frame.Part.State.Input) {
t.Fatalf("input %s is not valid JSON", frame.Part.State.Input)
}
var want, got any
if err := json.Unmarshal([]byte(tc.wantInput), &want); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(frame.Part.State.Input, &got); err != nil {
t.Fatalf("input %s: %v", frame.Part.State.Input, err)
}
if fmt.Sprintf("%v", want) != fmt.Sprintf("%v", got) {
t.Fatalf("input = %s, want %s", frame.Part.State.Input, tc.wantInput)
}
})
}
}
// TestChatToolInputFromPartialText pins the streaming case: input.ended's text
// that is not yet a complete JSON value must still surface, carried as a JSON
// string so the frame stays valid and the client can show the partial arguments.
func TestChatToolInputFromPartialText(t *testing.T) {
last := ""
frame, ok, err := NormalizeChatEvent([]byte(`{"type":"session.tool.input.ended","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","text":"{\"command\":\"l"}}`), "ses_1", &last)
if err != nil || !ok {
t.Fatalf("ok=%v err=%v", ok, err)
}
if frame.Part == nil || frame.Part.State == nil {
t.Fatalf("frame = %+v", frame)
}
if !json.Valid(frame.Part.State.Input) {
t.Fatalf("partial input %s is not valid JSON", frame.Part.State.Input)
}
var got string
if err := json.Unmarshal(frame.Part.State.Input, &got); err != nil {
t.Fatalf("partial input is not a JSON string: %s (%v)", frame.Part.State.Input, err)
}
if got != `{"command":"l` {
t.Fatalf("partial input text = %q", got)
}
}
// TestOpenStreamRejectsUnknownBackend pins that the shared construction path
// validates the backend before any process is started, so OpenStream never
// launches a serve for an unknown backend.
+22 -2
View File
@@ -1,6 +1,9 @@
package protocol
import "encoding/json"
import (
"encoding/json"
"fmt"
)
// Chat requests and responses are brokered by the Pipelines server (route B in
// the OpenCode chat research): the phone keeps talking only to /connect and the
@@ -213,8 +216,21 @@ type ChatMessage struct {
Parts []ChatPart `json:"parts"`
}
// ChatDeltaPartID is the stable part id an incremental text or reasoning delta
// stream addresses. OpenCode numbers one message's text and reasoning parts
// independently from 0, so the kind is part of the id: for a single message its
// first text delta and its first reasoning delta are distinct ids and a delta can
// never be appended into the other kind's part. The derivation is deterministic,
// so a reconnect or a resume re-addresses the same part, and it is mirrored in
// the client's ChatProtocol.deltaPartId (data/protocol/Chat.kt).
func ChatDeltaPartID(messageID, kind string, ordinal int) string {
return fmt.Sprintf("%s#%s#%d", messageID, kind, ordinal)
}
// ChatPart is one element of a ChatMessage. Type selects the meaning: text and
// reasoning use Text and ID, tool uses ID, Name, Executed, State and Time.
// reasoning use Text and ID, tool uses ID, Name, Executed, State and Time. A text
// or reasoning part's ID is ChatDeltaPartID(message id, kind, ordinal), so the two
// kinds never share an id within a message.
type ChatPart struct {
Type string `json:"type"`
ID string `json:"id,omitempty"`
@@ -227,6 +243,10 @@ type ChatPart struct {
// ChatToolState is the lifecycle of a tool call. Input is a raw JSON value: a
// partial-JSON string while streaming, an object once running or completed.
// OpenCode spreads one call across several frames (name in input.started, the
// arguments in input.ended's text or called's input, then progress/success
// carrying neither), so Input is absent on the frames that do not carry it and
// every consumer retains the value it already learned instead of blanking it.
// Content is the tool's textual/file result, Metadata its provider stats,
// Error the structured failure for status "error".
type ChatToolState struct {
+68 -13
View File
@@ -108,10 +108,12 @@ func (h *chatHub) clock() time.Time {
}
// apply folds one pushed frame into the retained state. Text and reasoning
// deltas append to their message's part; a tool frame upserts the full part
// (accounting for its whole size) so the latest lifecycle status, input,
// content, metadata, error and time are retained under the frame's message id.
// Step and usage frames update the session's latest progress.
// deltas append to their message's part; a tool frame upserts the part (merging
// the fields it carries onto what is retained, and accounting for the whole
// size) so the latest lifecycle status, name, input, content, metadata, error and
// time are retained under the frame's message id and a later frame that omits a
// field never blanks it. Step and usage frames update the session's latest
// progress.
func (h *chatHub) apply(frame protocol.ChatEventFrame) {
if frame.SessionID == "" {
return
@@ -181,29 +183,82 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) {
if frame.Part == nil || frame.Part.ID == "" || turn.overflow {
return
}
// The tool part is retained whole, so size the complete part (input,
// content, metadata, error and time) and charge the difference against
// the cap on every upsert, not just the first-seen name.
incoming := chatPartBytes(frame.Part)
// The tool part is retained whole, so size the complete, merged part
// (input, content, metadata, error and time) and charge the difference
// against the cap on every upsert, not just the first-seen name.
part, ok := message.parts[frame.Part.ID]
if !ok {
if !turn.retain(incoming) {
merged := mergeChatToolPart(nil, frame.Part)
if !turn.retain(chatPartBytes(&merged)) {
return
}
part = &chatPart{}
message.order = append(message.order, frame.Part.ID)
message.parts[frame.Part.ID] = part
} else if previous := chatPartBytes(&part.part); incoming > previous {
message.parts[frame.Part.ID] = &chatPart{part: merged}
return
}
merged := mergeChatToolPart(&part.part, frame.Part)
incoming := chatPartBytes(&merged)
previous := chatPartBytes(&part.part)
if incoming > previous {
if !turn.retain(incoming - previous) {
return
}
} else {
turn.shrink(previous - incoming)
}
part.part = *frame.Part
part.part = merged
}
}
// mergeChatToolPart folds one tool frame onto the part retained from the frames
// before it. A tool call is reported across several frames that each carry only
// part of it — input.started the name, input.ended the arguments as text, called
// the parsed input, progress metadata, and success the output with no name and no
// input — so a field the broker already learned is never blanked by a later frame
// that omits it. A non-empty incoming field wins; otherwise the retained value is
// kept.
func mergeChatToolPart(previous, incoming *protocol.ChatPart) protocol.ChatPart {
if previous == nil {
return *incoming
}
merged := *incoming
if merged.Type == "" {
merged.Type = previous.Type
}
if merged.Name == "" {
merged.Name = previous.Name
}
merged.Executed = merged.Executed || previous.Executed
if merged.Time == nil {
merged.Time = previous.Time
}
switch {
case incoming.State == nil:
merged.State = previous.State
case previous.State == nil:
// The incoming state stands on its own; merged already carries it.
default:
state := *incoming.State
if state.Status == "" {
state.Status = previous.State.Status
}
if len(state.Input) == 0 {
state.Input = previous.State.Input
}
if len(state.Content) == 0 {
state.Content = previous.State.Content
}
if state.Metadata == nil {
state.Metadata = previous.State.Metadata
}
if state.Error == nil {
state.Error = previous.State.Error
}
merged.State = &state
}
return merged
}
// chatPartBytes estimates the retained size of a part so the per-session cap
// accounts for its whole state instead of only the first-seen name. It walks the
// fields rather than re-marshalling, so the cost matches the bytes it stores.
+47 -3
View File
@@ -267,8 +267,8 @@ func chatFrame(kind, delta string, part *protocol.ChatPart, step *protocol.ChatS
func scriptedChat() *fakeChat {
return &fakeChat{script: []protocol.ChatEventFrame{
chatFrame(protocol.ChatEventText, "hello ", &protocol.ChatPart{Type: protocol.ChatPartText, ID: "msg_1#0"}, nil),
chatFrame(protocol.ChatEventText, "world", &protocol.ChatPart{Type: protocol.ChatPartText, ID: "msg_1#0"}, nil),
chatFrame(protocol.ChatEventText, "hello ", &protocol.ChatPart{Type: protocol.ChatPartText, ID: protocol.ChatDeltaPartID("msg_1", protocol.ChatPartText, 0)}, nil),
chatFrame(protocol.ChatEventText, "world", &protocol.ChatPart{Type: protocol.ChatPartText, ID: protocol.ChatDeltaPartID("msg_1", protocol.ChatPartText, 0)}, nil),
chatFrame(protocol.ChatEventTool, "", &protocol.ChatPart{Type: protocol.ChatPartTool, ID: "call_1", Name: "shell", State: &protocol.ChatToolState{Status: protocol.ChatToolCompleted, Input: []byte(`{"command":"ls"}`)}}, nil),
chatFrame(protocol.ChatEventStep, "", nil, &protocol.ChatStep{Status: protocol.ChatStepEnded, Index: 1}),
}}
@@ -451,7 +451,7 @@ func TestChatMessagesResumesRetainedState(t *testing.T) {
if message.ID != "msg_1" || message.Role != protocol.ChatRoleAssistant || len(message.Parts) != 2 {
t.Fatalf("resumed message = %+v", message)
}
if message.Parts[0].Type != protocol.ChatPartText || message.Parts[0].ID != "msg_1#0" || message.Parts[0].Text != "hello world" {
if message.Parts[0].Type != protocol.ChatPartText || message.Parts[0].ID != protocol.ChatDeltaPartID("msg_1", protocol.ChatPartText, 0) || message.Parts[0].Text != "hello world" {
t.Fatalf("resumed text part = %+v", message.Parts[0])
}
if message.Parts[1].Type != protocol.ChatPartTool || message.Parts[1].State == nil || message.Parts[1].State.Status != protocol.ChatToolCompleted {
@@ -895,6 +895,50 @@ func TestChatHubChargesToolStateBytes(t *testing.T) {
}
}
// TestChatHubRetainsToolNameAndInputAcrossLifecycle pins that a tool call keeps
// its name and its input for its whole lifecycle in the retained state a resume
// serves: the name arrives only in input.started, the arguments in input.ended's
// text or called's input, and then progress/success carry neither, so no later
// frame may blank a field the hub already learned.
func TestChatHubRetainsToolNameAndInputAcrossLifecycle(t *testing.T) {
hub := newChatHub()
apply := func(part *protocol.ChatPart) {
hub.apply(protocol.ChatEventFrame{SessionID: "ses_1", MessageID: "msg_1", Kind: protocol.ChatEventTool, Part: part})
}
// input.started: the name, no arguments yet.
apply(&protocol.ChatPart{Type: protocol.ChatPartTool, ID: "call_1", Name: "shell", State: &protocol.ChatToolState{Status: protocol.ChatToolStreaming}})
// input.ended/called: the arguments, no name.
apply(&protocol.ChatPart{Type: protocol.ChatPartTool, ID: "call_1", State: &protocol.ChatToolState{Status: protocol.ChatToolRunning, Input: json.RawMessage(`{"command":"ls"}`)}})
// progress: metadata only.
apply(&protocol.ChatPart{Type: protocol.ChatPartTool, ID: "call_1", State: &protocol.ChatToolState{Status: protocol.ChatToolRunning, Metadata: map[string]any{"exit": 0}}})
// success: the output, no name and no input.
apply(&protocol.ChatPart{Type: protocol.ChatPartTool, ID: "call_1", State: &protocol.ChatToolState{Status: protocol.ChatToolCompleted, Content: []protocol.ChatToolContent{{Type: "text", Text: "ok"}}}})
messages := hub.snapshot("ses_1")
if len(messages) != 1 || len(messages[0].Parts) != 1 {
t.Fatalf("snapshot = %+v", messages)
}
part := messages[0].Parts[0]
if part.Name != "shell" {
t.Fatalf("tool name was blanked by a later frame: %+v", part)
}
if part.State == nil || part.State.Status != protocol.ChatToolCompleted {
t.Fatalf("tool state = %+v", part.State)
}
var input struct {
Command string `json:"command"`
}
if err := json.Unmarshal(part.State.Input, &input); err != nil || input.Command != "ls" {
t.Fatalf("tool input was blanked by a later frame: %s (err %v)", part.State.Input, err)
}
if len(part.State.Content) != 1 || part.State.Content[0].Text != "ok" {
t.Fatalf("tool content = %+v", part.State.Content)
}
if part.State.Metadata["exit"] == nil {
t.Fatalf("tool metadata was blanked by a later frame: %+v", part.State.Metadata)
}
}
// TestChatPromptSubscribesBeforePrompt pins the subscribe-before-prompt order:
// the firehose has no replay cursor, so the subscription must be attached before
// the prompt is posted or deltas emitted in between are lost.
@@ -2,6 +2,7 @@ package server
import (
"context"
"fmt"
"net/http/httptest"
"path/filepath"
"strings"
@@ -212,6 +213,95 @@ func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
}
}
// TestRunEventSubscribeSnapshotLargerThanBuffer proves a subscribe whose
// snapshot holds more runs than the subscription's bounded buffer delivers the
// snapshot in full, then the ack, then a live delta for a run started
// afterwards. Before the snapshot fix, the subscription wrote its own snapshot
// through the bounded channel, so sendMessage's drop-a-slow-subscriber policy
// fired on the snapshot itself: the subscription closed part-way through, the
// client saw at most runEventSubscriberBuffer frames, and it never received the
// ack or a single later delta.
func TestRunEventSubscribeSnapshotLargerThanBuffer(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
const seeded = runEventSubscriberBuffer + 8
// Seed more finished runs than the buffer holds, each in its own workspace so
// no two runs share a tree, then wait for every one to settle. A blank-scope
// subscribe sees them all.
seededIDs := make([]int64, 0, seeded)
for i := 0; i < seeded; i++ {
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: fmt.Sprintf("snapshot-%d", i), Workspace: eventWorkspace(t), Input: map[string]any{"prompt": "test"}})
if err != nil {
t.Fatal(err)
}
seededIDs = append(seededIDs, run.ID)
}
handler.runner.wg.Wait()
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: ""})
snapshotCtx, cancelSnapshot := context.WithTimeout(ctx, 8*time.Second)
defer cancelSnapshot()
seen := map[int64]bool{}
var ack *protocol.Message
for ack == nil || len(seen) < seeded {
var message protocol.Message
if err := wsjson.Read(snapshotCtx, client.conn, &message); err != nil {
t.Fatalf("snapshot stopped after %d of %d frames (ack=%v): %v", len(seen), seeded, ack != nil, err)
}
switch message.Type {
case protocol.RunEvent:
if message.RunEvent != nil {
seen[message.RunEvent.RunID] = true
}
case protocol.Subscribe:
got := message
ack = &got
// The snapshot precedes the ack: when the ack is read every snapshot
// frame must already have been, or a frame was reordered after it.
if len(seen) != seeded {
t.Fatalf("ack arrived after %d of %d snapshot frames", len(seen), seeded)
}
}
}
if ack.ID != "snap" {
t.Fatalf("subscribe ack = %+v", *ack)
}
for _, id := range seededIDs {
if !seen[id] {
t.Fatalf("snapshot omitted run %d: got %d of %d frames", id, len(seen), seeded)
}
}
// The subscription must outlive its own snapshot: a run started after it must
// still deliver a delta, through to its terminal frame.
late := eventWorkspace(t)
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "late", PipelineID: ids[0], Version: 1, RequestKey: "snapshot-late", Workspace: late, Input: map[string]any{"prompt": "test"}})
deltaCtx, cancelDelta := context.WithTimeout(ctx, 8*time.Second)
defer cancelDelta()
var terminal *protocol.RunEventFrame
for terminal == nil {
var message protocol.Message
if err := wsjson.Read(deltaCtx, client.conn, &message); err != nil {
t.Fatalf("no terminal delta after the snapshot: %v", err)
}
if message.Type != protocol.RunEvent || message.RunEvent == nil || message.RunEvent.Workspace != late {
continue
}
switch message.RunEvent.Status {
case "succeeded", "failed", "canceled":
terminal = message.RunEvent
}
}
if terminal.RunID == 0 || terminal.Outcome != "success" {
t.Fatalf("late terminal frame = %+v", terminal)
}
}
// TestMergeRunEventFrameKeepsIdentity proves a coalesced frame keeps the run's
// identity from an earlier transition when a later delta omits it, so the merged
// frame a client actually receives still carries what its row needs.
+104 -16
View File
@@ -54,9 +54,16 @@ type runCoalescer struct {
}
// runSubscription is one connection's watch over a workspace (and optionally
// explicit run ids). Frames are handed to a dedicated writer goroutine so a slow
// explicit run ids). Deltas are handed to a dedicated writer goroutine so a slow
// or broken subscriber never stalls the producer, and its buffered channel
// bounds how far behind it may fall.
//
// A subscription starts in its snapshot phase: the subscribing connection
// writes the snapshot and the ack itself (writeSnapshot) and only then starts
// the writer goroutine (ready). mu guards that phase, so the drop-a-slow-
// subscriber policy can never fire on the snapshot or lose its ack: while
// snapshotting a delta is held in pending instead of the bounded channel, and
// nothing writes to the channel until ready has released the held frames.
type runSubscription struct {
hub *runEventHub
clientID string
@@ -66,6 +73,10 @@ type runSubscription struct {
done chan struct{}
once sync.Once
write func(protocol.Message) error
mu sync.Mutex
snapshotting bool
pending []protocol.Message
}
func newRunEventHub() *runEventHub {
@@ -85,8 +96,11 @@ func (h *runEventHub) clock() time.Time {
return time.Now()
}
// subscribe registers one connection's watch and starts its writer goroutine. It
// returns nil when the connection already holds the per-client subscription cap.
// subscribe registers one connection's watch and returns it in its snapshot
// phase: the caller delivers the snapshot and the ack with writeSnapshot and
// then calls ready, which releases the deltas emitted meanwhile and starts the
// writer goroutine. It returns nil when the connection already holds the
// per-client subscription cap.
func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, write func(protocol.Message) error) *runSubscription {
if h == nil || write == nil {
return nil
@@ -97,12 +111,13 @@ func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, writ
return nil
}
sub := &runSubscription{
hub: h,
clientID: clientID,
workspace: workspace,
ch: make(chan protocol.Message, runEventSubscriberBuffer),
done: make(chan struct{}),
write: write,
hub: h,
clientID: clientID,
workspace: workspace,
ch: make(chan protocol.Message, runEventSubscriberBuffer),
done: make(chan struct{}),
write: write,
snapshotting: true,
}
for _, id := range runIDs {
if id <= 0 {
@@ -119,7 +134,8 @@ func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, writ
}
h.byClient[clientID][sub] = struct{}{}
h.mu.Unlock()
go sub.run()
// The writer goroutine starts only in ready, after the snapshot and the ack
// are on the wire, so a delta can never overtake them.
return sub
}
@@ -320,6 +336,47 @@ func (s *runSubscription) run() {
}
}
// writeSnapshot writes one frame of the subscription's snapshot directly on the
// subscribing connection. It runs only during the snapshot phase, before the
// writer goroutine exists, so it cannot reorder against a delta and is never
// subject to the bounded channel's drop policy. It returns false when the frame
// could not be written, having closed the subscription; the caller must then
// stop writing the snapshot.
func (s *runSubscription) writeSnapshot(msg protocol.Message) bool {
select {
case <-s.done:
return false
default:
}
if err := s.write(msg); err != nil {
s.close()
return false
}
return true
}
// ready ends the snapshot phase: the deltas held while the snapshot was written
// are moved into the channel in order and the writer goroutine starts. The
// channel is empty until then (sendMessage queues into pending) and pending is
// bounded by the channel's capacity, so the move cannot block. A subscription
// closed during its snapshot starts no goroutine at all.
func (s *runSubscription) ready() {
s.mu.Lock()
select {
case <-s.done:
s.mu.Unlock()
return
default:
}
for _, msg := range s.pending {
s.ch <- msg
}
s.pending = nil
s.snapshotting = false
s.mu.Unlock()
go s.run()
}
func (s *runSubscription) send(frame protocol.RunEventFrame) {
s.sendMessage(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame})
}
@@ -330,6 +387,23 @@ func (s *runSubscription) sendMessage(msg protocol.Message) {
return
default:
}
s.mu.Lock()
if s.snapshotting {
// The snapshot is still being written on the subscribing connection and
// no writer goroutine is draining yet, so closing here would truncate a
// snapshot in flight and swallow its ack. Hold the delta instead, bounded
// the same as the channel: the policy can only ever affect deltas, since
// the snapshot and the ack are written directly by the subscriber.
if len(s.pending) >= runEventSubscriberBuffer {
s.mu.Unlock()
s.close()
return
}
s.pending = append(s.pending, msg)
s.mu.Unlock()
return
}
s.mu.Unlock()
select {
case s.ch <- msg:
default:
@@ -395,9 +469,12 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
// handleRunEvents dispatches one subscribe/unsubscribe request and reports
// whether it was a run-event type (so the caller never falls through to the
// generic unknown-command response). Subscribe registers the watch, sends the
// current state of the watched runs through the subscription, then acks; the
// snapshot therefore precedes every later delta.
// generic unknown-command response). Subscribe registers the watch, writes the
// current state of the watched runs as its snapshot, acks, and only then lets
// deltas flow: the snapshot and the ack go straight out on the subscribing
// connection, so a snapshot larger than the subscription's buffer can no longer
// drop the subscription it is seeding, and the ordering is always snapshot,
// ack, deltas.
func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner, clientID string, write func(protocol.Message) error) bool {
switch request.Type {
case protocol.Subscribe:
@@ -414,6 +491,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
// workspace-scoped read the polling clients use, instead of a new query.
summaries, err := runs.listRuns()
if err != nil {
sub.close()
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
return true
}
@@ -421,7 +499,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
if !sub.matches(summary.Workspace, summary.ID) {
continue
}
sub.send(protocol.RunEventFrame{
frame := protocol.RunEventFrame{
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
@@ -434,9 +512,19 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
})
}
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
// The connection write failed; the subscription is closed and the
// read loop will stop on its next read error, so there is nothing
// left to reply to.
return true
}
}
sub.sendMessage(protocol.Message{Type: protocol.Subscribe, ID: request.ID})
if err := write(protocol.Message{Type: protocol.Subscribe, ID: request.ID}); err != nil {
sub.close()
return true
}
sub.ready()
return true
case protocol.Unsubscribe:
events.unsubscribeClient(clientID)
+1 -1
View File
@@ -42,5 +42,5 @@ say why is a review finding.
you checked.
- Terminal width matters: verify at a narrow width as well as a wide one, and never assume colour
support (`plain-nocolor-*` exists for a reason).
- Do not restart the deployed server, the lanes or ntfy to test a TUI change: run the binary against a
- Do not restart the deployed server or ntfy to test a TUI change: run the binary against a
scratch server or use the fixtures.