feat(codex): retain all tool items and expose bounded command output

This commit is contained in:
2026-10-20 18:07:00 +01:00
parent 9cdfd9d9ab
commit 83fef10331
6 changed files with 529 additions and 76 deletions
+76 -76
View File
@@ -54,6 +54,8 @@ type Chat struct {
subs map[string]map[chan protocol.ChatEventFrame]struct{}
forms map[string]pendingForm
approvals map[string]pendingApproval
commands map[string]map[string]commandSnapshot
commandOrder uint64
turns map[string]string
emitted map[string]bool
closed bool
@@ -347,12 +349,6 @@ func (c *Chat) PromptWithMessageIDModel(ctx context.Context, id, text, messageID
return err
}
// Shells is empty: Codex background terminals are not exposed through the chat
// surface yet.
func (c *Chat) Shells(context.Context, string) ([]protocol.ChatShell, error) {
return nil, nil
}
// ActivateSkill is an OpenCode-only chat concept.
func (c *Chat) ActivateSkill(context.Context, string, string) error {
return errors.New("Codex chat has no out-of-band skill activation")
@@ -451,27 +447,45 @@ func (c *Chat) Sessions(ctx context.Context) ([]protocol.ChatSession, error) {
// Messages reads a thread's turns and flattens them into chat messages.
func (c *Chat) Messages(ctx context.Context, id string, limit int, cursor string) (protocol.ChatHistoryPage, error) {
_ = limit
_ = cursor
raw, err := c.call(ctx, "thread/read", map[string]any{"threadId": id, "includeTurns": true})
if limit <= 0 {
limit = 50
}
if limit > 100 {
limit = 100
}
params := map[string]any{"threadId": id, "limit": limit, "sortDirection": "desc", "itemsView": "full"}
if cursor != "" {
params["cursor"] = cursor
}
raw, err := c.call(ctx, "thread/turns/list", params)
if err != nil {
return protocol.ChatHistoryPage{}, err
}
var result struct {
Thread thread `json:"thread"`
Data []turn `json:"data"`
NextCursor *string `json:"nextCursor"`
}
if err := json.Unmarshal(raw, &result); err != nil {
return protocol.ChatHistoryPage{}, err
}
if result.Data == nil {
return protocol.ChatHistoryPage{}, errors.New("invalid Codex history page")
}
messages := []protocol.ChatMessage{}
for _, turn := range result.Thread.Turns {
for index := len(result.Data) - 1; index >= 0; index-- {
turn := result.Data[index]
for _, item := range turn.Items {
if msg, ok := item.chatMessage(id); ok {
messages = append(messages, msg)
}
}
}
return protocol.ChatHistoryPage{Messages: messages}, nil
page := protocol.ChatHistoryPage{Messages: messages}
if result.NextCursor != nil {
page.NextCursor = *result.NextCursor
page.HasMore = page.NextCursor != ""
}
return page, nil
}
// Answer replies to a pending requestUserInput request.
@@ -657,71 +671,38 @@ func (c *Chat) handleNotification(msg rpcMessage) {
c.markEmitted(params.ItemID)
c.emit(textFrame(params.ThreadID, params.ItemID, protocol.ChatEventReasoning, params.Delta))
}
case "item/commandExecution/outputDelta":
c.commandDelta(params.ThreadID, params.ItemID, params.Delta)
}
}
func (c *Chat) handleItem(threadID, itemID string, raw json.RawMessage, completed bool) {
var item struct {
ID string `json:"id"`
Type string `json:"type"`
Text string `json:"text"`
Status string `json:"status"`
Command string `json:"command"`
Query string `json:"query"`
Name string `json:"name"`
Input any `json:"input"`
Error string `json:"error"`
}
if err := json.Unmarshal(raw, &item); err != nil {
var record item
if err := json.Unmarshal(raw, &record); err != nil {
return
}
if item.ID == "" {
item.ID = itemID
if record.ID == "" {
record.ID = itemID
}
switch item.Type {
switch record.Type {
case "agentMessage":
if completed && item.Text != "" && !c.wasEmitted(item.ID) {
c.emit(textFrame(threadID, item.ID, protocol.ChatEventText, item.Text))
if completed && record.Text != "" && !c.wasEmitted(record.ID) {
c.emit(textFrame(threadID, record.ID, protocol.ChatEventText, record.Text))
}
case "reasoning":
if completed && item.Text != "" && !c.wasEmitted(item.ID) {
c.emit(textFrame(threadID, item.ID, protocol.ChatEventReasoning, item.Text))
text := record.reasoningText()
if completed && text != "" && !c.wasEmitted(record.ID) {
c.emit(textFrame(threadID, record.ID, protocol.ChatEventReasoning, text))
}
case "commandExecution", "fileChange", "mcpToolCall", "webSearch":
status := protocol.ChatToolRunning
if completed {
if strings.EqualFold(item.Status, "failed") || item.Error != "" {
status = protocol.ChatToolError
} else {
status = protocol.ChatToolCompleted
}
}
name := item.Type
var input any
switch item.Type {
case "commandExecution":
name = "shell"
input = map[string]any{"command": item.Command}
case "webSearch":
name = "websearch"
input = map[string]any{"query": item.Query}
default:
if item.Name != "" {
name = item.Name
}
input = item.Input
}
encoded, _ := json.Marshal(input)
state := &protocol.ChatToolState{Status: status, Input: encoded}
if item.Text != "" {
state.Content = []protocol.ChatToolContent{{Type: "text", Text: item.Text}}
}
if item.Error != "" {
state.Error = &protocol.ChatError{Message: item.Error}
case "userMessage":
return
default:
if record.Type == "commandExecution" {
c.recordCommand(threadID, record, true)
}
part := record.toolPart(completed)
c.emit(protocol.ChatEventFrame{
SessionID: threadID, MessageID: item.ID, Kind: protocol.ChatEventTool,
Part: &protocol.ChatPart{Type: protocol.ChatPartTool, ID: item.ID, Name: name, State: state},
SessionID: threadID, MessageID: record.ID, Kind: protocol.ChatEventTool, Part: &part,
})
}
}
@@ -769,17 +750,30 @@ type turn struct {
}
type item struct {
ID string `json:"id"`
ClientID string `json:"clientId"`
Type string `json:"type"`
Text string `json:"text"`
Content json.RawMessage `json:"content"`
Status string `json:"status"`
Command string `json:"command"`
Query string `json:"query"`
Name string `json:"name"`
Input any `json:"input"`
Error string `json:"error"`
Raw json.RawMessage `json:"-"`
ID string `json:"id"`
ClientID string `json:"clientId"`
Type string `json:"type"`
Text string `json:"text"`
Content json.RawMessage `json:"content"`
Status string `json:"status"`
Command string `json:"command"`
Query string `json:"query"`
Name string `json:"name"`
Input any `json:"input"`
Error json.RawMessage `json:"error"`
Arguments json.RawMessage `json:"arguments"`
Result json.RawMessage `json:"result"`
Output json.RawMessage `json:"output"`
ContentItems json.RawMessage `json:"contentItems"`
Changes json.RawMessage `json:"changes"`
Tool string `json:"tool"`
Namespace string `json:"namespace"`
Server string `json:"server"`
AggregatedOutput string `json:"aggregatedOutput"`
ExitCode *int `json:"exitCode"`
DurationMS *int64 `json:"durationMs"`
Summary []string `json:"summary"`
}
func (t thread) chatSession() protocol.ChatSession {
@@ -817,8 +811,14 @@ func (i item) chatMessage(sessionID string) (protocol.ChatMessage, bool) {
if i.Text != "" {
msg.Parts = append(msg.Parts, protocol.ChatPart{Type: protocol.ChatPartText, Text: i.Text})
}
case "reasoning":
msg.Role = protocol.ChatRoleAssistant
if text := i.reasoningText(); text != "" {
msg.Parts = append(msg.Parts, protocol.ChatPart{Type: protocol.ChatPartReasoning, Text: text})
}
default:
return protocol.ChatMessage{}, false
msg.Role = protocol.ChatRoleAssistant
msg.Parts = []protocol.ChatPart{i.toolPart(!strings.EqualFold(i.Status, "inProgress"))}
}
return msg, len(msg.Parts) > 0
}
+128
View File
@@ -0,0 +1,128 @@
package codex
import (
"encoding/json"
"strings"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func rawText(raw json.RawMessage) string {
if len(raw) == 0 || string(raw) == "null" {
return ""
}
var text string
if json.Unmarshal(raw, &text) == nil {
return text
}
return string(raw)
}
func boundedToolText(text string) string {
const limit = 64 << 10
if len(text) > limit {
return text[:limit] + "\n… (tool detail truncated at 64 KiB)"
}
return text
}
func (i item) reasoningText() string {
if i.Text != "" {
return i.Text
}
var content []string
_ = json.Unmarshal(i.Content, &content)
return strings.Join(append(append([]string{}, i.Summary...), content...), "\n\n")
}
func (i item) toolPart(completed bool) protocol.ChatPart {
name := i.Type
if i.Name != "" {
name = i.Name
}
if i.Tool != "" {
name = i.Tool
}
if i.Server != "" {
name = i.Server + "/" + name
} else if i.Namespace != "" {
name = i.Namespace + "/" + name
}
var input any = i.Input
if len(i.Arguments) > 0 {
input = i.Arguments
}
switch i.Type {
case "commandExecution":
name = "shell"
input = map[string]any{"command": i.Command}
case "webSearch":
name = "websearch"
if i.Query != "" {
input = map[string]any{"query": i.Query}
} else {
input = i.Raw
}
case "fileChange":
input = map[string]any{"changes": i.Changes}
}
if input == nil {
input = map[string]any{"type": i.Type, "detail": i.Raw}
}
encoded, _ := json.Marshal(input)
if len(encoded) > 64<<10 {
encoded, _ = json.Marshal(map[string]any{"truncated": true, "preview": boundedToolText(string(encoded))})
}
status := protocol.ChatToolRunning
if completed {
status = protocol.ChatToolCompleted
}
message := ""
if len(i.Error) > 0 && string(i.Error) != "null" {
var detail struct {
Message string `json:"message"`
}
if json.Unmarshal(i.Error, &detail) == nil && detail.Message != "" {
message = detail.Message
} else {
message = rawText(i.Error)
}
}
if message != "" || strings.EqualFold(i.Status, "failed") || strings.EqualFold(i.Status, "declined") || (completed && i.ExitCode != nil && *i.ExitCode != 0) {
status = protocol.ChatToolError
}
state := &protocol.ChatToolState{Status: status, Input: encoded}
output := i.AggregatedOutput
for _, raw := range []json.RawMessage{i.Result, i.Output, i.ContentItems} {
if text := rawText(raw); text != "" {
if output != "" {
output += "\n"
}
output += text
}
}
if i.Text != "" {
if output != "" {
output += "\n"
}
output += i.Text
}
if output != "" {
state.Content = []protocol.ChatToolContent{{Type: "text", Text: boundedToolText(output)}}
}
if message != "" {
state.Error = &protocol.ChatError{Message: boundedToolText(message)}
}
return protocol.ChatPart{Type: protocol.ChatPartTool, ID: i.ID, Name: name, State: state}
}
func (i *item) UnmarshalJSON(raw []byte) error {
type plain item
var value plain
if err := json.Unmarshal(raw, &value); err != nil {
return err
}
*i = item(value)
i.Raw = append(json.RawMessage(nil), raw...)
return nil
}
+71
View File
@@ -0,0 +1,71 @@
package codex
import (
"context"
"encoding/json"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestStructuredToolHistoryIsNotDropped(t *testing.T) {
chat := modelTestChat(t, func(method string, _ json.RawMessage) any {
if method != "thread/turns/list" {
t.Errorf("unexpected method: %s", method)
}
return json.RawMessage(`{"data":[{"items":[
{"id":"mcp","type":"mcpToolCall","server":"fixture","tool":"lookup","status":"failed","arguments":{"query":"test"},"error":{"message":"controlled failure"}},
{"id":"code","type":"dynamicToolCall","namespace":"functions","tool":"execute","status":"completed","arguments":{"code":"return 1"},"contentItems":[{"type":"inputText","text":"1"}]},
{"id":"output","type":"functionCallOutput","name":"execute","output":"full code-mode output"},
{"id":"future","type":"newProviderTool","input":{"safe":"visible"},"status":"completed"}
]}],"nextCursor":null}`)
})
page, err := chat.Messages(context.Background(), fixtureThread, 50, "")
if err != nil || len(page.Messages) != 4 {
t.Fatalf("tool history missing: %+v %v", page, err)
}
first := page.Messages[0].Parts[0]
if first.Name != "fixture/lookup" || first.State.Status != protocol.ChatToolError || first.State.Error.Message != "controlled failure" {
t.Fatalf("structured MCP failure lost: %+v", first)
}
if page.Messages[1].Parts[0].Name != "functions/execute" || !strings.Contains(page.Messages[1].Parts[0].State.Content[0].Text, "1") {
t.Fatal("code-mode output lost")
}
if page.Messages[2].Parts[0].State.Content[0].Text != "full code-mode output" {
t.Fatal("function output lost")
}
if page.Messages[3].Parts[0].Name != "newProviderTool" {
t.Fatal("unknown tool silently dropped")
}
}
func TestLiveStructuredMCPErrorIsRenderable(t *testing.T) {
c := &Chat{subs: map[string]map[chan protocol.ChatEventFrame]struct{}{}}
ch := c.subscribe(fixtureThread)
defer c.unsubscribe(fixtureThread, ch)
c.handleItem(fixtureThread, "", json.RawMessage(`{"id":"mcp","type":"mcpToolCall","server":"fixture","tool":"lookup","status":"failed","error":{"message":"controlled error"}}`), true)
select {
case frame := <-ch:
if frame.Part == nil || frame.Part.State.Error == nil || frame.Part.State.Error.Message != "controlled error" {
t.Fatalf("bad tool frame: %+v", frame)
}
default:
t.Fatal("structured tool event disappeared")
}
}
func TestHistoryPaginationPreservesOpaqueCursorAndOrder(t *testing.T) {
c := modelTestChat(t, func(method string, params json.RawMessage) any {
var p map[string]any
json.Unmarshal(params, &p)
if method != "thread/turns/list" || p["cursor"] != "opaque cursor" || p["limit"] != float64(100) || p["itemsView"] != "full" {
t.Errorf("wrong history request: %s %s", method, params)
}
return json.RawMessage(`{"data":[{"items":[{"id":"new","type":"agentMessage","text":"new"}]},{"items":[{"id":"old","type":"agentMessage","text":"old"}]}],"nextCursor":"next opaque"}`)
})
page, err := c.Messages(context.Background(), fixtureThread, 10000, "opaque cursor")
if err != nil || len(page.Messages) != 2 || page.Messages[0].ID != "old" || page.Messages[1].ID != "new" || !page.HasMore || page.NextCursor != "next opaque" {
t.Fatalf("history=%+v err=%v", page, err)
}
}
+1
View File
@@ -51,6 +51,7 @@ func modelTestChat(t *testing.T, handle func(string, json.RawMessage) any) *Chat
if err != nil {
t.Fatal(err)
}
conn.SetReadLimit(16 << 20) // Match the real App Server adapter's frame bound.
c := &Chat{conn: conn, ctx: ctx, cancel: cancel, pending: map[string]chan rpcResponse{}, subs: map[string]map[chan protocol.ChatEventFrame]struct{}{}, forms: map[string]pendingForm{}, turns: map[string]string{}, emitted: map[string]bool{}}
go c.readLoop()
t.Cleanup(func() { c.Close(); srv.Close() })
+182
View File
@@ -0,0 +1,182 @@
package codex
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"git.a-wan.net/alan/pipelines/internal/protocol"
"sort"
)
const commandOutputCap = 128 << 10
type commandSnapshot struct {
shell protocol.ChatShell
output string
size int64
order uint64
}
func codexShellID(thread, id string) string {
sum := sha256.Sum256([]byte(thread + "\x00" + id))
return "sh_codex_" + hex.EncodeToString(sum[:])
}
func commandRunning(i item) bool {
return i.Status == "inProgress" || i.Status == "running" || i.Status == ""
}
func (c *Chat) recordCommand(thread string, i item, emit bool) {
if i.ID == "" {
return
}
id := codexShellID(thread, i.ID)
c.mu.Lock()
if c.commands == nil {
c.commands = map[string]map[string]commandSnapshot{}
}
if c.commands[thread] == nil {
c.commands[thread] = map[string]commandSnapshot{}
}
old, exists := c.commands[thread][id]
// An older readback must not reopen a command whose exit we already saw.
if !emit && exists && old.shell.Status == protocol.ChatShellExited && commandRunning(i) {
c.mu.Unlock()
return
}
status := protocol.ChatShellExited
if commandRunning(i) {
status = protocol.ChatShellRunning
}
snapshot := old
snapshot.shell = protocol.ChatShell{ID: id, Command: i.Command, Status: status, Exit: i.ExitCode}
if snapshot.shell.Command == "" {
snapshot.shell.Command = old.shell.Command
}
if len(i.AggregatedOutput) > 0 && int64(len(i.AggregatedOutput)) >= old.size {
snapshot.output = i.AggregatedOutput
snapshot.size = int64(len(i.AggregatedOutput))
if len(snapshot.output) > commandOutputCap {
snapshot.output = snapshot.output[:commandOutputCap]
}
}
c.commandOrder++
snapshot.order = c.commandOrder
c.commands[thread][id] = snapshot
// Retain active commands plus a bounded recent exited/output cache. A lazy
// output read can recover older exits from the authoritative thread history.
for len(c.commands[thread]) > 128 {
key := ""
var order uint64
for candidate, value := range c.commands[thread] {
if value.shell.Status == protocol.ChatShellExited && (key == "" || value.order < order) {
key = candidate
order = value.order
}
}
if key == "" {
break
}
delete(c.commands[thread], key)
}
c.mu.Unlock()
if emit {
c.emit(protocol.ChatEventFrame{SessionID: thread, Kind: protocol.ChatEventShell, Shell: &snapshot.shell})
}
}
func (c *Chat) commandDelta(thread, id, delta string) {
if id == "" || delta == "" {
return
}
key := codexShellID(thread, id)
c.mu.Lock()
snapshot, ok := c.commands[thread][key]
if ok && snapshot.shell.Status == protocol.ChatShellRunning {
snapshot.size += int64(len(delta))
remaining := commandOutputCap - len(snapshot.output)
if remaining > 0 {
if len(delta) > remaining {
delta = delta[:remaining]
}
snapshot.output += delta
}
c.commands[thread][key] = snapshot
}
c.mu.Unlock()
}
// Shells uses the installed version's read-only thread API. It does not invent
// a background-terminal endpoint, resume a turn, or enable experimental flags.
func (c *Chat) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) {
c.mu.Lock()
before := c.commandOrder
c.mu.Unlock()
raw, err := c.call(ctx, "thread/read", map[string]any{"threadId": id, "includeTurns": true})
if err != nil {
return nil, err
}
var response struct {
Thread thread `json:"thread"`
}
if json.Unmarshal(raw, &response) != nil || response.Thread.ID != id {
return nil, errors.New("invalid Codex command readback")
}
seen := map[string]bool{}
for _, turn := range response.Thread.Turns {
for _, i := range turn.Items {
if i.Type == "commandExecution" {
seen[codexShellID(id, i.ID)] = true
c.recordCommand(id, i, false)
}
}
}
c.mu.Lock()
defer c.mu.Unlock()
result := []protocol.ChatShell{}
for key, snapshot := range c.commands[id] {
if !seen[key] && snapshot.order <= before {
delete(c.commands[id], key)
continue
}
if snapshot.shell.Status == protocol.ChatShellRunning {
result = append(result, snapshot.shell)
}
}
sort.Slice(result, func(i, j int) bool { return result[i].ID < result[j].ID })
return result, nil
}
func (c *Chat) ShellOutput(ctx context.Context, q protocol.ChatShellOutputQuery) (protocol.ChatShellOutputPage, error) {
if q.Cursor < 0 || q.Cursor > protocol.ChatShellOutputMaxCursor || q.Limit <= 0 || q.Limit > protocol.ChatShellOutputLimit {
return protocol.ChatShellOutputPage{}, errors.New("invalid bounded Codex output query")
}
c.mu.Lock()
for thread, commands := range c.commands {
if _, known := commands[q.ShellID]; known && thread != q.SessionID {
c.mu.Unlock()
return protocol.ChatShellOutputPage{}, errors.New("command belongs to another thread")
}
}
c.mu.Unlock()
if _, err := c.Shells(ctx, q.SessionID); err != nil {
return protocol.ChatShellOutputPage{}, err
}
c.mu.Lock()
snapshot, ok := c.commands[q.SessionID][q.ShellID]
c.mu.Unlock()
if !ok {
return protocol.ChatShellOutputPage{Shell: protocol.ChatShell{ID: q.ShellID}, Unavailable: true}, nil
}
start := q.Cursor
if start > int64(len(snapshot.output)) {
start = int64(len(snapshot.output))
}
end := start + int64(q.Limit)
if end > int64(len(snapshot.output)) {
end = int64(len(snapshot.output))
}
return protocol.ChatShellOutputPage{Shell: snapshot.shell, Output: snapshot.output[start:end], Cursor: end, Size: snapshot.size, Truncated: end < snapshot.size}, nil
}
+71
View File
@@ -0,0 +1,71 @@
package codex
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestCommandReadbackOutputAndOwnership(t *testing.T) {
var mu sync.Mutex
status, output := "inProgress", "hello"
calls := 0
chat := modelTestChat(t, func(method string, p json.RawMessage) any {
mu.Lock()
defer mu.Unlock()
calls++
if method != "thread/read" {
t.Errorf("unexpected method: %s", method)
}
return map[string]any{"thread": map[string]any{"id": fixtureThread, "turns": []any{map[string]any{"items": []any{map[string]any{"id": "cmd", "type": "commandExecution", "command": "printf hello", "status": status, "aggregatedOutput": output}}}}}}
})
shells, err := chat.Shells(context.Background(), fixtureThread)
if err != nil || len(shells) != 1 || shells[0].Status != protocol.ChatShellRunning {
t.Fatalf("shells=%+v err=%v", shells, err)
}
q := protocol.ChatShellOutputQuery{SessionID: fixtureThread, ShellID: shells[0].ID, Limit: 3}
page, err := chat.ShellOutput(context.Background(), q)
if err != nil || page.Output != "hel" || !page.Truncated || page.Size != 5 {
t.Fatalf("output=%+v err=%v", page, err)
}
mu.Lock()
before := calls
mu.Unlock()
foreign := q
foreign.SessionID = "foreign"
if _, err := chat.ShellOutput(context.Background(), foreign); err == nil {
t.Fatal("cross-thread output accepted")
}
mu.Lock()
if calls != before {
t.Error("foreign query reached upstream")
}
status = "completed"
output = strings.Repeat("x", commandOutputCap+100)
mu.Unlock()
shells, err = chat.Shells(context.Background(), fixtureThread)
if err != nil || len(shells) != 0 {
t.Fatalf("completed command stayed running: %+v %v", shells, err)
}
q.Limit = protocol.ChatShellOutputLimit
page, err = chat.ShellOutput(context.Background(), q)
if err != nil || len(page.Output) != q.Limit || !page.Truncated || page.Size != commandOutputCap+100 {
t.Fatalf("bounded output=%+v err=%v", page, err)
}
}
func TestLiveCommandExitCannotBeReopenedByOldSnapshot(t *testing.T) {
c := &Chat{}
c.recordCommand(fixtureThread, item{ID: "cmd", Command: "printf test", Status: "completed"}, true)
c.recordCommand(fixtureThread, item{ID: "cmd", Command: "printf test", Status: "inProgress"}, false)
c.mu.Lock()
snapshot := c.commands[fixtureThread][codexShellID(fixtureThread, "cmd")]
c.mu.Unlock()
if snapshot.shell.Status != protocol.ChatShellExited {
t.Fatal("old readback reopened completed command")
}
}