fix(codex): reconcile cold connections without resuming queued goals
This commit is contained in:
+54
-24
@@ -58,6 +58,7 @@ type Chat struct {
|
||||
approvals map[string]pendingApproval
|
||||
commands map[string]map[string]commandSnapshot
|
||||
commandOrder uint64
|
||||
starting map[string]bool
|
||||
turns map[string]string
|
||||
emitted map[string]bool
|
||||
closed bool
|
||||
@@ -299,24 +300,7 @@ func (c *Chat) unsubscribe(id string, ch chan protocol.ChatEventFrame) {
|
||||
|
||||
// Stream exposes one thread's notifications as chat frames.
|
||||
func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error {
|
||||
ch := c.subscribe(id)
|
||||
defer c.unsubscribe(id, ch)
|
||||
if ready != nil {
|
||||
ready()
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case frame, ok := <-ch:
|
||||
if !ok {
|
||||
return errors.New("Codex connection closed")
|
||||
}
|
||||
if err := emit(frame); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return c.streamWithReadback(ctx, id, ready, emit)
|
||||
}
|
||||
|
||||
// Prompt starts one turn. The user turn is already broadcast by the server, so
|
||||
@@ -359,7 +343,30 @@ func (c *Chat) PromptWithMessageIDModelFiles(ctx context.Context, id, text, mess
|
||||
if model != "" {
|
||||
params["model"] = model
|
||||
}
|
||||
_, err = c.call(ctx, "turn/start", params)
|
||||
c.mu.Lock()
|
||||
if c.starting == nil {
|
||||
c.starting = map[string]bool{}
|
||||
}
|
||||
c.starting[id] = true
|
||||
c.mu.Unlock()
|
||||
raw, err := c.call(ctx, "turn/start", params)
|
||||
var reply struct {
|
||||
Turn struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"turn"`
|
||||
}
|
||||
if err == nil {
|
||||
_ = json.Unmarshal(raw, &reply)
|
||||
}
|
||||
c.mu.Lock()
|
||||
delete(c.starting, id)
|
||||
if reply.Turn.ID != "" {
|
||||
c.turns[id] = reply.Turn.ID
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if err == nil && reply.Turn.ID != "" {
|
||||
c.emit(stepFrame(id, protocol.ChatStepStarted, reply.Turn.ID))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -493,6 +500,10 @@ func (c *Chat) Messages(ctx context.Context, id string, limit int, cursor string
|
||||
messages = append(messages, msg)
|
||||
}
|
||||
}
|
||||
if turn.ID != "" && rawText(turn.Error) != "" {
|
||||
part := item{ID: turn.ID + "#error", Type: "Codex turn", Status: "failed", Error: turn.Error, Input: map[string]any{"turn_id": turn.ID}}.toolPart(true)
|
||||
messages = append(messages, protocol.ChatMessage{ID: part.ID, SessionID: id, Role: protocol.ChatRoleAssistant, Parts: []protocol.ChatPart{part}})
|
||||
}
|
||||
}
|
||||
page := protocol.ChatHistoryPage{Messages: messages}
|
||||
if result.NextCursor != nil {
|
||||
@@ -664,13 +675,25 @@ func (c *Chat) handleNotification(msg rpcMessage) {
|
||||
if c.turns[params.ThreadID] == "" {
|
||||
c.turns[params.ThreadID] = params.Turn.ID
|
||||
}
|
||||
nativeID := c.turns[params.ThreadID]
|
||||
c.mu.Unlock()
|
||||
c.emit(stepFrame(params.ThreadID, "started"))
|
||||
c.emit(stepFrame(params.ThreadID, "started", nativeID))
|
||||
case "turn/completed":
|
||||
completedID := params.TurnID
|
||||
if completedID == "" {
|
||||
completedID = params.Turn.ID
|
||||
}
|
||||
if completedID == "" {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
if current := c.turns[params.ThreadID]; current != "" && current != completedID {
|
||||
c.mu.Unlock()
|
||||
return
|
||||
}
|
||||
delete(c.turns, params.ThreadID)
|
||||
c.mu.Unlock()
|
||||
c.emit(stepFrame(params.ThreadID, protocol.ChatStepEnded))
|
||||
c.emit(stepFrame(params.ThreadID, protocol.ChatStepEnded, completedID))
|
||||
case "item/started":
|
||||
c.handleItem(params.ThreadID, params.ItemID, params.Item, false)
|
||||
case "item/completed":
|
||||
@@ -740,10 +763,14 @@ func textFrame(threadID, itemID, kind, delta string) protocol.ChatEventFrame {
|
||||
}
|
||||
}
|
||||
|
||||
func stepFrame(threadID, status string) protocol.ChatEventFrame {
|
||||
func stepFrame(threadID, status string, nativeID ...string) protocol.ChatEventFrame {
|
||||
step := &protocol.ChatStep{Status: status}
|
||||
if len(nativeID) > 0 {
|
||||
step.TurnID = nativeID[0]
|
||||
}
|
||||
return protocol.ChatEventFrame{
|
||||
SessionID: threadID, Kind: protocol.ChatEventStep,
|
||||
Step: &protocol.ChatStep{Status: status},
|
||||
Step: step,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -760,7 +787,10 @@ type thread struct {
|
||||
}
|
||||
|
||||
type turn struct {
|
||||
Items []item `json:"items"`
|
||||
ID string `json:"id"`
|
||||
Status string `json:"status"`
|
||||
Error json.RawMessage `json:"error"`
|
||||
Items []item `json:"items"`
|
||||
}
|
||||
|
||||
type item struct {
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
package codex
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
type turnReadback struct {
|
||||
turn turn
|
||||
digest [32]byte
|
||||
}
|
||||
|
||||
func (c *Chat) latestTurn(ctx context.Context, id string) (turnReadback, error) {
|
||||
read, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
raw, err := c.call(read, "thread/turns/list", map[string]any{"threadId": id, "limit": 1, "sortDirection": "desc", "itemsView": "full"})
|
||||
if err != nil {
|
||||
return turnReadback{}, err
|
||||
}
|
||||
var page struct {
|
||||
Data []json.RawMessage `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(raw, &page) != nil || page.Data == nil {
|
||||
return turnReadback{}, errors.New("invalid Codex turn readback")
|
||||
}
|
||||
if len(page.Data) == 0 {
|
||||
return turnReadback{}, nil
|
||||
}
|
||||
var current turn
|
||||
if json.Unmarshal(page.Data[0], ¤t) != nil {
|
||||
return turnReadback{}, errors.New("invalid Codex turn readback")
|
||||
}
|
||||
return turnReadback{turn: current, digest: sha256.Sum256(page.Data[0])}, nil
|
||||
}
|
||||
|
||||
func activeTurn(t turn) bool { return t.ID != "" && t.Status == "inProgress" }
|
||||
func terminalTurn(t turn) bool {
|
||||
return t.ID != "" && (t.Status == "completed" || t.Status == "failed" || t.Status == "interrupted")
|
||||
}
|
||||
|
||||
// A fresh App Server connection may accept turn/start without sending thread
|
||||
// notifications. Reconcile read-only history instead of thread/resume, which
|
||||
// can awaken queued goals. Resync requests replace history, never fake deltas.
|
||||
func (c *Chat) streamWithReadback(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error {
|
||||
ch := c.subscribe(id)
|
||||
defer c.unsubscribe(id, ch)
|
||||
last, _ := c.latestTurn(ctx, id)
|
||||
if ready != nil {
|
||||
ready()
|
||||
}
|
||||
if activeTurn(last.turn) {
|
||||
c.mu.Lock()
|
||||
c.turns[id] = last.turn.ID
|
||||
c.mu.Unlock()
|
||||
c.emit(stepFrame(id, protocol.ChatStepStarted, last.turn.ID))
|
||||
}
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case frame, ok := <-ch:
|
||||
if !ok {
|
||||
return errors.New("Codex connection closed")
|
||||
}
|
||||
if err := emit(frame); err != nil {
|
||||
return err
|
||||
}
|
||||
case <-ticker.C:
|
||||
current, err := c.latestTurn(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current.digest == last.digest {
|
||||
continue
|
||||
}
|
||||
c.mu.Lock()
|
||||
pending := c.starting[id]
|
||||
expected := c.turns[id]
|
||||
if pending || (expected != "" && current.turn.ID != "" && expected != current.turn.ID) {
|
||||
c.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
started := activeTurn(current.turn) && !activeTurn(last.turn)
|
||||
if activeTurn(current.turn) {
|
||||
c.turns[id] = current.turn.ID
|
||||
}
|
||||
ended := terminalTurn(current.turn) && expected == current.turn.ID
|
||||
if ended {
|
||||
delete(c.turns, id)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
last = current
|
||||
if started {
|
||||
c.emit(stepFrame(id, protocol.ChatStepStarted, current.turn.ID))
|
||||
}
|
||||
c.emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventResync})
|
||||
if ended {
|
||||
c.emit(stepFrame(id, protocol.ChatStepEnded, current.turn.ID))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package codex
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func TestStreamReconcilesAcceptedTurnWithoutNativeNotifications(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
started := false
|
||||
c := modelTestChat(t, func(method string, _ json.RawMessage) any {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
switch method {
|
||||
case "turn/start":
|
||||
started = true
|
||||
return map[string]any{"turn": map[string]any{"id": "turn-new"}}
|
||||
case "thread/turns/list":
|
||||
if !started {
|
||||
return json.RawMessage(`{"data":[],"nextCursor":null}`)
|
||||
}
|
||||
return json.RawMessage(`{"data":[{"id":"turn-new","status":"completed","items":[{"id":"reply","type":"agentMessage","text":"done"}]}],"nextCursor":null}`)
|
||||
default:
|
||||
t.Errorf("readback must never resume/configure: %s", method)
|
||||
return nil
|
||||
}
|
||||
})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
ready, ended := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
sawResync := false
|
||||
go c.Stream(ctx, fixtureThread, func() { close(ready) }, func(frame protocol.ChatEventFrame) error {
|
||||
if frame.Kind == protocol.ChatEventResync {
|
||||
sawResync = true
|
||||
}
|
||||
if frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded {
|
||||
if frame.Step.TurnID != "turn-new" {
|
||||
t.Error("wrong turn completed")
|
||||
}
|
||||
once.Do(func() { close(ended) })
|
||||
}
|
||||
return nil
|
||||
})
|
||||
<-ready
|
||||
if err := c.Prompt(ctx, fixtureThread, "tool-free verification"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-ended:
|
||||
if !sawResync {
|
||||
t.Fatal("completion without authoritative history refresh")
|
||||
}
|
||||
case <-time.After(4 * time.Second):
|
||||
t.Fatal("accepted turn stayed running without native notifications")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLateCompletionDoesNotClearNewCodexTurn(t *testing.T) {
|
||||
c := &Chat{turns: map[string]string{fixtureThread: "new"}, subs: map[string]map[chan protocol.ChatEventFrame]struct{}{}}
|
||||
ch := c.subscribe(fixtureThread)
|
||||
defer c.unsubscribe(fixtureThread, ch)
|
||||
c.handleNotification(rpcMessage{Method: "turn/completed", Params: json.RawMessage(`{"threadId":"` + fixtureThread + `","turn":{"id":"old","status":"completed"}}`)})
|
||||
if c.turns[fixtureThread] != "new" {
|
||||
t.Fatal("old completion released new turn")
|
||||
}
|
||||
select {
|
||||
case <-ch:
|
||||
t.Fatal("stale terminal frame emitted")
|
||||
default:
|
||||
}
|
||||
}
|
||||
@@ -105,6 +105,7 @@ const (
|
||||
ChatEventReasoning = ChatPartReasoning
|
||||
ChatEventTool = ChatPartTool
|
||||
ChatEventStep = "step"
|
||||
ChatEventResync = "resync"
|
||||
ChatEventUsage = "usage"
|
||||
ChatEventUser = "user"
|
||||
// ChatEventShell reports the lifecycle of one shell the agent started. It
|
||||
@@ -443,6 +444,7 @@ type ChatEventFrame struct {
|
||||
// ChatStep is a progress marker for the current step of a turn.
|
||||
type ChatStep struct {
|
||||
Status string `json:"status"`
|
||||
TurnID string `json:"turn_id,omitempty"`
|
||||
Index int `json:"index,omitempty"`
|
||||
// Intermediate distinguishes a provider step from authoritative turn completion.
|
||||
Intermediate bool `json:"intermediate,omitempty"`
|
||||
|
||||
@@ -147,6 +147,13 @@ func checkChatBrowserLifecycle(t *testing.T, backend string) {
|
||||
handler := HandlerWithOptions(ctx, Options{Chat: browserModelFixture{fake, forms}}, store)
|
||||
defer func() { cancel(); WaitRuns(handler) }()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
if req.URL.Path == "/fixture-resync" && req.Method == http.MethodPost {
|
||||
fake.mu.Lock()
|
||||
fake.messages = append(fake.messages, protocol.ChatMessage{ID: "msg_resync", Role: protocol.ChatRoleAssistant, Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "Authoritative resync evidence"}}})
|
||||
fake.mu.Unlock()
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
if req.URL.Path == "/fixture-shell" && req.Method == http.MethodPost {
|
||||
fake.mu.Lock()
|
||||
fake.shells = []protocol.ChatShell{{ID: "sh_" + backend + "_fixture", Status: protocol.ChatShellRunning, Command: "printf controlled"}}
|
||||
|
||||
@@ -147,6 +147,7 @@ type chatRoom struct {
|
||||
pollerStarted bool
|
||||
wake chan struct{}
|
||||
turnDone chan struct{}
|
||||
nativeTurnID string
|
||||
}
|
||||
|
||||
func newChatRoom(base context.Context, sessionID string) *chatRoom {
|
||||
@@ -373,6 +374,15 @@ func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if room != nil && frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.TurnID != "" && !frame.Step.Intermediate {
|
||||
if frame.Step.Status == protocol.ChatStepStarted {
|
||||
room.nativeTurnID = frame.Step.TurnID
|
||||
}
|
||||
if frame.Step.Status == protocol.ChatStepEnded && room.nativeTurnID != frame.Step.TurnID {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
}
|
||||
h.applyLocked(frame)
|
||||
buffered := false
|
||||
var targets []*chatSub
|
||||
@@ -475,6 +485,7 @@ func (h *chatHub) replaceShellsLocked(sessionID string, shells []protocol.ChatSh
|
||||
// wait. Callers must hold h.mu.
|
||||
func (h *chatHub) finishTurnLocked(room *chatRoom) {
|
||||
room.running = false
|
||||
room.nativeTurnID = ""
|
||||
if room.turnDone != nil {
|
||||
close(room.turnDone)
|
||||
room.turnDone = nil
|
||||
@@ -660,6 +671,7 @@ func (h *chatHub) nextPrompt(room *chatRoom) (*chatQueuedPrompt, chan struct{},
|
||||
p := room.queue[0]
|
||||
room.queue = room.queue[1:]
|
||||
room.running = true
|
||||
room.nativeTurnID = ""
|
||||
room.turnDone = make(chan struct{})
|
||||
done := room.turnDone
|
||||
h.mu.Unlock()
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLateHarnessCompletionCannotReleaseNextChatTurn(t *testing.T) {
|
||||
h := newChatHub()
|
||||
room := newChatRoom(context.Background(), "ses_turn")
|
||||
defer room.cancel()
|
||||
room.running = true
|
||||
room.nativeTurnID = "new"
|
||||
room.turnDone = make(chan struct{})
|
||||
h.rooms[room.sessionID] = room
|
||||
h.emitRoom(room, protocol.ChatEventFrame{SessionID: room.sessionID, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepEnded, TurnID: "old"}})
|
||||
if !room.running {
|
||||
t.Fatal("old terminal frame freed next turn queue")
|
||||
}
|
||||
h.emitRoom(room, protocol.ChatEventFrame{SessionID: room.sessionID, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepEnded, TurnID: "new"}})
|
||||
if room.running {
|
||||
t.Fatal("matching terminal frame did not finish turn")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user