Files
pipelines/internal/codex/usage_test.go
T
alan 83b60990d2 feat(codex): retain native session usage snapshots for chat
Verify typed ThreadTokenUsage/TokenUsageBreakdown/Thread/Turn against openai/codex@6fb1c3903d. Use native totalTokens and latest-call context tokens without re-adding cached/reasoning subsets. Replay bounded per-connection snapshots and clear on room/source release.

Cost stays unavailable: no subscription USD usage is reported by this notification. Throughput stays unavailable even with native Turn.durationMs because neither last-call nor cumulative-session output is a compatible turn-specific numerator. Thread readback exposes no token usage; no extra reads, resume, or live provider calls added.

Checks: go test -race ./internal/codex; go vet ./internal/codex; go test ./internal/server ./internal/protocol; git diff --check.
2026-10-21 00:07:12 +01:00

162 lines
6.0 KiB
Go

package codex
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// Synthetic values in the current upstream typed notification shape; no live
// Codex process, provider, configuration, or session is used by these tests.
func usageFixture(t *testing.T) rpcMessage {
t.Helper()
raw, err := os.ReadFile("testdata/token-usage-updated.json")
if err != nil {
t.Fatal(err)
}
var msg rpcMessage
if err := json.Unmarshal(raw, &msg); err != nil {
t.Fatal(err)
}
return msg
}
func usageTestChat() *Chat {
return &Chat{subs: map[string]map[chan protocol.ChatEventFrame]struct{}{}, turns: map[string]string{"thread": "turn_new"}, starting: map[string]bool{"thread": true}}
}
func TestUsageNativeSnapshotAndDuplicate(t *testing.T) {
c := usageTestChat()
ch := c.subscribe("thread")
other := c.subscribe("other")
for range 2 {
c.handleNotification(usageFixture(t))
f := <-ch
u := f.Usage
if f.Kind != protocol.ChatEventUsage || f.SessionID != "thread" || f.NativeTurnID != "turn_new" || f.MessageID != "" || f.Part != nil || f.Step != nil {
t.Fatalf("usage is not an item/step: %+v", f)
}
if u.Scope != "session" || *u.TotalTokens != 12000 || *u.ContextTokens != 3500 || *u.ContextWindow != 258400 || u.InputTokens != 10000 || u.OutputTokens != 2000 {
t.Fatalf("snapshot added subsets, cumulative context, or duplicate: %+v", u)
}
if u.CostUSD != nil || u.OutputTokensPerSecond != nil || u.RateBasis != "" {
t.Fatalf("unreported metrics invented: %+v", u)
}
}
if len(other) != 0 || c.turns["thread"] != "turn_new" || !c.starting["thread"] {
t.Fatal("usage leaked across threads or changed busy state")
}
c.handleNotification(rpcMessage{Method: "thread/tokenUsage/updated", Params: json.RawMessage(`{"threadId":"other","tokenUsage":{"total":{"totalTokens":42}}}`)})
if len(ch) != 0 || (<-other).Usage.TotalTokens == nil {
t.Fatal("foreign session notification misrouted")
}
}
func TestUsageMissingZeroAndInvalid(t *testing.T) {
c := usageTestChat()
ch := c.subscribe("thread")
for _, raw := range []string{
`{}`, `{"threadId":"thread"}`, `{"threadId":"thread","tokenUsage":null}`,
`{"tokenUsage":{}}`, `{"threadId":"thread","tokenUsage":{"total":{"totalTokens":"bad"}}}`,
} {
c.handleNotification(rpcMessage{Method: "thread/tokenUsage/updated", Params: json.RawMessage(raw)})
}
if len(ch) != 0 {
t.Fatal("missing or invalid source became fabricated usage")
}
c.handleNotification(usageFixture(t))
<-ch
// A full sparse snapshot clears prior values instead of leaving stale values.
c.receiveUsage(json.RawMessage(`{"threadId":"thread","tokenUsage":{"total":{"totalTokens":0,"inputTokens":0,"outputTokens":0},"last":{"totalTokens":0},"modelContextWindow":0}}`))
f := <-ch
if f.NativeTurnID != "" || f.Usage.TotalTokens == nil || *f.Usage.TotalTokens != 0 || f.Usage.ContextTokens == nil || *f.Usage.ContextTokens != 0 || f.Usage.ContextWindow == nil || *f.Usage.ContextWindow != 0 {
t.Fatalf("reported zero lost or turn invented: %+v", f)
}
c.receiveUsage(json.RawMessage(`{"threadId":"thread","tokenUsage":{"total":{"totalTokens":-1},"last":{},"modelContextWindow":null}}`))
u := (<-ch).Usage
if u.TotalTokens != nil || u.ContextTokens != nil || u.ContextWindow != nil || u.CostUSD != nil || u.OutputTokensPerSecond != nil {
t.Fatalf("unavailable values became zero or stale: %+v", u)
}
}
func TestUsageStaleTurnCannotRegressOrFreeBusySlot(t *testing.T) {
c := usageTestChat()
ch := c.subscribe("thread")
c.handleNotification(usageFixture(t))
<-ch
c.receiveUsage(json.RawMessage(`{"threadId":"thread","turnId":"turn_old","tokenUsage":{"total":{"totalTokens":10}}}`))
if len(ch) != 0 || *c.usage["thread"].frame.Usage.TotalTokens != 12000 || c.turns["thread"] != "turn_new" {
t.Fatal("stale native turn changed snapshot or busy state")
}
c.handleNotification(rpcMessage{Method: "turn/completed", Params: json.RawMessage(`{"threadId":"thread","turn":{"id":"turn_new","status":"completed","startedAt":100,"completedAt":110,"durationMs":10000}}`)})
<-ch
// Even with native timing, neither session output nor last model-call
// output is a compatible whole-turn numerator. Do not fabricate a rate.
u := c.usage["thread"].frame.Usage
if u.OutputTokensPerSecond != nil || u.RateBasis != "" {
t.Fatal("duration paired with incompatible output counter")
}
c.receiveUsage(json.RawMessage(`{"threadId":"thread","turnId":"turn_old","tokenUsage":{"total":{"totalTokens":10}}}`))
if len(ch) != 0 {
t.Fatal("old turn regressed completed turn usage")
}
}
func TestUsageReattachRetainedWithoutExtraReadback(t *testing.T) {
reads := 0
c := modelTestChat(t, func(method string, _ json.RawMessage) any {
if method != "thread/turns/list" {
t.Errorf("metrics added unexpected RPC: %s", method)
}
reads++
return json.RawMessage(`{"data":[]}`)
})
c.handleNotification(usageFixture(t))
stop := errors.New("snapshot received")
for range 2 {
err := c.Stream(context.Background(), "thread", nil, func(f protocol.ChatEventFrame) error {
if f.Usage == nil || *f.Usage.TotalTokens != 12000 || *f.Usage.ContextTokens != 3500 {
t.Fatalf("reattach lost snapshot: %+v", f)
}
return stop
})
if !errors.Is(err, stop) {
t.Fatal(err)
}
}
if reads != 2 {
t.Fatalf("metrics added reads: %d", reads)
}
c.ReleaseShells("thread")
ch := c.subscribe("thread")
if len(ch) != 0 || len(c.usage) != 0 {
t.Fatal("released room retained usage")
}
}
func TestUsageRetentionBoundAndTransportLoss(t *testing.T) {
c := usageTestChat()
for i := range retainedUsageLimit + 10 {
c.receiveUsage(json.RawMessage(fmt.Sprintf(`{"threadId":"thread_%d","tokenUsage":{"total":{"totalTokens":%d}}}`, i, i)))
}
if len(c.usage) != retainedUsageLimit {
t.Fatalf("unbounded usage cache: %d", len(c.usage))
}
if _, ok := c.usage["thread_0"]; ok {
t.Fatal("oldest usage was not evicted")
}
c.failPending(errors.New("transport lost"))
if len(c.usage) != 0 {
t.Fatal("transport ownership retained usage")
}
c.handleNotification(usageFixture(t))
if len(c.usage) != 0 {
t.Fatal("lost source regained retained usage")
}
}