83b60990d2
Verify typed ThreadTokenUsage/TokenUsageBreakdown/Thread/Turn against openai/codex@6fb1c3903d. Use native totalTokens and latest-call context tokens without re-adding cached/reasoning subsets. Replay bounded per-connection snapshots and clear on room/source release. Cost stays unavailable: no subscription USD usage is reported by this notification. Throughput stays unavailable even with native Turn.durationMs because neither last-call nor cumulative-session output is a compatible turn-specific numerator. Thread readback exposes no token usage; no extra reads, resume, or live provider calls added. Checks: go test -race ./internal/codex; go vet ./internal/codex; go test ./internal/server ./internal/protocol; git diff --check.
162 lines
6.0 KiB
Go
162 lines
6.0 KiB
Go
package codex
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// Synthetic values in the current upstream typed notification shape; no live
|
|
// Codex process, provider, configuration, or session is used by these tests.
|
|
func usageFixture(t *testing.T) rpcMessage {
|
|
t.Helper()
|
|
raw, err := os.ReadFile("testdata/token-usage-updated.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var msg rpcMessage
|
|
if err := json.Unmarshal(raw, &msg); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return msg
|
|
}
|
|
|
|
func usageTestChat() *Chat {
|
|
return &Chat{subs: map[string]map[chan protocol.ChatEventFrame]struct{}{}, turns: map[string]string{"thread": "turn_new"}, starting: map[string]bool{"thread": true}}
|
|
}
|
|
|
|
func TestUsageNativeSnapshotAndDuplicate(t *testing.T) {
|
|
c := usageTestChat()
|
|
ch := c.subscribe("thread")
|
|
other := c.subscribe("other")
|
|
for range 2 {
|
|
c.handleNotification(usageFixture(t))
|
|
f := <-ch
|
|
u := f.Usage
|
|
if f.Kind != protocol.ChatEventUsage || f.SessionID != "thread" || f.NativeTurnID != "turn_new" || f.MessageID != "" || f.Part != nil || f.Step != nil {
|
|
t.Fatalf("usage is not an item/step: %+v", f)
|
|
}
|
|
if u.Scope != "session" || *u.TotalTokens != 12000 || *u.ContextTokens != 3500 || *u.ContextWindow != 258400 || u.InputTokens != 10000 || u.OutputTokens != 2000 {
|
|
t.Fatalf("snapshot added subsets, cumulative context, or duplicate: %+v", u)
|
|
}
|
|
if u.CostUSD != nil || u.OutputTokensPerSecond != nil || u.RateBasis != "" {
|
|
t.Fatalf("unreported metrics invented: %+v", u)
|
|
}
|
|
}
|
|
if len(other) != 0 || c.turns["thread"] != "turn_new" || !c.starting["thread"] {
|
|
t.Fatal("usage leaked across threads or changed busy state")
|
|
}
|
|
c.handleNotification(rpcMessage{Method: "thread/tokenUsage/updated", Params: json.RawMessage(`{"threadId":"other","tokenUsage":{"total":{"totalTokens":42}}}`)})
|
|
if len(ch) != 0 || (<-other).Usage.TotalTokens == nil {
|
|
t.Fatal("foreign session notification misrouted")
|
|
}
|
|
}
|
|
|
|
func TestUsageMissingZeroAndInvalid(t *testing.T) {
|
|
c := usageTestChat()
|
|
ch := c.subscribe("thread")
|
|
for _, raw := range []string{
|
|
`{}`, `{"threadId":"thread"}`, `{"threadId":"thread","tokenUsage":null}`,
|
|
`{"tokenUsage":{}}`, `{"threadId":"thread","tokenUsage":{"total":{"totalTokens":"bad"}}}`,
|
|
} {
|
|
c.handleNotification(rpcMessage{Method: "thread/tokenUsage/updated", Params: json.RawMessage(raw)})
|
|
}
|
|
if len(ch) != 0 {
|
|
t.Fatal("missing or invalid source became fabricated usage")
|
|
}
|
|
c.handleNotification(usageFixture(t))
|
|
<-ch
|
|
// A full sparse snapshot clears prior values instead of leaving stale values.
|
|
c.receiveUsage(json.RawMessage(`{"threadId":"thread","tokenUsage":{"total":{"totalTokens":0,"inputTokens":0,"outputTokens":0},"last":{"totalTokens":0},"modelContextWindow":0}}`))
|
|
f := <-ch
|
|
if f.NativeTurnID != "" || f.Usage.TotalTokens == nil || *f.Usage.TotalTokens != 0 || f.Usage.ContextTokens == nil || *f.Usage.ContextTokens != 0 || f.Usage.ContextWindow == nil || *f.Usage.ContextWindow != 0 {
|
|
t.Fatalf("reported zero lost or turn invented: %+v", f)
|
|
}
|
|
c.receiveUsage(json.RawMessage(`{"threadId":"thread","tokenUsage":{"total":{"totalTokens":-1},"last":{},"modelContextWindow":null}}`))
|
|
u := (<-ch).Usage
|
|
if u.TotalTokens != nil || u.ContextTokens != nil || u.ContextWindow != nil || u.CostUSD != nil || u.OutputTokensPerSecond != nil {
|
|
t.Fatalf("unavailable values became zero or stale: %+v", u)
|
|
}
|
|
}
|
|
|
|
func TestUsageStaleTurnCannotRegressOrFreeBusySlot(t *testing.T) {
|
|
c := usageTestChat()
|
|
ch := c.subscribe("thread")
|
|
c.handleNotification(usageFixture(t))
|
|
<-ch
|
|
c.receiveUsage(json.RawMessage(`{"threadId":"thread","turnId":"turn_old","tokenUsage":{"total":{"totalTokens":10}}}`))
|
|
if len(ch) != 0 || *c.usage["thread"].frame.Usage.TotalTokens != 12000 || c.turns["thread"] != "turn_new" {
|
|
t.Fatal("stale native turn changed snapshot or busy state")
|
|
}
|
|
c.handleNotification(rpcMessage{Method: "turn/completed", Params: json.RawMessage(`{"threadId":"thread","turn":{"id":"turn_new","status":"completed","startedAt":100,"completedAt":110,"durationMs":10000}}`)})
|
|
<-ch
|
|
// Even with native timing, neither session output nor last model-call
|
|
// output is a compatible whole-turn numerator. Do not fabricate a rate.
|
|
u := c.usage["thread"].frame.Usage
|
|
if u.OutputTokensPerSecond != nil || u.RateBasis != "" {
|
|
t.Fatal("duration paired with incompatible output counter")
|
|
}
|
|
c.receiveUsage(json.RawMessage(`{"threadId":"thread","turnId":"turn_old","tokenUsage":{"total":{"totalTokens":10}}}`))
|
|
if len(ch) != 0 {
|
|
t.Fatal("old turn regressed completed turn usage")
|
|
}
|
|
}
|
|
|
|
func TestUsageReattachRetainedWithoutExtraReadback(t *testing.T) {
|
|
reads := 0
|
|
c := modelTestChat(t, func(method string, _ json.RawMessage) any {
|
|
if method != "thread/turns/list" {
|
|
t.Errorf("metrics added unexpected RPC: %s", method)
|
|
}
|
|
reads++
|
|
return json.RawMessage(`{"data":[]}`)
|
|
})
|
|
c.handleNotification(usageFixture(t))
|
|
stop := errors.New("snapshot received")
|
|
for range 2 {
|
|
err := c.Stream(context.Background(), "thread", nil, func(f protocol.ChatEventFrame) error {
|
|
if f.Usage == nil || *f.Usage.TotalTokens != 12000 || *f.Usage.ContextTokens != 3500 {
|
|
t.Fatalf("reattach lost snapshot: %+v", f)
|
|
}
|
|
return stop
|
|
})
|
|
if !errors.Is(err, stop) {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if reads != 2 {
|
|
t.Fatalf("metrics added reads: %d", reads)
|
|
}
|
|
c.ReleaseShells("thread")
|
|
ch := c.subscribe("thread")
|
|
if len(ch) != 0 || len(c.usage) != 0 {
|
|
t.Fatal("released room retained usage")
|
|
}
|
|
}
|
|
|
|
func TestUsageRetentionBoundAndTransportLoss(t *testing.T) {
|
|
c := usageTestChat()
|
|
for i := range retainedUsageLimit + 10 {
|
|
c.receiveUsage(json.RawMessage(fmt.Sprintf(`{"threadId":"thread_%d","tokenUsage":{"total":{"totalTokens":%d}}}`, i, i)))
|
|
}
|
|
if len(c.usage) != retainedUsageLimit {
|
|
t.Fatalf("unbounded usage cache: %d", len(c.usage))
|
|
}
|
|
if _, ok := c.usage["thread_0"]; ok {
|
|
t.Fatal("oldest usage was not evicted")
|
|
}
|
|
c.failPending(errors.New("transport lost"))
|
|
if len(c.usage) != 0 {
|
|
t.Fatal("transport ownership retained usage")
|
|
}
|
|
c.handleNotification(usageFixture(t))
|
|
if len(c.usage) != 0 {
|
|
t.Fatal("lost source regained retained usage")
|
|
}
|
|
}
|