package codex import ( "context" "encoding/json" "errors" "fmt" "os" "testing" "git.a-wan.net/alan/pipelines/internal/protocol" ) // Synthetic values in the current upstream typed notification shape; no live // Codex process, provider, configuration, or session is used by these tests. func usageFixture(t *testing.T) rpcMessage { t.Helper() raw, err := os.ReadFile("testdata/token-usage-updated.json") if err != nil { t.Fatal(err) } var msg rpcMessage if err := json.Unmarshal(raw, &msg); err != nil { t.Fatal(err) } return msg } func usageTestChat() *Chat { return &Chat{subs: map[string]map[chan protocol.ChatEventFrame]struct{}{}, turns: map[string]string{"thread": "turn_new"}, starting: map[string]bool{"thread": true}} } func TestUsageNativeSnapshotAndDuplicate(t *testing.T) { c := usageTestChat() ch := c.subscribe("thread") other := c.subscribe("other") for range 2 { c.handleNotification(usageFixture(t)) f := <-ch u := f.Usage if f.Kind != protocol.ChatEventUsage || f.SessionID != "thread" || f.NativeTurnID != "turn_new" || f.MessageID != "" || f.Part != nil || f.Step != nil { t.Fatalf("usage is not an item/step: %+v", f) } if u.Scope != "session" || *u.TotalTokens != 12000 || *u.ContextTokens != 3500 || *u.ContextWindow != 258400 || u.InputTokens != 10000 || u.OutputTokens != 2000 { t.Fatalf("snapshot added subsets, cumulative context, or duplicate: %+v", u) } if u.CostUSD != nil || u.OutputTokensPerSecond != nil || u.RateBasis != "" { t.Fatalf("unreported metrics invented: %+v", u) } } if len(other) != 0 || c.turns["thread"] != "turn_new" || !c.starting["thread"] { t.Fatal("usage leaked across threads or changed busy state") } c.handleNotification(rpcMessage{Method: "thread/tokenUsage/updated", Params: json.RawMessage(`{"threadId":"other","tokenUsage":{"total":{"totalTokens":42}}}`)}) if len(ch) != 0 || (<-other).Usage.TotalTokens == nil { t.Fatal("foreign session notification misrouted") } } func TestUsageMissingZeroAndInvalid(t *testing.T) { c := usageTestChat() ch := c.subscribe("thread") for _, raw := range []string{ `{}`, `{"threadId":"thread"}`, `{"threadId":"thread","tokenUsage":null}`, `{"tokenUsage":{}}`, `{"threadId":"thread","tokenUsage":{"total":{"totalTokens":"bad"}}}`, } { c.handleNotification(rpcMessage{Method: "thread/tokenUsage/updated", Params: json.RawMessage(raw)}) } if len(ch) != 0 { t.Fatal("missing or invalid source became fabricated usage") } c.handleNotification(usageFixture(t)) <-ch // A full sparse snapshot clears prior values instead of leaving stale values. c.receiveUsage(json.RawMessage(`{"threadId":"thread","tokenUsage":{"total":{"totalTokens":0,"inputTokens":0,"outputTokens":0},"last":{"totalTokens":0},"modelContextWindow":0}}`)) f := <-ch if f.NativeTurnID != "" || f.Usage.TotalTokens == nil || *f.Usage.TotalTokens != 0 || f.Usage.ContextTokens == nil || *f.Usage.ContextTokens != 0 || f.Usage.ContextWindow == nil || *f.Usage.ContextWindow != 0 { t.Fatalf("reported zero lost or turn invented: %+v", f) } c.receiveUsage(json.RawMessage(`{"threadId":"thread","tokenUsage":{"total":{"totalTokens":-1},"last":{},"modelContextWindow":null}}`)) u := (<-ch).Usage if u.TotalTokens != nil || u.ContextTokens != nil || u.ContextWindow != nil || u.CostUSD != nil || u.OutputTokensPerSecond != nil { t.Fatalf("unavailable values became zero or stale: %+v", u) } } func TestUsageStaleTurnCannotRegressOrFreeBusySlot(t *testing.T) { c := usageTestChat() ch := c.subscribe("thread") c.handleNotification(usageFixture(t)) <-ch c.receiveUsage(json.RawMessage(`{"threadId":"thread","turnId":"turn_old","tokenUsage":{"total":{"totalTokens":10}}}`)) if len(ch) != 0 || *c.usage["thread"].frame.Usage.TotalTokens != 12000 || c.turns["thread"] != "turn_new" { t.Fatal("stale native turn changed snapshot or busy state") } c.handleNotification(rpcMessage{Method: "turn/completed", Params: json.RawMessage(`{"threadId":"thread","turn":{"id":"turn_new","status":"completed","startedAt":100,"completedAt":110,"durationMs":10000}}`)}) <-ch // Even with native timing, neither session output nor last model-call // output is a compatible whole-turn numerator. Do not fabricate a rate. u := c.usage["thread"].frame.Usage if u.OutputTokensPerSecond != nil || u.RateBasis != "" { t.Fatal("duration paired with incompatible output counter") } c.receiveUsage(json.RawMessage(`{"threadId":"thread","turnId":"turn_old","tokenUsage":{"total":{"totalTokens":10}}}`)) if len(ch) != 0 { t.Fatal("old turn regressed completed turn usage") } } func TestUsageReattachRetainedWithoutExtraReadback(t *testing.T) { reads := 0 c := modelTestChat(t, func(method string, _ json.RawMessage) any { if method != "thread/turns/list" { t.Errorf("metrics added unexpected RPC: %s", method) } reads++ return json.RawMessage(`{"data":[]}`) }) c.handleNotification(usageFixture(t)) stop := errors.New("snapshot received") for range 2 { err := c.Stream(context.Background(), "thread", nil, func(f protocol.ChatEventFrame) error { if f.Usage == nil || *f.Usage.TotalTokens != 12000 || *f.Usage.ContextTokens != 3500 { t.Fatalf("reattach lost snapshot: %+v", f) } return stop }) if !errors.Is(err, stop) { t.Fatal(err) } } if reads != 2 { t.Fatalf("metrics added reads: %d", reads) } c.ReleaseShells("thread") ch := c.subscribe("thread") if len(ch) != 0 || len(c.usage) != 0 { t.Fatal("released room retained usage") } } func TestUsageRetentionBoundAndTransportLoss(t *testing.T) { c := usageTestChat() for i := range retainedUsageLimit + 10 { c.receiveUsage(json.RawMessage(fmt.Sprintf(`{"threadId":"thread_%d","tokenUsage":{"total":{"totalTokens":%d}}}`, i, i))) } if len(c.usage) != retainedUsageLimit { t.Fatalf("unbounded usage cache: %d", len(c.usage)) } if _, ok := c.usage["thread_0"]; ok { t.Fatal("oldest usage was not evicted") } c.failPending(errors.New("transport lost")) if len(c.usage) != 0 { t.Fatal("transport ownership retained usage") } c.handleNotification(usageFixture(t)) if len(c.usage) != 0 { t.Fatal("lost source regained retained usage") } }