Expose native OpenCode chat usage snapshots and replay session metrics

Read V2 session accounting, latest assistant prompt/cache components and native request-to-stream boundary. Resolve capacity by location/provider/catalog id with bounded metadata caching. Coalesce active/final readbacks; retain message-less session snapshots without changing admission or compaction state.

Provenance: https://opencode.ai/v2/openapi.json and anomalyco/opencode V2 4617210822bdbb31e5749751afbfb4584ccedac8: packages/schema/src/token-usage.ts, session-event.ts; packages/core/src/session/usage.ts, projector.ts, message-updater.ts, runner/step.ts. Native output is visible output, reasoning separate. Session totals include all five non-overlapping components. Generation rate uses created-to-streamed, not tool-inclusive completion.

Limitations: metrics remain unknown for missing components, absent timing/model metadata, reads beyond bounded latest-message/catalog budgets, or context preceding compaction. No local pricing, token estimation or default capacity. V1 gets no invented cumulative metrics; older V2 events remain available when readback is unavailable. No deployment or protocol/web/docs changes.

Checks: go test ./...; focused usage/stream race tests; go vet ./internal/opencode ./internal/server; go build ./...; git diff --check.
This commit is contained in:
2026-10-21 00:12:25 +01:00
parent 83b60990d2
commit ce3482b057
10 changed files with 733 additions and 15 deletions
+112 -10
View File
@@ -42,7 +42,9 @@ type Chat struct {
// operation and status on the shared connection, so two calls must not
// overlap; the permission poller runs alongside the read loop's
// prompt/interrupt/reply calls.
mu sync.Mutex
mu sync.Mutex
usageMu sync.Mutex
usageModels map[string]usageModelCache
}
// NewChat brokers chat for one owned OpenCode endpoint. The connection must be
@@ -757,6 +759,8 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
ctx, cancel := context.WithCancel(ctx)
defer cancel()
var delivery sync.Mutex
usageReadbackAvailable := false // protected by delivery, fallback for older API builds
var pendingUsage *protocol.ChatUsage // latest cumulative event, never a sum
state.mu.Lock()
busy := state.Active
state.mu.Unlock()
@@ -764,12 +768,28 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
busy = value
state.mu.Lock()
state.Active = value
state.usageDirty = true
if !value {
state.Compaction = nil
}
state.mu.Unlock()
}
deliver := func(frame protocol.ChatEventFrame) error {
// Native usage is cumulative but lacks context/model/timing. Coalesce it
// into one complete readback; never erase richer metrics with a partial
// usage event or issue HTTP reads for every token delta.
if frame.Kind == protocol.ChatEventUsage {
state.mu.Lock()
state.usageDirty = true
state.mu.Unlock()
delivery.Lock()
defer delivery.Unlock()
if usageReadbackAvailable {
pendingUsage = frame.Usage
return nil
}
return emit(frame)
}
delivery.Lock()
defer delivery.Unlock()
if frame.Kind == protocol.ChatEventStep || frame.Kind == protocol.ChatEventText || frame.Kind == protocol.ChatEventTool || frame.Kind == protocol.ChatEventReasoning || (frame.Compaction != nil && frame.Compaction.Status == "running") {
@@ -829,10 +849,81 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
}
}()
defer func() { cancel(); <-watchDone }()
usageRefreshGate := make(chan struct{}, 1)
refreshUsage := func() error {
if c.conn.Backend == "v1" {
return nil
}
select {
case usageRefreshGate <- struct{}{}:
defer func() { <-usageRefreshGate }()
default:
return nil
}
readCtx, readCancel := context.WithTimeout(ctx, 5*time.Second)
usage, err := c.Usage(readCtx, id)
readCancel()
if err != nil {
// A failed enrichment read must not lose authoritative native totals.
delivery.Lock()
defer delivery.Unlock()
usageReadbackAvailable = false
if pendingUsage != nil {
usage := pendingUsage
pendingUsage = nil
return emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventUsage, Usage: usage})
}
return nil // unavailable metrics must not break chat
}
delivery.Lock()
defer delivery.Unlock()
usageReadbackAvailable = true
pendingUsage = nil
return emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventUsage, Usage: usage})
}
if err := refreshUsage(); err != nil {
return err
}
usageDone := make(chan struct{})
go func() {
defer close(usageDone)
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
}
state.mu.Lock()
refresh := state.usageDirty || state.Active
state.usageDirty = false
state.mu.Unlock()
if refresh {
if err := refreshUsage(); err != nil {
cancel()
_ = body.Close()
return
}
}
}
}()
defer func() { cancel(); <-usageDone }()
if ready != nil {
ready()
}
return c.scanChatEventsWithState(ctx, body, id, state, deliver)
err = c.scanChatEventsWithState(ctx, body, id, state, deliver)
if ctx.Err() == nil {
state.mu.Lock()
dirty := state.usageDirty
state.mu.Unlock()
if dirty {
if refreshErr := refreshUsage(); refreshErr != nil {
return refreshErr
}
}
}
return err
}
// scanChatEvents turns an SSE body into protocol frames, one bounded frame at a
@@ -859,6 +950,20 @@ func (c *Chat) scanChatEventsWithState(ctx context.Context, body io.Reader, sess
if len(payload) == 0 {
continue
}
var usageTrigger struct {
Type string `json:"type"`
Data struct {
SessionID string `json:"sessionID"`
} `json:"data"`
}
if json.Unmarshal(payload, &usageTrigger) == nil && usageTrigger.Data.SessionID == sessionID {
switch usageTrigger.Type {
case "session.usage.updated", "session.step.ended", "session.step.failed", "session.compaction.ended", "session.compaction.failed", "session.model.selected", "session.moved", "session.execution.succeeded", "session.execution.failed", "session.execution.interrupted":
state.mu.Lock()
state.usageDirty = true
state.mu.Unlock()
}
}
frame, ok, err := NormalizeChatEvent(payload, sessionID, state)
if err != nil {
return err
@@ -1056,6 +1161,7 @@ type ChatStreamState struct {
Forms map[string]string
readGate chan struct{}
readback *ShellSnapshot
usageDirty bool
}
func (s *ChatStreamState) lastMessageID() string {
@@ -1206,9 +1312,9 @@ func NormalizeChatEvent(payload []byte, sessionID string, stream *ChatStreamStat
if messageID == "" && stream != nil {
messageID = stream.lastMessageID()
}
// Every pushed frame must carry a stable message id so the client can
// attribute it; without one this is not a chat frame.
if messageID == "" {
// Usage is session state; native V2 UsageUpdated has no message id. Other
// message frames still require a stable id for transcript attribution.
if messageID == "" && outer.Type != "session.usage.updated" {
return protocol.ChatEventFrame{}, false, nil
}
frame := protocol.ChatEventFrame{SessionID: sessionID, MessageID: messageID}
@@ -1234,11 +1340,7 @@ func NormalizeChatEvent(payload []byte, sessionID string, stream *ChatStreamStat
frame.Part = &protocol.ChatPart{Type: protocol.ChatPartTool, ID: data.ID, Name: data.Name, State: state}
case outer.Type == "session.usage.updated":
frame.Kind = protocol.ChatEventUsage
input, output := data.InputTokens, data.OutputTokens
if data.Tokens.Input != 0 || data.Tokens.Output != 0 {
input, output = data.Tokens.Input, data.Tokens.Output
}
frame.Usage = &protocol.ChatUsage{InputTokens: input, OutputTokens: output, Cost: data.Cost}
frame.Usage = decodeUsageSnapshot(outer.Data)
default:
return protocol.ChatEventFrame{}, false, nil
}
+9 -3
View File
@@ -7,6 +7,7 @@ import (
"io"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"sync"
"sync/atomic"
@@ -37,7 +38,7 @@ func TestNormalizeChatEvent(t *testing.T) {
{name: "tool running", payload: `{"type":"session.tool.called","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","name":"shell","input":{"command":"ls"}}}`, wantOK: true, wantKind: protocol.ChatEventTool, wantStatus: protocol.ChatToolRunning},
{name: "tool completed", payload: `{"type":"session.tool.success","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","name":"shell","content":[{"type":"text","text":"ok"}]}}`, wantOK: true, wantKind: protocol.ChatEventTool, wantStatus: protocol.ChatToolCompleted},
{name: "tool failed", payload: `{"type":"session.tool.failed","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","id":"call_1","name":"shell","error":{"type":"tool","message":"boom"}}}`, wantOK: true, wantKind: protocol.ChatEventTool, wantStatus: protocol.ChatToolError},
{name: "usage", payload: `{"type":"session.usage.updated","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","tokens":{"input":12,"output":34},"cost":0.001}}`, wantOK: true, wantKind: protocol.ChatEventUsage, wantUsage: &protocol.ChatUsage{InputTokens: 12, OutputTokens: 34, Cost: 0.001}},
{name: "usage", payload: `{"type":"session.usage.updated","data":{"sessionID":"ses_1","assistantMessageID":"msg_1","tokens":{"input":12,"output":34},"cost":0.001}}`, wantOK: true, wantKind: protocol.ChatEventUsage, wantUsage: &protocol.ChatUsage{InputTokens: 12, OutputTokens: 34, Cost: 0.001, Scope: "session", CostUSD: usageFloat(.001)}},
{name: "message id fallback", payload: `{"type":"session.step.started","data":{"sessionID":"ses_1","index":1}}`, state: prior, wantOK: true, wantKind: protocol.ChatEventStep, wantMsg: "msg_prev", wantStep: protocol.ChatStepStarted},
{name: "foreign session skipped", payload: `{"type":"session.text.delta","data":{"sessionID":"ses_other","delta":"x"}}`, wantOK: false},
{name: "unknown type skipped", payload: `{"type":"session.created","data":{"sessionID":"ses_1"}}`, wantOK: false},
@@ -69,7 +70,7 @@ func TestNormalizeChatEvent(t *testing.T) {
if tc.wantStep != "" && (frame.Step == nil || frame.Step.Status != tc.wantStep) {
t.Fatalf("step = %+v, want %q", frame.Step, tc.wantStep)
}
if tc.wantUsage != nil && (frame.Usage == nil || *frame.Usage != *tc.wantUsage) {
if tc.wantUsage != nil && !reflect.DeepEqual(frame.Usage, tc.wantUsage) {
t.Fatalf("usage = %+v, want %+v", frame.Usage, tc.wantUsage)
}
})
@@ -99,6 +100,8 @@ func TestChatStreamFiltersFrames(t *testing.T) {
case r.Method == http.MethodGet && r.URL.Path == "/api/session/ses_1/message":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":[]}`))
case r.Method == http.MethodGet && r.URL.Path == "/api/session/ses_1":
w.WriteHeader(http.StatusNotFound) // older source flavor: events only
case r.Method == http.MethodGet && r.URL.Path == "/api/event":
w.Header().Set("Content-Type", "text/event-stream")
for _, frame := range frames {
@@ -784,12 +787,15 @@ func TestNormalizeChatEventWithoutMessageID(t *testing.T) {
for _, payload := range []string{
`{"type":"session.text.delta","data":{"sessionID":"ses_1","ordinal":0,"delta":"x"}}`,
`{"type":"session.tool.called","data":{"sessionID":"ses_1","id":"call_1","name":"shell","input":{"command":"ls"}}}`,
`{"type":"session.usage.updated","data":{"sessionID":"ses_1","tokens":{"input":1,"output":2}}}`,
} {
if _, ok, err := NormalizeChatEvent([]byte(payload), "ses_1", nil); err != nil || ok {
t.Fatalf("event without a message id produced a frame: ok=%v err=%v", ok, err)
}
}
frame, ok, err := NormalizeChatEvent([]byte(`{"type":"session.usage.updated","data":{"sessionID":"ses_1","tokens":{"input":1,"output":2}}}`), "ses_1", nil)
if err != nil || !ok || frame.MessageID != "" || frame.Usage == nil || frame.Usage.Scope != "session" {
t.Fatalf("message-less session usage=%+v ok=%v err=%v", frame, ok, err)
}
}
// TestChatV1ConnectionDrivesThroughCall drives the adapter through a real
+219
View File
@@ -0,0 +1,219 @@
package opencode
import (
"context"
"encoding/json"
"fmt"
"io"
"math"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// Provenance: OpenCode V2 4617210822bdbb31e5749751afbfb4584ccedac8,
// packages/schema/src/token-usage.ts, session-event.ts and
// packages/core/src/session/{usage,projector,message-updater}.ts, runner/step.ts.
// UsageUpdated is a cumulative projection, not a delta. Native output is visible
// output (reasoning is separate); input excludes cache. Never price locally.
// message-updater sets created from Step.Started.started (request dispatch),
// streamed at Step.Streamed (provider body exhausted), completed after tools.
// Thus created->streamed is generation incl. provider latency, NOT tool time.
type usageTokens struct {
Input *int64 `json:"input"`
Output *int64 `json:"output"`
Reasoning *int64 `json:"reasoning"`
Cache struct {
Read *int64 `json:"read"`
Write *int64 `json:"write"`
} `json:"cache"`
}
func tokenSum(values ...*int64) *int64 {
var total int64
for _, value := range values {
if value == nil || *value < 0 || *value > math.MaxInt64-total {
return nil
}
total += *value
}
return &total
}
func decodeUsageSnapshot(data []byte) *protocol.ChatUsage {
var record struct {
Tokens usageTokens `json:"tokens"`
Cost *float64 `json:"cost"`
Input int `json:"inputTokens"`
Output int `json:"outputTokens"`
}
if json.Unmarshal(data, &record) != nil {
return &protocol.ChatUsage{Scope: "session"}
}
u := &protocol.ChatUsage{Scope: "session", InputTokens: record.Input, OutputTokens: record.Output}
if record.Tokens.Input != nil {
u.InputTokens = int(*record.Tokens.Input)
}
if record.Tokens.Output != nil {
u.OutputTokens = int(*record.Tokens.Output)
}
u.TotalTokens = tokenSum(record.Tokens.Input, record.Tokens.Output, record.Tokens.Reasoning, record.Tokens.Cache.Read, record.Tokens.Cache.Write)
if record.Cost != nil && *record.Cost >= 0 && !math.IsNaN(*record.Cost) && !math.IsInf(*record.Cost, 0) {
u.CostUSD = record.Cost
u.Cost = *record.Cost
}
return u
}
type usageModelCache struct {
limit *int64
at time.Time
}
type usageMessage struct {
Type string `json:"type"`
Tokens usageTokens `json:"tokens"`
Time struct {
Created *float64 `json:"created"`
Streamed *float64 `json:"streamed"`
Completed *float64 `json:"completed"`
} `json:"time"`
}
// Usage reads at most one session, one latest assistant and one latest compaction.
// No unbounded history, CLI export, model/provider creation or inference. Missing
// fields remain unknown. After compaction an old prompt's usage is not current.
func (c *Chat) Usage(ctx context.Context, id string) (*protocol.ChatUsage, error) {
if !ValidSessionID(id) {
return nil, fmt.Errorf("invalid OpenCode session id")
}
if c.conn.Backend == "v1" {
return &protocol.ChatUsage{Scope: "session"}, nil
}
data, err := c.call(ctx, "get", "/api/session/"+id, nil)
if err != nil {
return nil, err
}
var session struct {
Data json.RawMessage `json:"data"`
}
if json.Unmarshal(data, &session) != nil || len(session.Data) == 0 {
return nil, fmt.Errorf("invalid OpenCode usage session")
}
u := decodeUsageSnapshot(session.Data)
var info struct {
ID string `json:"id"`
Model struct {
ID string `json:"id"`
Provider string `json:"providerID"`
} `json:"model"`
Location struct {
Directory string `json:"directory"`
} `json:"location"`
}
if json.Unmarshal(session.Data, &info) != nil || info.ID != id {
return nil, fmt.Errorf("invalid OpenCode usage session")
}
u.ContextWindow = c.usageModelLimit(ctx, info.Location.Directory, info.Model.Provider, info.Model.ID)
latest := func(kind string) (*usageMessage, bool) {
body, e := c.usageRead(ctx, "/api/session/"+id+"/message?type="+kind+"&order=desc&limit=1", c.conn.Directory, 4<<20)
if e != nil {
return nil, false
}
var page struct {
Data []usageMessage `json:"data"`
}
if json.Unmarshal(body, &page) != nil || page.Data == nil || len(page.Data) > 1 {
return nil, false
}
if len(page.Data) == 0 {
return nil, true
}
if page.Data[0].Type != kind {
return nil, false
}
return &page.Data[0], true
}
a, _ := latest("assistant")
if a == nil {
return u, nil
}
compaction, boundaryKnown := latest("compaction")
// Failure to establish the compaction boundary must not revive old context.
// Empty compaction pages are valid, unlike failed/malformed reads.
if boundaryKnown && a.Time.Created != nil && (compaction == nil || (compaction.Time.Created != nil && *compaction.Time.Created < *a.Time.Created)) {
u.ContextTokens = tokenSum(a.Tokens.Input, a.Tokens.Cache.Read, a.Tokens.Cache.Write)
}
if a.Time.Completed != nil && a.Time.Created != nil && a.Time.Streamed != nil && *a.Time.Created >= 0 && *a.Time.Streamed > *a.Time.Created && *a.Time.Completed >= *a.Time.Streamed && a.Tokens.Output != nil && *a.Tokens.Output >= 0 {
rate := float64(*a.Tokens.Output) * 1000 / (*a.Time.Streamed - *a.Time.Created)
if !math.IsNaN(rate) && !math.IsInf(rate, 0) {
u.OutputTokensPerSecond = &rate
u.RateBasis = "generation"
}
}
return u, nil
}
// Reuses the authenticated, location-scoped transport, with an explicit bounded
// allowance for large model catalogs. It never changes the shared connection's
// directory or stage export/format pin. Metadata is cached by catalog id AND
// provider AND location, not the upstream modelID (aliases can differ).
func (c *Chat) usageRead(ctx context.Context, path, directory string, budget int64) ([]byte, error) {
c.mu.Lock()
conn := *c.conn
c.mu.Unlock()
conn.Directory = directory
body, err := conn.HistoryNative(ctx, path)
if err != nil {
return nil, err
}
defer body.Close()
data, err := io.ReadAll(io.LimitReader(body, budget+1))
if err != nil {
return nil, err
}
if int64(len(data)) > budget {
return nil, fmt.Errorf("OpenCode metrics read exceeds budget")
}
return data, nil
}
func (c *Chat) usageModelLimit(ctx context.Context, directory, provider, id string) *int64 {
if directory == "" || provider == "" || id == "" {
return nil
}
key := directory + "\x00" + provider + "\x00" + id
c.usageMu.Lock()
cached, ok := c.usageModels[key]
c.usageMu.Unlock()
if ok && time.Since(cached.at) < 5*time.Minute {
return cached.limit
}
var limit *int64
data, err := c.usageRead(ctx, "/api/model", directory, 16<<20)
if err == nil {
var catalog struct {
Data []struct {
ID string `json:"id"`
Provider string `json:"providerID"`
Limit struct {
Context *int64 `json:"context"`
} `json:"limit"`
} `json:"data"`
}
if json.Unmarshal(data, &catalog) == nil {
for _, m := range catalog.Data {
if m.ID == id && m.Provider == provider && m.Limit.Context != nil && *m.Limit.Context >= 0 {
limit = m.Limit.Context
break
}
}
}
}
c.usageMu.Lock()
if c.usageModels == nil || len(c.usageModels) >= 128 {
c.usageModels = make(map[string]usageModelCache)
}
c.usageModels[key] = usageModelCache{limit: limit, at: time.Now()}
c.usageMu.Unlock()
return limit
}
+272
View File
@@ -0,0 +1,272 @@
package opencode
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
const usageFixtureTokens = `"tokens":{"input":100,"output":20,"reasoning":5,"cache":{"read":50,"write":10}}`
func TestUsageNativeSnapshots(t *testing.T) {
for _, tc := range []struct {
name, fields string
total *int64
cost *float64
}{
{"missing", ``, nil, nil},
{"zero", `"cost":0,"tokens":{"input":0,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}`, usageInt(0), usageFloat(0)},
{"partial", `"tokens":{"input":100,"output":20}`, nil, nil},
{"native", `"cost":0.42,` + usageFixtureTokens, usageInt(185), usageFloat(.42)},
} {
t.Run(tc.name, func(t *testing.T) {
payload := `{"type":"session.usage.updated","data":{"sessionID":"ses_usage"`
if tc.fields != "" {
payload += "," + tc.fields
}
payload += `}}`
for i := 0; i < 2; i++ {
frame, ok, err := NormalizeChatEvent([]byte(payload), "ses_usage", nil)
if err != nil || !ok || frame.MessageID != "" || frame.Usage.Scope != "session" {
t.Fatalf("frame=%+v ok=%v err=%v", frame, ok, err)
}
assertUsageInt(t, frame.Usage.TotalTokens, tc.total)
if (frame.Usage.CostUSD == nil) != (tc.cost == nil) || tc.cost != nil && *frame.Usage.CostUSD != *tc.cost {
t.Fatalf("cost=%v", frame.Usage.CostUSD)
}
}
})
}
}
func usageInt(n int64) *int64 { return &n }
func usageFloat(n float64) *float64 { return &n }
func assertUsageInt(t *testing.T, got, want *int64) {
t.Helper()
if (got == nil) != (want == nil) || got != nil && *got != *want {
t.Fatalf("got %v want %v", got, want)
}
}
func TestUsageReadbackNativeProvenanceAndModelCache(t *testing.T) {
model := "alias-a"
compaction := false
missingCompaction := false
modelReads := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.Method != "GET" {
t.Errorf("unexpected write %s", r.Method)
w.WriteHeader(405)
return
}
switch r.URL.Path {
case "/api/session/ses_usage":
fmt.Fprintf(w, `{"data":{"id":"ses_usage","model":{"id":%q,"providerID":"p"},"location":{"directory":"/actual"},"cost":0,`+usageFixtureTokens+`}}`, model)
case "/api/model":
modelReads++
if r.URL.Query().Get("directory") != "/actual" {
t.Errorf("wrong model location: %s", r.URL.RawQuery)
}
// Larger than Call's 1MiB guard; match catalog id not modelID or
// another provider's otherwise-identical id.
fmt.Fprintf(w, `{"padding":%q,"data":[{"id":"alias-a","modelID":"upstream","providerID":"wrong","limit":{"context":1}},{"id":"alias-a","modelID":"upstream","providerID":"p","limit":{"context":200000}},{"id":"alias-b","modelID":"upstream","providerID":"p","limit":{"context":300000}}]}`, strings.Repeat("x", MaxResponse+10))
case "/api/session/ses_usage/message":
if r.URL.Query().Get("limit") != "1" || r.URL.Query().Get("order") != "desc" {
t.Errorf("unbounded metrics history %s", r.URL.RawQuery)
}
if r.URL.Query().Get("type") == "assistant" {
fmt.Fprint(w, `{"data":[{"type":"assistant","time":{"created":1000,"streamed":3000,"completed":23000},`+usageFixtureTokens+`}]}`)
} else if missingCompaction {
w.WriteHeader(503)
} else if compaction {
fmt.Fprint(w, `{"data":[{"type":"compaction","time":{"created":4000}}]}`)
} else {
fmt.Fprint(w, `{"data":[]}`)
}
default:
t.Errorf("unexpected path %s", r.URL.Path)
w.WriteHeader(404)
}
}))
defer server.Close()
chat := NewChat(&Connection{Backend: "v2", URL: server.URL, Directory: "/other"})
for i := 0; i < 2; i++ {
u, err := chat.Usage(context.Background(), "ses_usage")
if err != nil {
t.Fatal(err)
}
assertUsageInt(t, u.TotalTokens, usageInt(185))
assertUsageInt(t, u.ContextTokens, usageInt(160))
assertUsageInt(t, u.ContextWindow, usageInt(200000))
if u.CostUSD == nil || *u.CostUSD != 0 || u.OutputTokensPerSecond == nil || *u.OutputTokensPerSecond != 10 || u.RateBasis != "generation" {
t.Fatalf("usage %+v", u)
}
}
if modelReads != 1 {
t.Fatalf("model reads=%d", modelReads)
}
model = "alias-b"
u, err := chat.Usage(context.Background(), "ses_usage")
if err != nil {
t.Fatal(err)
}
assertUsageInt(t, u.ContextWindow, usageInt(300000))
if modelReads != 2 {
t.Fatalf("model switch did not refresh: %d", modelReads)
}
compaction = true
u, err = chat.Usage(context.Background(), "ses_usage")
if err != nil {
t.Fatal(err)
}
assertUsageInt(t, u.ContextTokens, nil)
assertUsageInt(t, u.TotalTokens, usageInt(185))
compaction = false
missingCompaction = true
u, err = chat.Usage(context.Background(), "ses_usage")
if err != nil {
t.Fatal(err)
}
assertUsageInt(t, u.ContextTokens, nil)
}
func TestUsageReadBudgetAndUnknownTiming(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, strings.Repeat("x", 1025))
}))
defer server.Close()
c := NewChat(&Connection{Backend: "v2", URL: server.URL})
if _, err := c.usageRead(context.Background(), "/api/model", "/scope", 1024); err == nil {
t.Fatal("accepted over-budget response")
}
u, err := NewChat(&Connection{Backend: "v1"}).Usage(context.Background(), "ses_usage")
if err != nil || u.TotalTokens != nil || u.ContextTokens != nil || u.OutputTokensPerSecond != nil {
t.Fatalf("fabricated unsupported backend metrics: %+v %v", u, err)
}
}
func TestUsagePartialFieldsAndTimingNeverInvented(t *testing.T) {
for _, tc := range []struct {
name, tokens, timing, model string
context, limit *int64
rate *float64
}{
{"missing", ``, `"created":1000,"completed":5000`, ``, nil, nil, nil},
{"no-provider-boundary", usageFixtureTokens, `"created":1000,"completed":5000`, ``, usageInt(160), nil, nil},
{"no-completion", usageFixtureTokens, `"created":1000,"streamed":3000`, ``, usageInt(160), nil, nil},
{"reversed", usageFixtureTokens, `"created":3000,"streamed":1000,"completed":5000`, ``, usageInt(160), nil, nil},
{"completion-before-stream", usageFixtureTokens, `"created":1000,"streamed":5000,"completed":3000`, ``, usageInt(160), nil, nil},
{"zero-native-output", `"tokens":{"input":0,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}`, `"created":1000,"streamed":3000,"completed":5000`, `"model":{"id":"m","providerID":"p"},"location":{"directory":"/actual"},`, usageInt(0), usageInt(0), usageFloat(0)},
} {
t.Run(tc.name, func(t *testing.T) {
reads := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reads++
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/session/ses_usage":
fmt.Fprint(w, `{"data":{"id":"ses_usage",`+tc.model+`"cost":0}}`)
case "/api/model":
fmt.Fprint(w, `{"data":[{"id":"m","providerID":"p","limit":{"context":0}}]}`)
case "/api/session/ses_usage/message":
if r.URL.Query().Get("type") == "compaction" {
fmt.Fprint(w, `{"data":[]}`)
return
}
tokens := tc.tokens
if tokens != "" {
tokens = "," + tokens
}
fmt.Fprint(w, `{"data":[{"type":"assistant","time":{`+tc.timing+`}`+tokens+`}]}`)
default:
t.Errorf("unexpected request %s", r.URL.Path)
w.WriteHeader(404)
}
}))
defer server.Close()
chat := NewChat(&Connection{Backend: "v2", URL: server.URL})
u, err := chat.Usage(context.Background(), "ses_usage")
if err != nil {
t.Fatal(err)
}
assertUsageInt(t, u.ContextTokens, tc.context)
assertUsageInt(t, u.ContextWindow, tc.limit)
if (u.OutputTokensPerSecond == nil) != (tc.rate == nil) || tc.rate != nil && *u.OutputTokensPerSecond != *tc.rate {
t.Fatalf("rate=%v", u.OutputTokensPerSecond)
}
if u.OutputTokensPerSecond == nil && u.RateBasis != "" {
t.Fatal("invented rate basis")
}
if reads > 4 {
t.Fatalf("unbounded burst: %d", reads)
}
})
}
}
func TestUsageStreamIdleAttachAndSettlement(t *testing.T) {
var mu sync.Mutex
settled := false
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/event":
w.Header().Set("Content-Type", "text/event-stream")
w.WriteHeader(200)
w.(http.Flusher).Flush()
select {
case <-r.Context().Done():
return
case <-time.After(200 * time.Millisecond):
}
mu.Lock()
settled = true
mu.Unlock()
fmt.Fprint(w, "data: {\"type\":\"session.usage.updated\",\"data\":{\"sessionID\":\"ses_usage\",\"cost\":0}}\n\n")
w.(http.Flusher).Flush()
<-r.Context().Done()
case "/api/session/ses_usage":
mu.Lock()
done := settled
mu.Unlock()
n := 0
if done {
n = 1
}
fmt.Fprintf(w, `{"data":{"id":"ses_usage","cost":0,"tokens":{"input":%d,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}`, n)
case "/api/session/active":
fmt.Fprint(w, `{"data":{}}`)
case "/api/session/ses_usage/message":
fmt.Fprint(w, `{"data":[]}`)
default:
t.Errorf("unexpected %s", r.URL.Path)
w.WriteHeader(404)
}
}))
defer server.Close()
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
defer cancel()
var totals []int64
chat := NewChat(&Connection{Backend: "v2", URL: server.URL})
_ = chat.Stream(ctx, "ses_usage", nil, func(f protocol.ChatEventFrame) error {
if f.Kind == protocol.ChatEventUsage && f.Usage.TotalTokens != nil {
totals = append(totals, *f.Usage.TotalTokens)
if *f.Usage.TotalTokens == 1 {
cancel()
}
}
return nil
})
if len(totals) != 2 || totals[0] != 0 || totals[1] != 1 {
t.Fatalf("idle/settlement snapshots=%v", totals)
}
}
+4
View File
@@ -209,6 +209,10 @@ func TestChatStreamRealV2Shapes(t *testing.T) {
_, _ = w.Write([]byte(`{"data":[]}`))
return
}
if r.URL.Path == "/api/session/ses_1" {
w.WriteHeader(http.StatusNotFound)
return
}
if r.Method != http.MethodGet || r.URL.Path != "/api/event" {
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
return
+47 -1
View File
@@ -162,7 +162,7 @@ func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) {
turn.step = frame.Step
return
case protocol.ChatEventUsage:
turn.usage = frame.Usage
turn.usage = cloneChatUsage(frame.Usage)
return
case protocol.ChatEventShell:
// A shell frame carries no message id (it outlives the tool call that
@@ -276,6 +276,52 @@ func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) {
}
}
func cloneChatUsage(usage *protocol.ChatUsage) *protocol.ChatUsage {
if usage == nil {
return nil
}
copy := *usage
if usage.TotalTokens != nil {
value := *usage.TotalTokens
copy.TotalTokens = &value
}
if usage.ContextTokens != nil {
value := *usage.ContextTokens
copy.ContextTokens = &value
}
if usage.ContextWindow != nil {
value := *usage.ContextWindow
copy.ContextWindow = &value
}
if usage.CostUSD != nil {
value := *usage.CostUSD
copy.CostUSD = &value
}
if usage.OutputTokensPerSecond != nil {
value := *usage.OutputTokensPerSecond
copy.OutputTokensPerSecond = &value
}
return &copy
}
// Usage is session state, not an actor step. Replace even on native counter
// resets; never sum snapshots or derive admission/running state from counters.
// Enqueue replay under the fold lock, ordered with live usage (no I/O here).
func (h *chatHub) replayUsage(sub *chatSub) {
h.mu.Lock()
slow := false
if h.rooms[sub.room.sessionID] == sub.room && sub.room.subs[sub.clientID] == sub {
if turn := h.turns[sub.room.sessionID]; turn != nil && turn.usage != nil {
frame := protocol.ChatEventFrame{SessionID: sub.room.sessionID, Kind: protocol.ChatEventUsage, Usage: cloneChatUsage(turn.usage)}
slow = !enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame})
}
}
h.mu.Unlock()
if slow {
sub.close()
}
}
// applyShell folds one shell frame into the session's retained shells. A running
// shell is upserted (keyed by its id) so a client that reattaches can be
// re-synced the processes still running; an exited shell is dropped, which is
+1 -1
View File
@@ -425,7 +425,7 @@ func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) {
}
// Shell replay and live lifecycle delivery must enqueue in the same order
// as folding. Otherwise an exit can overtake an older running replay.
if (frame.Kind == protocol.ChatEventShell || frame.Kind == protocol.ChatEventCompaction) && !buffered {
if (frame.Kind == protocol.ChatEventShell || frame.Kind == protocol.ChatEventCompaction || frame.Kind == protocol.ChatEventUsage) && !buffered {
msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}
var slow []*chatSub
for _, sub := range targets {
+1
View File
@@ -244,4 +244,5 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH
// shell half of the resume.
hub.replayShells(sub)
hub.replayCompaction(sub)
hub.replayUsage(sub)
}
+4
View File
@@ -886,6 +886,8 @@ func TestChatInterruptEndsRunningPrompt(t *testing.T) {
case r.Method == http.MethodGet && r.URL.Path == "/api/session/ses_1/message":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":[]}`))
case r.Method == http.MethodGet && r.URL.Path == "/api/session/ses_1":
w.WriteHeader(http.StatusNotFound) // metrics unavailable on this source flavor
default:
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
}
@@ -1390,6 +1392,8 @@ func realAdapterChat(t *testing.T, frames []string) []protocol.ChatEventFrame {
case r.Method == http.MethodGet && (r.URL.Path == "/api/session/ses_1/permission" || r.URL.Path == "/api/session/ses_1/form" || r.URL.Path == "/api/session/ses_1/message"):
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`))
case r.Method == http.MethodGet && r.URL.Path == "/api/session/ses_1":
w.WriteHeader(http.StatusNotFound) // events-only source flavor
case r.Method == http.MethodGet && r.URL.Path == "/api/shell":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":[]}`))
+64
View File
@@ -0,0 +1,64 @@
package server
import (
"context"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestUsageRetainedWithoutMessageReplayResetAndAdmissionIsolation(t *testing.T) {
h := newChatHub()
room := newChatRoom(context.Background(), "ses_usage")
defer room.cancel()
room.running = true
room.nativeTurnID = "turn_keep"
room.turnDone = make(chan struct{})
h.rooms[room.sessionID] = room
sub := &chatSub{hub: h, room: room, clientID: "watcher", ch: make(chan protocol.Message, 32), done: make(chan struct{})}
room.subs[sub.clientID] = sub
h.apply(protocol.ChatEventFrame{SessionID: room.sessionID, Kind: protocol.ChatEventCompaction, Compaction: &protocol.ChatCompaction{Status: "running"}})
for _, n := range []int64{123, 123, 0, 5} {
total := n
u := &protocol.ChatUsage{Scope: "session", TotalTokens: &total}
h.emitRoom(room, protocol.ChatEventFrame{SessionID: room.sessionID, Kind: protocol.ChatEventUsage, Usage: u})
live := (<-sub.ch).ChatEvent
if live.MessageID != "" || *live.Usage.TotalTokens != n {
t.Fatalf("live=%+v", live)
}
total = 999 // producer-owned data must not mutate retained state
h.replayUsage(sub)
frame := (<-sub.ch).ChatEvent
if frame == nil || frame.Kind != protocol.ChatEventUsage || frame.MessageID != "" || frame.Usage.TotalTokens == nil || *frame.Usage.TotalTokens != n {
t.Fatalf("replay=%+v", frame)
}
*frame.Usage.TotalTokens = 888 // neither may a consumer
if !room.running || room.nativeTurnID != "turn_keep" || h.turns[room.sessionID].step != nil || h.turns[room.sessionID].compaction == nil {
t.Fatal("usage changed turn/admission/compaction")
}
select {
case <-room.turnDone:
t.Fatal("usage settled turn")
default:
}
}
if got := h.snapshot(room.sessionID); len(got) != 0 {
t.Fatalf("usage created transcript messages %+v", got)
}
// A full snapshot may legitimately reset every unknown field to nil.
h.emitRoom(room, protocol.ChatEventFrame{SessionID: room.sessionID, Kind: protocol.ChatEventUsage, Usage: &protocol.ChatUsage{Scope: "session"}})
<-sub.ch
h.replayUsage(sub)
if f := (<-sub.ch).ChatEvent; f.Usage.TotalTokens != nil {
t.Fatal("unknown snapshot accumulated old counter")
}
other := newChatRoom(context.Background(), "ses_other")
defer other.cancel()
h.rooms[other.sessionID] = other
otherSub := &chatSub{hub: h, room: other, clientID: "other", ch: make(chan protocol.Message, 4), done: make(chan struct{})}
other.subs[otherSub.clientID] = otherSub
h.replayUsage(otherSub)
if len(otherSub.ch) != 0 {
t.Fatal("foreign usage replayed")
}
}