ce3482b057
Read V2 session accounting, latest assistant prompt/cache components and native request-to-stream boundary. Resolve capacity by location/provider/catalog id with bounded metadata caching. Coalesce active/final readbacks; retain message-less session snapshots without changing admission or compaction state. Provenance: https://opencode.ai/v2/openapi.json and anomalyco/opencode V2 4617210822bdbb31e5749751afbfb4584ccedac8: packages/schema/src/token-usage.ts, session-event.ts; packages/core/src/session/usage.ts, projector.ts, message-updater.ts, runner/step.ts. Native output is visible output, reasoning separate. Session totals include all five non-overlapping components. Generation rate uses created-to-streamed, not tool-inclusive completion. Limitations: metrics remain unknown for missing components, absent timing/model metadata, reads beyond bounded latest-message/catalog budgets, or context preceding compaction. No local pricing, token estimation or default capacity. V1 gets no invented cumulative metrics; older V2 events remain available when readback is unavailable. No deployment or protocol/web/docs changes. Checks: go test ./...; focused usage/stream race tests; go vet ./internal/opencode ./internal/server; go build ./...; git diff --check.
273 lines
9.6 KiB
Go
273 lines
9.6 KiB
Go
package opencode
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
const usageFixtureTokens = `"tokens":{"input":100,"output":20,"reasoning":5,"cache":{"read":50,"write":10}}`
|
|
|
|
func TestUsageNativeSnapshots(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name, fields string
|
|
total *int64
|
|
cost *float64
|
|
}{
|
|
{"missing", ``, nil, nil},
|
|
{"zero", `"cost":0,"tokens":{"input":0,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}`, usageInt(0), usageFloat(0)},
|
|
{"partial", `"tokens":{"input":100,"output":20}`, nil, nil},
|
|
{"native", `"cost":0.42,` + usageFixtureTokens, usageInt(185), usageFloat(.42)},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
payload := `{"type":"session.usage.updated","data":{"sessionID":"ses_usage"`
|
|
if tc.fields != "" {
|
|
payload += "," + tc.fields
|
|
}
|
|
payload += `}}`
|
|
for i := 0; i < 2; i++ {
|
|
frame, ok, err := NormalizeChatEvent([]byte(payload), "ses_usage", nil)
|
|
if err != nil || !ok || frame.MessageID != "" || frame.Usage.Scope != "session" {
|
|
t.Fatalf("frame=%+v ok=%v err=%v", frame, ok, err)
|
|
}
|
|
assertUsageInt(t, frame.Usage.TotalTokens, tc.total)
|
|
if (frame.Usage.CostUSD == nil) != (tc.cost == nil) || tc.cost != nil && *frame.Usage.CostUSD != *tc.cost {
|
|
t.Fatalf("cost=%v", frame.Usage.CostUSD)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func usageInt(n int64) *int64 { return &n }
|
|
func usageFloat(n float64) *float64 { return &n }
|
|
func assertUsageInt(t *testing.T, got, want *int64) {
|
|
t.Helper()
|
|
if (got == nil) != (want == nil) || got != nil && *got != *want {
|
|
t.Fatalf("got %v want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestUsageReadbackNativeProvenanceAndModelCache(t *testing.T) {
|
|
model := "alias-a"
|
|
compaction := false
|
|
missingCompaction := false
|
|
modelReads := 0
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
if r.Method != "GET" {
|
|
t.Errorf("unexpected write %s", r.Method)
|
|
w.WriteHeader(405)
|
|
return
|
|
}
|
|
switch r.URL.Path {
|
|
case "/api/session/ses_usage":
|
|
fmt.Fprintf(w, `{"data":{"id":"ses_usage","model":{"id":%q,"providerID":"p"},"location":{"directory":"/actual"},"cost":0,`+usageFixtureTokens+`}}`, model)
|
|
case "/api/model":
|
|
modelReads++
|
|
if r.URL.Query().Get("directory") != "/actual" {
|
|
t.Errorf("wrong model location: %s", r.URL.RawQuery)
|
|
}
|
|
// Larger than Call's 1MiB guard; match catalog id not modelID or
|
|
// another provider's otherwise-identical id.
|
|
fmt.Fprintf(w, `{"padding":%q,"data":[{"id":"alias-a","modelID":"upstream","providerID":"wrong","limit":{"context":1}},{"id":"alias-a","modelID":"upstream","providerID":"p","limit":{"context":200000}},{"id":"alias-b","modelID":"upstream","providerID":"p","limit":{"context":300000}}]}`, strings.Repeat("x", MaxResponse+10))
|
|
case "/api/session/ses_usage/message":
|
|
if r.URL.Query().Get("limit") != "1" || r.URL.Query().Get("order") != "desc" {
|
|
t.Errorf("unbounded metrics history %s", r.URL.RawQuery)
|
|
}
|
|
if r.URL.Query().Get("type") == "assistant" {
|
|
fmt.Fprint(w, `{"data":[{"type":"assistant","time":{"created":1000,"streamed":3000,"completed":23000},`+usageFixtureTokens+`}]}`)
|
|
} else if missingCompaction {
|
|
w.WriteHeader(503)
|
|
} else if compaction {
|
|
fmt.Fprint(w, `{"data":[{"type":"compaction","time":{"created":4000}}]}`)
|
|
} else {
|
|
fmt.Fprint(w, `{"data":[]}`)
|
|
}
|
|
default:
|
|
t.Errorf("unexpected path %s", r.URL.Path)
|
|
w.WriteHeader(404)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
chat := NewChat(&Connection{Backend: "v2", URL: server.URL, Directory: "/other"})
|
|
for i := 0; i < 2; i++ {
|
|
u, err := chat.Usage(context.Background(), "ses_usage")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assertUsageInt(t, u.TotalTokens, usageInt(185))
|
|
assertUsageInt(t, u.ContextTokens, usageInt(160))
|
|
assertUsageInt(t, u.ContextWindow, usageInt(200000))
|
|
if u.CostUSD == nil || *u.CostUSD != 0 || u.OutputTokensPerSecond == nil || *u.OutputTokensPerSecond != 10 || u.RateBasis != "generation" {
|
|
t.Fatalf("usage %+v", u)
|
|
}
|
|
}
|
|
if modelReads != 1 {
|
|
t.Fatalf("model reads=%d", modelReads)
|
|
}
|
|
model = "alias-b"
|
|
u, err := chat.Usage(context.Background(), "ses_usage")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assertUsageInt(t, u.ContextWindow, usageInt(300000))
|
|
if modelReads != 2 {
|
|
t.Fatalf("model switch did not refresh: %d", modelReads)
|
|
}
|
|
compaction = true
|
|
u, err = chat.Usage(context.Background(), "ses_usage")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assertUsageInt(t, u.ContextTokens, nil)
|
|
assertUsageInt(t, u.TotalTokens, usageInt(185))
|
|
compaction = false
|
|
missingCompaction = true
|
|
u, err = chat.Usage(context.Background(), "ses_usage")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assertUsageInt(t, u.ContextTokens, nil)
|
|
}
|
|
|
|
func TestUsageReadBudgetAndUnknownTiming(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
fmt.Fprint(w, strings.Repeat("x", 1025))
|
|
}))
|
|
defer server.Close()
|
|
c := NewChat(&Connection{Backend: "v2", URL: server.URL})
|
|
if _, err := c.usageRead(context.Background(), "/api/model", "/scope", 1024); err == nil {
|
|
t.Fatal("accepted over-budget response")
|
|
}
|
|
u, err := NewChat(&Connection{Backend: "v1"}).Usage(context.Background(), "ses_usage")
|
|
if err != nil || u.TotalTokens != nil || u.ContextTokens != nil || u.OutputTokensPerSecond != nil {
|
|
t.Fatalf("fabricated unsupported backend metrics: %+v %v", u, err)
|
|
}
|
|
}
|
|
|
|
func TestUsagePartialFieldsAndTimingNeverInvented(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name, tokens, timing, model string
|
|
context, limit *int64
|
|
rate *float64
|
|
}{
|
|
{"missing", ``, `"created":1000,"completed":5000`, ``, nil, nil, nil},
|
|
{"no-provider-boundary", usageFixtureTokens, `"created":1000,"completed":5000`, ``, usageInt(160), nil, nil},
|
|
{"no-completion", usageFixtureTokens, `"created":1000,"streamed":3000`, ``, usageInt(160), nil, nil},
|
|
{"reversed", usageFixtureTokens, `"created":3000,"streamed":1000,"completed":5000`, ``, usageInt(160), nil, nil},
|
|
{"completion-before-stream", usageFixtureTokens, `"created":1000,"streamed":5000,"completed":3000`, ``, usageInt(160), nil, nil},
|
|
{"zero-native-output", `"tokens":{"input":0,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}`, `"created":1000,"streamed":3000,"completed":5000`, `"model":{"id":"m","providerID":"p"},"location":{"directory":"/actual"},`, usageInt(0), usageInt(0), usageFloat(0)},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
reads := 0
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
reads++
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch r.URL.Path {
|
|
case "/api/session/ses_usage":
|
|
fmt.Fprint(w, `{"data":{"id":"ses_usage",`+tc.model+`"cost":0}}`)
|
|
case "/api/model":
|
|
fmt.Fprint(w, `{"data":[{"id":"m","providerID":"p","limit":{"context":0}}]}`)
|
|
case "/api/session/ses_usage/message":
|
|
if r.URL.Query().Get("type") == "compaction" {
|
|
fmt.Fprint(w, `{"data":[]}`)
|
|
return
|
|
}
|
|
tokens := tc.tokens
|
|
if tokens != "" {
|
|
tokens = "," + tokens
|
|
}
|
|
fmt.Fprint(w, `{"data":[{"type":"assistant","time":{`+tc.timing+`}`+tokens+`}]}`)
|
|
default:
|
|
t.Errorf("unexpected request %s", r.URL.Path)
|
|
w.WriteHeader(404)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
chat := NewChat(&Connection{Backend: "v2", URL: server.URL})
|
|
u, err := chat.Usage(context.Background(), "ses_usage")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assertUsageInt(t, u.ContextTokens, tc.context)
|
|
assertUsageInt(t, u.ContextWindow, tc.limit)
|
|
if (u.OutputTokensPerSecond == nil) != (tc.rate == nil) || tc.rate != nil && *u.OutputTokensPerSecond != *tc.rate {
|
|
t.Fatalf("rate=%v", u.OutputTokensPerSecond)
|
|
}
|
|
if u.OutputTokensPerSecond == nil && u.RateBasis != "" {
|
|
t.Fatal("invented rate basis")
|
|
}
|
|
if reads > 4 {
|
|
t.Fatalf("unbounded burst: %d", reads)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestUsageStreamIdleAttachAndSettlement(t *testing.T) {
|
|
var mu sync.Mutex
|
|
settled := false
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch r.URL.Path {
|
|
case "/api/event":
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
w.WriteHeader(200)
|
|
w.(http.Flusher).Flush()
|
|
select {
|
|
case <-r.Context().Done():
|
|
return
|
|
case <-time.After(200 * time.Millisecond):
|
|
}
|
|
mu.Lock()
|
|
settled = true
|
|
mu.Unlock()
|
|
fmt.Fprint(w, "data: {\"type\":\"session.usage.updated\",\"data\":{\"sessionID\":\"ses_usage\",\"cost\":0}}\n\n")
|
|
w.(http.Flusher).Flush()
|
|
<-r.Context().Done()
|
|
case "/api/session/ses_usage":
|
|
mu.Lock()
|
|
done := settled
|
|
mu.Unlock()
|
|
n := 0
|
|
if done {
|
|
n = 1
|
|
}
|
|
fmt.Fprintf(w, `{"data":{"id":"ses_usage","cost":0,"tokens":{"input":%d,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}`, n)
|
|
case "/api/session/active":
|
|
fmt.Fprint(w, `{"data":{}}`)
|
|
case "/api/session/ses_usage/message":
|
|
fmt.Fprint(w, `{"data":[]}`)
|
|
default:
|
|
t.Errorf("unexpected %s", r.URL.Path)
|
|
w.WriteHeader(404)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
|
|
defer cancel()
|
|
var totals []int64
|
|
chat := NewChat(&Connection{Backend: "v2", URL: server.URL})
|
|
_ = chat.Stream(ctx, "ses_usage", nil, func(f protocol.ChatEventFrame) error {
|
|
if f.Kind == protocol.ChatEventUsage && f.Usage.TotalTokens != nil {
|
|
totals = append(totals, *f.Usage.TotalTokens)
|
|
if *f.Usage.TotalTokens == 1 {
|
|
cancel()
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if len(totals) != 2 || totals[0] != 0 || totals[1] != 1 {
|
|
t.Fatalf("idle/settlement snapshots=%v", totals)
|
|
}
|
|
}
|