Files
alan 910028a75c Keep last reported OpenCode metrics while the newest assistant is in flight
Read a bounded newest-first page of eight assistants (4 MiB, no cursor traversal), selecting context and completed generation rate independently from the latest valid measurement matching Session.Info.model provider/id. Require assistant model identity; never pair an old model's measurements with the selected model's window. Context remains unknown across a newer compaction until a subsequent prompt reports usage; the last completed matching rate may survive compaction.

Unknown model metadata is cached for 15 seconds instead of five minutes, while known capacities retain their five-minute cache. Provenance remains canonical V2 Model.Ref, TokenUsage and request-created/streamed/completed fields cited in chat_usage.go. No live endpoint access, inference or service changes.

Isolated HTTP regressions cover in-flight/null/empty newest tokens with prior known metrics, post-compaction boundaries, independent prompt/rate selection, switched/missing model identity, duplicate known-zero measurements, over-limit page rejection and metadata recovery. Checked full internal/opencode suite, focused usage race suite, go vet ./internal/opencode and git diff --check.
2026-10-21 00:26:48 +01:00

423 lines
16 KiB
Go

package opencode
import (
"context"
"fmt"
"math"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
const usageFixtureTokens = `"tokens":{"input":100,"output":20,"reasoning":5,"cache":{"read":50,"write":10}}`
func TestUsageNativeSnapshots(t *testing.T) {
for _, tc := range []struct {
name, fields string
total *int64
cost *float64
}{
{"missing", ``, nil, nil},
{"zero", `"cost":0,"tokens":{"input":0,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}`, usageInt(0), usageFloat(0)},
{"partial", `"tokens":{"input":100,"output":20}`, nil, nil},
{"native", `"cost":0.42,` + usageFixtureTokens, usageInt(185), usageFloat(.42)},
} {
t.Run(tc.name, func(t *testing.T) {
payload := `{"type":"session.usage.updated","data":{"sessionID":"ses_usage"`
if tc.fields != "" {
payload += "," + tc.fields
}
payload += `}}`
for i := 0; i < 2; i++ {
frame, ok, err := NormalizeChatEvent([]byte(payload), "ses_usage", nil)
if err != nil || !ok || frame.MessageID != "" || frame.Usage.Scope != "session" {
t.Fatalf("frame=%+v ok=%v err=%v", frame, ok, err)
}
assertUsageInt(t, frame.Usage.TotalTokens, tc.total)
if (frame.Usage.CostUSD == nil) != (tc.cost == nil) || tc.cost != nil && *frame.Usage.CostUSD != *tc.cost {
t.Fatalf("cost=%v", frame.Usage.CostUSD)
}
}
})
}
}
func usageInt(n int64) *int64 { return &n }
func usageFloat(n float64) *float64 { return &n }
func assertUsageInt(t *testing.T, got, want *int64) {
t.Helper()
if (got == nil) != (want == nil) || got != nil && *got != *want {
t.Fatalf("got %v want %v", got, want)
}
}
func TestUsageReadbackNativeProvenanceAndModelCache(t *testing.T) {
model := "alias-a"
compaction := false
missingCompaction := false
modelReads := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.Method != "GET" {
t.Errorf("unexpected write %s", r.Method)
w.WriteHeader(405)
return
}
switch r.URL.Path {
case "/api/session/ses_usage":
fmt.Fprintf(w, `{"data":{"id":"ses_usage","model":{"id":%q,"providerID":"p"},"location":{"directory":"/actual"},"cost":0,`+usageFixtureTokens+`}}`, model)
case "/api/model":
modelReads++
if r.URL.Query().Get("directory") != "/actual" {
t.Errorf("wrong model location: %s", r.URL.RawQuery)
}
// Larger than Call's 1MiB guard; match catalog id not modelID or
// another provider's otherwise-identical id.
fmt.Fprintf(w, `{"padding":%q,"data":[{"id":"alias-a","modelID":"upstream","providerID":"wrong","limit":{"context":1}},{"id":"alias-a","modelID":"upstream","providerID":"p","limit":{"context":200000}},{"id":"alias-b","modelID":"upstream","providerID":"p","limit":{"context":300000}}]}`, strings.Repeat("x", MaxResponse+10))
case "/api/session/ses_usage/message":
limit := "1"
if r.URL.Query().Get("type") == "assistant" {
limit = "8"
}
if r.URL.Query().Get("limit") != limit || r.URL.Query().Get("order") != "desc" {
t.Errorf("unbounded metrics history %s", r.URL.RawQuery)
}
if r.URL.Query().Get("type") == "assistant" {
fmt.Fprintf(w, `{"data":[{"type":"assistant","model":{"id":%q,"providerID":"p"},"time":{"created":1000,"streamed":3000,"completed":23000},`+usageFixtureTokens+`}]}`, model)
} else if missingCompaction {
w.WriteHeader(503)
} else if compaction {
fmt.Fprint(w, `{"data":[{"type":"compaction","time":{"created":4000}}]}`)
} else {
fmt.Fprint(w, `{"data":[]}`)
}
default:
t.Errorf("unexpected path %s", r.URL.Path)
w.WriteHeader(404)
}
}))
defer server.Close()
chat := NewChat(&Connection{Backend: "v2", URL: server.URL, Directory: "/other"})
for i := 0; i < 2; i++ {
u, err := chat.Usage(context.Background(), "ses_usage")
if err != nil {
t.Fatal(err)
}
assertUsageInt(t, u.TotalTokens, usageInt(185))
assertUsageInt(t, u.ContextTokens, usageInt(160))
assertUsageInt(t, u.ContextWindow, usageInt(200000))
if u.CostUSD == nil || *u.CostUSD != 0 || u.OutputTokensPerSecond == nil || *u.OutputTokensPerSecond != 10 || u.RateBasis != "generation" {
t.Fatalf("usage %+v", u)
}
}
if modelReads != 1 {
t.Fatalf("model reads=%d", modelReads)
}
model = "alias-b"
u, err := chat.Usage(context.Background(), "ses_usage")
if err != nil {
t.Fatal(err)
}
assertUsageInt(t, u.ContextWindow, usageInt(300000))
if modelReads != 2 {
t.Fatalf("model switch did not refresh: %d", modelReads)
}
compaction = true
u, err = chat.Usage(context.Background(), "ses_usage")
if err != nil {
t.Fatal(err)
}
assertUsageInt(t, u.ContextTokens, nil)
assertUsageInt(t, u.TotalTokens, usageInt(185))
compaction = false
missingCompaction = true
u, err = chat.Usage(context.Background(), "ses_usage")
if err != nil {
t.Fatal(err)
}
assertUsageInt(t, u.ContextTokens, nil)
}
func TestUsageReadBudgetAndUnknownTiming(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, strings.Repeat("x", 1025))
}))
defer server.Close()
c := NewChat(&Connection{Backend: "v2", URL: server.URL})
if _, err := c.usageRead(context.Background(), "/api/model", "/scope", 1024); err == nil {
t.Fatal("accepted over-budget response")
}
u, err := NewChat(&Connection{Backend: "v1"}).Usage(context.Background(), "ses_usage")
if err != nil || u.TotalTokens != nil || u.ContextTokens != nil || u.OutputTokensPerSecond != nil {
t.Fatalf("fabricated unsupported backend metrics: %+v %v", u, err)
}
}
func TestUsagePartialFieldsAndTimingNeverInvented(t *testing.T) {
for _, tc := range []struct {
name, tokens, timing, model string
context, limit *int64
rate *float64
}{
{"missing", ``, `"created":1000,"completed":5000`, ``, nil, nil, nil},
{"no-provider-boundary", usageFixtureTokens, `"created":1000,"completed":5000`, ``, usageInt(160), nil, nil},
{"no-completion", usageFixtureTokens, `"created":1000,"streamed":3000`, ``, usageInt(160), nil, nil},
{"reversed", usageFixtureTokens, `"created":3000,"streamed":1000,"completed":5000`, ``, usageInt(160), nil, nil},
{"completion-before-stream", usageFixtureTokens, `"created":1000,"streamed":5000,"completed":3000`, ``, usageInt(160), nil, nil},
{"zero-native-output", `"tokens":{"input":0,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}`, `"created":1000,"streamed":3000,"completed":5000`, `"model":{"id":"m","providerID":"p"},"location":{"directory":"/actual"},`, usageInt(0), usageInt(0), usageFloat(0)},
} {
t.Run(tc.name, func(t *testing.T) {
if tc.model == "" {
tc.model = `"model":{"id":"m","providerID":"p"},`
}
reads := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reads++
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/session/ses_usage":
fmt.Fprint(w, `{"data":{"id":"ses_usage",`+tc.model+`"cost":0}}`)
case "/api/model":
fmt.Fprint(w, `{"data":[{"id":"m","providerID":"p","limit":{"context":0}}]}`)
case "/api/session/ses_usage/message":
if r.URL.Query().Get("type") == "compaction" {
fmt.Fprint(w, `{"data":[]}`)
return
}
tokens := tc.tokens
if tokens != "" {
tokens = "," + tokens
}
fmt.Fprint(w, `{"data":[{"type":"assistant","model":{"id":"m","providerID":"p"},"time":{`+tc.timing+`}`+tokens+`}]}`)
default:
t.Errorf("unexpected request %s", r.URL.Path)
w.WriteHeader(404)
}
}))
defer server.Close()
chat := NewChat(&Connection{Backend: "v2", URL: server.URL})
u, err := chat.Usage(context.Background(), "ses_usage")
if err != nil {
t.Fatal(err)
}
assertUsageInt(t, u.ContextTokens, tc.context)
assertUsageInt(t, u.ContextWindow, tc.limit)
if (u.OutputTokensPerSecond == nil) != (tc.rate == nil) || tc.rate != nil && *u.OutputTokensPerSecond != *tc.rate {
t.Fatalf("rate=%v", u.OutputTokensPerSecond)
}
if u.OutputTokensPerSecond == nil && u.RateBasis != "" {
t.Fatal("invented rate basis")
}
if reads > 4 {
t.Fatalf("unbounded burst: %d", reads)
}
})
}
}
func TestUsageStreamIdleAttachAndSettlement(t *testing.T) {
var mu sync.Mutex
settled := false
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/event":
w.Header().Set("Content-Type", "text/event-stream")
w.WriteHeader(200)
w.(http.Flusher).Flush()
select {
case <-r.Context().Done():
return
case <-time.After(200 * time.Millisecond):
}
mu.Lock()
settled = true
mu.Unlock()
fmt.Fprint(w, "data: {\"type\":\"session.usage.updated\",\"data\":{\"sessionID\":\"ses_usage\",\"cost\":0}}\n\n")
w.(http.Flusher).Flush()
<-r.Context().Done()
case "/api/session/ses_usage":
mu.Lock()
done := settled
mu.Unlock()
n := 0
if done {
n = 1
}
fmt.Fprintf(w, `{"data":{"id":"ses_usage","cost":0,"tokens":{"input":%d,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}`, n)
case "/api/session/active":
fmt.Fprint(w, `{"data":{}}`)
case "/api/session/ses_usage/message":
fmt.Fprint(w, `{"data":[]}`)
default:
t.Errorf("unexpected %s", r.URL.Path)
w.WriteHeader(404)
}
}))
defer server.Close()
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
defer cancel()
var totals []int64
chat := NewChat(&Connection{Backend: "v2", URL: server.URL})
_ = chat.Stream(ctx, "ses_usage", nil, func(f protocol.ChatEventFrame) error {
if f.Kind == protocol.ChatEventUsage && f.Usage.TotalTokens != nil {
totals = append(totals, *f.Usage.TotalTokens)
if *f.Usage.TotalTokens == 1 {
cancel()
}
}
return nil
})
if len(totals) != 2 || totals[0] != 0 || totals[1] != 1 {
t.Fatalf("idle/settlement snapshots=%v", totals)
}
}
func TestUsageRecentAssistantsRetainLastMatchingMeasurements(t *testing.T) {
const selected = `"model":{"id":"m","providerID":"p"}`
const knownTokens = `"tokens":{"input":996,"output":71,"reasoning":0,"cache":{"read":479616,"write":0}}`
const knownTime = `"created":1792538187795,"streamed":1792538196498,"completed":1792538197150`
assistant := func(model, timing, tokens string) string {
if model != "" {
model += ","
}
return `{"type":"assistant",` + model + `"time":{` + timing + `},` + tokens + `}`
}
known := assistant(selected, knownTime, knownTokens)
inflight := assistant(selected, `"created":1792538200000,"streamed":1792538201000`, `"tokens":null`)
zero := assistant(selected, `"created":1792538198000,"streamed":1792538199000,"completed":1792538199500`, `"tokens":{"input":0,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}`)
rate := 71.0 / 8.703
for _, tc := range []struct {
name, sessionModel, messages, compactions string
context *int64
rate *float64
}{
{"inflight-previous-known", selected, inflight + "," + known, `[]`, usageInt(480612), &rate},
{"empty-newest-previous-known", selected, assistant(selected, `"created":1792538200000`, `"tokens":{}`) + "," + known, `[]`, usageInt(480612), &rate},
{"post-compaction-no-new-prompt", selected, inflight + "," + known, `[{"type":"compaction","time":{"created":1792538198000}}]`, nil, &rate},
// Context and rate are independently last-reported, not necessarily the
// same assistant: the newest prompt can report tokens before completion.
{"new-prompt-after-compaction", selected, assistant(selected, `"created":1792538200000`, knownTokens) + "," + known, `[{"type":"compaction","time":{"created":1792538198000}}]`, usageInt(480612), &rate},
{"switched-model", `"model":{"id":"new","providerID":"p"}`, inflight + "," + known, `[]`, nil, nil},
{"wrong-provider", `"model":{"id":"m","providerID":"other"}`, known, `[]`, nil, nil},
{"missing-assistant-model", selected, assistant("", knownTime, knownTokens), `[]`, nil, nil},
{"missing-session-model", ``, known, `[]`, nil, nil},
{"duplicate-zero-is-known", selected, inflight + "," + zero + "," + zero + "," + known, `[]`, usageInt(0), usageFloat(0)},
{"reject-over-limit", selected, strings.TrimSuffix(strings.Repeat(known+",", usageAssistantLimit+1), ","), `[]`, nil, nil},
} {
t.Run(tc.name, func(t *testing.T) {
reads := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reads++
w.Header().Set("Content-Type", "application/json")
if r.Method != "GET" {
t.Errorf("unexpected write %s", r.Method)
w.WriteHeader(405)
return
}
switch r.URL.Path {
case "/api/session/ses_usage":
model := tc.sessionModel
if model != "" {
model += ","
}
fmt.Fprint(w, `{"data":{"id":"ses_usage",`+model+`"location":{"directory":"/scope"},"cost":0,`+usageFixtureTokens+`}}`)
case "/api/model":
fmt.Fprint(w, `{"data":[{"id":"m","providerID":"p","limit":{"context":1000000}},{"id":"new","providerID":"p","limit":{"context":2000000}}]}`)
case "/api/session/ses_usage/message":
if r.URL.Query().Get("order") != "desc" || r.URL.Query().Get("cursor") != "" {
t.Errorf("unexpected history request %s", r.URL.RawQuery)
}
switch r.URL.Query().Get("type") {
case "assistant":
if r.URL.Query().Get("limit") != "8" {
t.Errorf("unbounded assistant request %s", r.URL.RawQuery)
}
fmt.Fprint(w, `{"data":[`+tc.messages+`]}`)
case "compaction":
if r.URL.Query().Get("limit") != "1" {
t.Errorf("unbounded compaction request %s", r.URL.RawQuery)
}
fmt.Fprint(w, `{"data":`+tc.compactions+`}`)
default:
t.Errorf("unexpected history type %s", r.URL.RawQuery)
w.WriteHeader(404)
}
default:
t.Errorf("unexpected path %s", r.URL.Path)
w.WriteHeader(404)
}
}))
defer server.Close()
chat := NewChat(&Connection{Backend: "v2", URL: server.URL})
for i := 0; i < 2; i++ {
u, err := chat.Usage(context.Background(), "ses_usage")
if err != nil {
t.Fatal(err)
}
assertUsageInt(t, u.ContextTokens, tc.context)
assertUsageInt(t, u.TotalTokens, usageInt(185))
if (u.OutputTokensPerSecond == nil) != (tc.rate == nil) || tc.rate != nil && math.Abs(*u.OutputTokensPerSecond-*tc.rate) > 1e-9 {
t.Fatalf("rate=%v want=%v", u.OutputTokensPerSecond, tc.rate)
}
if tc.rate != nil && u.RateBasis != "generation" || tc.rate == nil && u.RateBasis != "" {
t.Fatalf("basis=%q", u.RateBasis)
}
}
if reads > 7 {
t.Fatalf("unbounded two-read burst=%d", reads)
}
})
}
}
func TestUsageUnknownModelMetadataRetriesSoonerThanKnownMetadata(t *testing.T) {
reads := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reads++
w.Header().Set("Content-Type", "application/json")
if reads == 1 {
w.WriteHeader(503)
return
}
fmt.Fprint(w, `{"data":[{"id":"m","providerID":"p","limit":{"context":1000000}}]}`)
}))
defer server.Close()
chat := NewChat(&Connection{Backend: "v2", URL: server.URL})
for i := 0; i < 2; i++ {
assertUsageInt(t, chat.usageModelLimit(context.Background(), "/scope", "p", "m"), nil)
}
if reads != 1 {
t.Fatalf("failure not briefly cached: %d", reads)
}
key := "/scope\x00p\x00m"
age := func(d time.Duration) {
chat.usageMu.Lock()
entry := chat.usageModels[key]
entry.at = time.Now().Add(-d)
chat.usageModels[key] = entry
chat.usageMu.Unlock()
}
age(16 * time.Second)
assertUsageInt(t, chat.usageModelLimit(context.Background(), "/scope", "p", "m"), usageInt(1000000))
if reads != 2 {
t.Fatalf("unknown metadata did not recover: %d", reads)
}
age(16 * time.Second)
assertUsageInt(t, chat.usageModelLimit(context.Background(), "/scope", "p", "m"), usageInt(1000000))
if reads != 2 {
t.Fatalf("known metadata lost five-minute cache: %d", reads)
}
age(6 * time.Minute)
assertUsageInt(t, chat.usageModelLimit(context.Background(), "/scope", "p", "m"), usageInt(1000000))
if reads != 3 {
t.Fatalf("known metadata did not refresh: %d", reads)
}
}