package opencode import ( "context" "fmt" "math" "net/http" "net/http/httptest" "strings" "sync" "testing" "time" "git.a-wan.net/alan/pipelines/internal/protocol" ) const usageFixtureTokens = `"tokens":{"input":100,"output":20,"reasoning":5,"cache":{"read":50,"write":10}}` func TestUsageNativeSnapshots(t *testing.T) { for _, tc := range []struct { name, fields string total *int64 cost *float64 }{ {"missing", ``, nil, nil}, {"zero", `"cost":0,"tokens":{"input":0,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}`, usageInt(0), usageFloat(0)}, {"partial", `"tokens":{"input":100,"output":20}`, nil, nil}, {"native", `"cost":0.42,` + usageFixtureTokens, usageInt(185), usageFloat(.42)}, } { t.Run(tc.name, func(t *testing.T) { payload := `{"type":"session.usage.updated","data":{"sessionID":"ses_usage"` if tc.fields != "" { payload += "," + tc.fields } payload += `}}` for i := 0; i < 2; i++ { frame, ok, err := NormalizeChatEvent([]byte(payload), "ses_usage", nil) if err != nil || !ok || frame.MessageID != "" || frame.Usage.Scope != "session" { t.Fatalf("frame=%+v ok=%v err=%v", frame, ok, err) } assertUsageInt(t, frame.Usage.TotalTokens, tc.total) if (frame.Usage.CostUSD == nil) != (tc.cost == nil) || tc.cost != nil && *frame.Usage.CostUSD != *tc.cost { t.Fatalf("cost=%v", frame.Usage.CostUSD) } } }) } } func usageInt(n int64) *int64 { return &n } func usageFloat(n float64) *float64 { return &n } func assertUsageInt(t *testing.T, got, want *int64) { t.Helper() if (got == nil) != (want == nil) || got != nil && *got != *want { t.Fatalf("got %v want %v", got, want) } } func TestUsageReadbackNativeProvenanceAndModelCache(t *testing.T) { model := "alias-a" compaction := false missingCompaction := false modelReads := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") if r.Method != "GET" { t.Errorf("unexpected write %s", r.Method) w.WriteHeader(405) return } switch r.URL.Path { case "/api/session/ses_usage": fmt.Fprintf(w, `{"data":{"id":"ses_usage","model":{"id":%q,"providerID":"p"},"location":{"directory":"/actual"},"cost":0,`+usageFixtureTokens+`}}`, model) case "/api/model": modelReads++ if r.URL.Query().Get("directory") != "/actual" { t.Errorf("wrong model location: %s", r.URL.RawQuery) } // Larger than Call's 1MiB guard; match catalog id not modelID or // another provider's otherwise-identical id. fmt.Fprintf(w, `{"padding":%q,"data":[{"id":"alias-a","modelID":"upstream","providerID":"wrong","limit":{"context":1}},{"id":"alias-a","modelID":"upstream","providerID":"p","limit":{"context":200000}},{"id":"alias-b","modelID":"upstream","providerID":"p","limit":{"context":300000}}]}`, strings.Repeat("x", MaxResponse+10)) case "/api/session/ses_usage/message": limit := "1" if r.URL.Query().Get("type") == "assistant" { limit = "8" } if r.URL.Query().Get("limit") != limit || r.URL.Query().Get("order") != "desc" { t.Errorf("unbounded metrics history %s", r.URL.RawQuery) } if r.URL.Query().Get("type") == "assistant" { fmt.Fprintf(w, `{"data":[{"type":"assistant","model":{"id":%q,"providerID":"p"},"time":{"created":1000,"streamed":3000,"completed":23000},`+usageFixtureTokens+`}]}`, model) } else if missingCompaction { w.WriteHeader(503) } else if compaction { fmt.Fprint(w, `{"data":[{"type":"compaction","time":{"created":4000}}]}`) } else { fmt.Fprint(w, `{"data":[]}`) } default: t.Errorf("unexpected path %s", r.URL.Path) w.WriteHeader(404) } })) defer server.Close() chat := NewChat(&Connection{Backend: "v2", URL: server.URL, Directory: "/other"}) for i := 0; i < 2; i++ { u, err := chat.Usage(context.Background(), "ses_usage") if err != nil { t.Fatal(err) } assertUsageInt(t, u.TotalTokens, usageInt(185)) assertUsageInt(t, u.ContextTokens, usageInt(160)) assertUsageInt(t, u.ContextWindow, usageInt(200000)) if u.CostUSD == nil || *u.CostUSD != 0 || u.OutputTokensPerSecond == nil || *u.OutputTokensPerSecond != 10 || u.RateBasis != "generation" { t.Fatalf("usage %+v", u) } } if modelReads != 1 { t.Fatalf("model reads=%d", modelReads) } model = "alias-b" u, err := chat.Usage(context.Background(), "ses_usage") if err != nil { t.Fatal(err) } assertUsageInt(t, u.ContextWindow, usageInt(300000)) if modelReads != 2 { t.Fatalf("model switch did not refresh: %d", modelReads) } compaction = true u, err = chat.Usage(context.Background(), "ses_usage") if err != nil { t.Fatal(err) } assertUsageInt(t, u.ContextTokens, nil) assertUsageInt(t, u.TotalTokens, usageInt(185)) compaction = false missingCompaction = true u, err = chat.Usage(context.Background(), "ses_usage") if err != nil { t.Fatal(err) } assertUsageInt(t, u.ContextTokens, nil) } func TestUsageReadBudgetAndUnknownTiming(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, strings.Repeat("x", 1025)) })) defer server.Close() c := NewChat(&Connection{Backend: "v2", URL: server.URL}) if _, err := c.usageRead(context.Background(), "/api/model", "/scope", 1024); err == nil { t.Fatal("accepted over-budget response") } u, err := NewChat(&Connection{Backend: "v1"}).Usage(context.Background(), "ses_usage") if err != nil || u.TotalTokens != nil || u.ContextTokens != nil || u.OutputTokensPerSecond != nil { t.Fatalf("fabricated unsupported backend metrics: %+v %v", u, err) } } func TestUsagePartialFieldsAndTimingNeverInvented(t *testing.T) { for _, tc := range []struct { name, tokens, timing, model string context, limit *int64 rate *float64 }{ {"missing", ``, `"created":1000,"completed":5000`, ``, nil, nil, nil}, {"no-provider-boundary", usageFixtureTokens, `"created":1000,"completed":5000`, ``, usageInt(160), nil, nil}, {"no-completion", usageFixtureTokens, `"created":1000,"streamed":3000`, ``, usageInt(160), nil, nil}, {"reversed", usageFixtureTokens, `"created":3000,"streamed":1000,"completed":5000`, ``, usageInt(160), nil, nil}, {"completion-before-stream", usageFixtureTokens, `"created":1000,"streamed":5000,"completed":3000`, ``, usageInt(160), nil, nil}, {"zero-native-output", `"tokens":{"input":0,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}`, `"created":1000,"streamed":3000,"completed":5000`, `"model":{"id":"m","providerID":"p"},"location":{"directory":"/actual"},`, usageInt(0), usageInt(0), usageFloat(0)}, } { t.Run(tc.name, func(t *testing.T) { if tc.model == "" { tc.model = `"model":{"id":"m","providerID":"p"},` } reads := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reads++ w.Header().Set("Content-Type", "application/json") switch r.URL.Path { case "/api/session/ses_usage": fmt.Fprint(w, `{"data":{"id":"ses_usage",`+tc.model+`"cost":0}}`) case "/api/model": fmt.Fprint(w, `{"data":[{"id":"m","providerID":"p","limit":{"context":0}}]}`) case "/api/session/ses_usage/message": if r.URL.Query().Get("type") == "compaction" { fmt.Fprint(w, `{"data":[]}`) return } tokens := tc.tokens if tokens != "" { tokens = "," + tokens } fmt.Fprint(w, `{"data":[{"type":"assistant","model":{"id":"m","providerID":"p"},"time":{`+tc.timing+`}`+tokens+`}]}`) default: t.Errorf("unexpected request %s", r.URL.Path) w.WriteHeader(404) } })) defer server.Close() chat := NewChat(&Connection{Backend: "v2", URL: server.URL}) u, err := chat.Usage(context.Background(), "ses_usage") if err != nil { t.Fatal(err) } assertUsageInt(t, u.ContextTokens, tc.context) assertUsageInt(t, u.ContextWindow, tc.limit) if (u.OutputTokensPerSecond == nil) != (tc.rate == nil) || tc.rate != nil && *u.OutputTokensPerSecond != *tc.rate { t.Fatalf("rate=%v", u.OutputTokensPerSecond) } if u.OutputTokensPerSecond == nil && u.RateBasis != "" { t.Fatal("invented rate basis") } if reads > 4 { t.Fatalf("unbounded burst: %d", reads) } }) } } func TestUsageStreamIdleAttachAndSettlement(t *testing.T) { var mu sync.Mutex settled := false server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch r.URL.Path { case "/api/event": w.Header().Set("Content-Type", "text/event-stream") w.WriteHeader(200) w.(http.Flusher).Flush() select { case <-r.Context().Done(): return case <-time.After(200 * time.Millisecond): } mu.Lock() settled = true mu.Unlock() fmt.Fprint(w, "data: {\"type\":\"session.usage.updated\",\"data\":{\"sessionID\":\"ses_usage\",\"cost\":0}}\n\n") w.(http.Flusher).Flush() <-r.Context().Done() case "/api/session/ses_usage": mu.Lock() done := settled mu.Unlock() n := 0 if done { n = 1 } fmt.Fprintf(w, `{"data":{"id":"ses_usage","cost":0,"tokens":{"input":%d,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}`, n) case "/api/session/active": fmt.Fprint(w, `{"data":{}}`) case "/api/session/ses_usage/message": fmt.Fprint(w, `{"data":[]}`) default: t.Errorf("unexpected %s", r.URL.Path) w.WriteHeader(404) } })) defer server.Close() ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) defer cancel() var totals []int64 chat := NewChat(&Connection{Backend: "v2", URL: server.URL}) _ = chat.Stream(ctx, "ses_usage", nil, func(f protocol.ChatEventFrame) error { if f.Kind == protocol.ChatEventUsage && f.Usage.TotalTokens != nil { totals = append(totals, *f.Usage.TotalTokens) if *f.Usage.TotalTokens == 1 { cancel() } } return nil }) if len(totals) != 2 || totals[0] != 0 || totals[1] != 1 { t.Fatalf("idle/settlement snapshots=%v", totals) } } func TestUsageRecentAssistantsRetainLastMatchingMeasurements(t *testing.T) { const selected = `"model":{"id":"m","providerID":"p"}` const knownTokens = `"tokens":{"input":996,"output":71,"reasoning":0,"cache":{"read":479616,"write":0}}` const knownTime = `"created":1792538187795,"streamed":1792538196498,"completed":1792538197150` assistant := func(model, timing, tokens string) string { if model != "" { model += "," } return `{"type":"assistant",` + model + `"time":{` + timing + `},` + tokens + `}` } known := assistant(selected, knownTime, knownTokens) inflight := assistant(selected, `"created":1792538200000,"streamed":1792538201000`, `"tokens":null`) zero := assistant(selected, `"created":1792538198000,"streamed":1792538199000,"completed":1792538199500`, `"tokens":{"input":0,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}`) rate := 71.0 / 8.703 for _, tc := range []struct { name, sessionModel, messages, compactions string context *int64 rate *float64 }{ {"inflight-previous-known", selected, inflight + "," + known, `[]`, usageInt(480612), &rate}, {"empty-newest-previous-known", selected, assistant(selected, `"created":1792538200000`, `"tokens":{}`) + "," + known, `[]`, usageInt(480612), &rate}, {"post-compaction-no-new-prompt", selected, inflight + "," + known, `[{"type":"compaction","time":{"created":1792538198000}}]`, nil, &rate}, // Context and rate are independently last-reported, not necessarily the // same assistant: the newest prompt can report tokens before completion. {"new-prompt-after-compaction", selected, assistant(selected, `"created":1792538200000`, knownTokens) + "," + known, `[{"type":"compaction","time":{"created":1792538198000}}]`, usageInt(480612), &rate}, {"switched-model", `"model":{"id":"new","providerID":"p"}`, inflight + "," + known, `[]`, nil, nil}, {"wrong-provider", `"model":{"id":"m","providerID":"other"}`, known, `[]`, nil, nil}, {"missing-assistant-model", selected, assistant("", knownTime, knownTokens), `[]`, nil, nil}, {"missing-session-model", ``, known, `[]`, nil, nil}, {"duplicate-zero-is-known", selected, inflight + "," + zero + "," + zero + "," + known, `[]`, usageInt(0), usageFloat(0)}, {"reject-over-limit", selected, strings.TrimSuffix(strings.Repeat(known+",", usageAssistantLimit+1), ","), `[]`, nil, nil}, } { t.Run(tc.name, func(t *testing.T) { reads := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reads++ w.Header().Set("Content-Type", "application/json") if r.Method != "GET" { t.Errorf("unexpected write %s", r.Method) w.WriteHeader(405) return } switch r.URL.Path { case "/api/session/ses_usage": model := tc.sessionModel if model != "" { model += "," } fmt.Fprint(w, `{"data":{"id":"ses_usage",`+model+`"location":{"directory":"/scope"},"cost":0,`+usageFixtureTokens+`}}`) case "/api/model": fmt.Fprint(w, `{"data":[{"id":"m","providerID":"p","limit":{"context":1000000}},{"id":"new","providerID":"p","limit":{"context":2000000}}]}`) case "/api/session/ses_usage/message": if r.URL.Query().Get("order") != "desc" || r.URL.Query().Get("cursor") != "" { t.Errorf("unexpected history request %s", r.URL.RawQuery) } switch r.URL.Query().Get("type") { case "assistant": if r.URL.Query().Get("limit") != "8" { t.Errorf("unbounded assistant request %s", r.URL.RawQuery) } fmt.Fprint(w, `{"data":[`+tc.messages+`]}`) case "compaction": if r.URL.Query().Get("limit") != "1" { t.Errorf("unbounded compaction request %s", r.URL.RawQuery) } fmt.Fprint(w, `{"data":`+tc.compactions+`}`) default: t.Errorf("unexpected history type %s", r.URL.RawQuery) w.WriteHeader(404) } default: t.Errorf("unexpected path %s", r.URL.Path) w.WriteHeader(404) } })) defer server.Close() chat := NewChat(&Connection{Backend: "v2", URL: server.URL}) for i := 0; i < 2; i++ { u, err := chat.Usage(context.Background(), "ses_usage") if err != nil { t.Fatal(err) } assertUsageInt(t, u.ContextTokens, tc.context) assertUsageInt(t, u.TotalTokens, usageInt(185)) if (u.OutputTokensPerSecond == nil) != (tc.rate == nil) || tc.rate != nil && math.Abs(*u.OutputTokensPerSecond-*tc.rate) > 1e-9 { t.Fatalf("rate=%v want=%v", u.OutputTokensPerSecond, tc.rate) } if tc.rate != nil && u.RateBasis != "generation" || tc.rate == nil && u.RateBasis != "" { t.Fatalf("basis=%q", u.RateBasis) } } if reads > 7 { t.Fatalf("unbounded two-read burst=%d", reads) } }) } } func TestUsageUnknownModelMetadataRetriesSoonerThanKnownMetadata(t *testing.T) { reads := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reads++ w.Header().Set("Content-Type", "application/json") if reads == 1 { w.WriteHeader(503) return } fmt.Fprint(w, `{"data":[{"id":"m","providerID":"p","limit":{"context":1000000}}]}`) })) defer server.Close() chat := NewChat(&Connection{Backend: "v2", URL: server.URL}) for i := 0; i < 2; i++ { assertUsageInt(t, chat.usageModelLimit(context.Background(), "/scope", "p", "m"), nil) } if reads != 1 { t.Fatalf("failure not briefly cached: %d", reads) } key := "/scope\x00p\x00m" age := func(d time.Duration) { chat.usageMu.Lock() entry := chat.usageModels[key] entry.at = time.Now().Add(-d) chat.usageModels[key] = entry chat.usageMu.Unlock() } age(16 * time.Second) assertUsageInt(t, chat.usageModelLimit(context.Background(), "/scope", "p", "m"), usageInt(1000000)) if reads != 2 { t.Fatalf("unknown metadata did not recover: %d", reads) } age(16 * time.Second) assertUsageInt(t, chat.usageModelLimit(context.Background(), "/scope", "p", "m"), usageInt(1000000)) if reads != 2 { t.Fatalf("known metadata lost five-minute cache: %d", reads) } age(6 * time.Minute) assertUsageInt(t, chat.usageModelLimit(context.Background(), "/scope", "p", "m"), usageInt(1000000)) if reads != 3 { t.Fatalf("known metadata did not refresh: %d", reads) } }