From 3227d61caf1cf6735878e2b9b68a0ca6e95d6c94 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Tue, 3 Nov 2026 22:16:49 +0000 Subject: [PATCH] Token usage: keep cache/reasoning split, count only recorded steps, cost only when positive; captures fixture, docs and notes --- README.md | 5 +- app/lib/features/runs/domain/run_tokens.dart | 80 ++++++++++------ .../features/runs/ui/run_detail_content.dart | 11 ++- .../runs/ui/run_detail_visit_facts.dart | 3 +- app/notes/run-tokens.md | 48 ++++++++++ .../features/runs/domain/run_tokens_test.dart | 55 ++++++++++- .../ui/run_detail_summary_fields_test.dart | 2 +- .../runs/ui/run_stage_tokens_test.dart | 2 +- app/tool/probe_usage_fixture.dart | 92 +++++++++++++++++++ app/tool/search_selection_probe_entry.dart | 22 ++++- docs/cli.md | 32 +++++-- internal/cli/commands.go | 2 +- internal/pipeline/run_usage.go | 10 +- internal/pipeline/run_usage_test.go | 85 +++++++++++++++++ internal/pipeline/stats.go | 19 +++- internal/protocol/runs.go | 24 +++++ internal/protocol/stats.go | 6 ++ internal/server/runner.go | 4 + internal/server/usage_test.go | 6 +- 19 files changed, 447 insertions(+), 61 deletions(-) create mode 100644 app/notes/run-tokens.md create mode 100644 app/tool/probe_usage_fixture.dart diff --git a/README.md b/README.md index 17849fef..4a106e10 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,10 @@ pipelines harness list # status of all four **Runs** - Runs list with filters, multi-select and delete, and sorting by newest, - oldest, **largest on disk**, duration, pipeline or status. + oldest, **largest on disk**, **most tokens**, duration, pipeline or status. + Each row shows the run's size and reported token total (`22.4M tokens`, + `≥` when some stages did not report); the run Summary adds fresh versus + cached tokens, the in/out split and the reported cost. - Run detail tabs: Summary, Timeline, and Artifacts and Changes when the run has any. - Artifacts viewer for files a stage produced, including the recorded session diff --git a/app/lib/features/runs/domain/run_tokens.dart b/app/lib/features/runs/domain/run_tokens.dart index 034d7677..4189c7d7 100644 --- a/app/lib/features/runs/domain/run_tokens.dart +++ b/app/lib/features/runs/domain/run_tokens.dart @@ -11,6 +11,7 @@ class TokenUsage { required this.partial, this.input, this.output, + this.cacheRead, this.cost, this.costPartial = false, this.stages, @@ -27,7 +28,16 @@ class TokenUsage { final int? input; final int? output; - /// The reported cost in USD, only when some stage reported one. + /// The cache-read share of [total], only when every counted stage reported + /// its cache components. Older rows never recorded them, so they have none. + final int? cacheRead; + + /// The tokens that were not cache reads, when [cacheRead] is known. + int? get fresh => cacheRead == null ? null : total - cacheRead!; + + /// The reported cost in USD, only when a positive cost was reported. Some + /// providers (subscription models) report a cost of zero or none at all; + /// that is "not reported", never a known zero. final double? cost; /// Some stages reported no cost, so [cost] is a lower bound. @@ -51,14 +61,16 @@ TokenUsage? runTokenUsage(JsonMap run) { if (tokenStages <= 0 || total == null) return null; final costStages = _int(raw['cost_known_stages']) ?? 0; final split = raw['split_known'] == true; + final cost = costStages > 0 ? _positive(raw['cost_usd']) : null; return TokenUsage( total: total, partial: stages != null && tokenStages < stages, // The server omits a zero in/out, so a known split makes absence zero. input: split ? _int(raw['tokens_in']) ?? 0 : null, output: split ? _int(raw['tokens_out']) ?? 0 : null, - cost: costStages > 0 ? _double(raw['cost_usd']) ?? 0 : null, - costPartial: costStages > 0 && stages != null && costStages < stages, + cacheRead: _cacheRead(raw, total), + cost: cost, + costPartial: cost != null && stages != null && costStages < stages, stages: stages, tokenStages: tokenStages, ); @@ -77,7 +89,8 @@ TokenUsage? stageTokenUsage(JsonMap step) { partial: false, input: split ? _int(raw['tokens_in']) ?? 0 : null, output: split ? _int(raw['tokens_out']) ?? 0 : null, - cost: raw['cost_known'] == true ? _double(raw['cost_usd']) ?? 0 : null, + cacheRead: _cacheRead(raw, total), + cost: raw['cost_known'] == true ? _positive(raw['cost_usd']) : null, ); } @@ -109,37 +122,46 @@ String formatTokenCount(int tokens) { String tokenUsageLabel(TokenUsage usage) => '${usage.partial ? '≥' : ''}${formatTokenCount(usage.total)} tokens'; -/// The detail's token value: the compact total, the in/out split when +/// The detail's token value: the compact total, then fresh versus cached +/// when every stage reported its cache components, the in/out split when /// reported, and how many stages reported when the total is partial. -/// `22.4M (20M in, 2.4M out)`, `≥9 · 1 of 2 stages reported`. -String tokenUsageDetail(TokenUsage usage) { - final parts = [ - [ - '${usage.partial ? '≥' : ''}${formatTokenCount(usage.total)}', - if (usage.input != null && usage.output != null) - '(${formatTokenCount(usage.input!)} in, ' - '${formatTokenCount(usage.output!)} out)', - ].join(' '), - if (usage.partial && usage.stages != null) - '${usage.tokenStages} of ${usage.stages} stages reported', - ]; - return parts.join(' · '); -} +/// `22.4M · 1.1M fresh, 21.3M cached · 900K in, 150K out`, +/// `≥9 · 1 of 2 stages reported`. +String tokenUsageDetail(TokenUsage usage) => [ + '${usage.partial ? '≥' : ''}${formatTokenCount(usage.total)}', + if (usage.fresh != null) + '${formatTokenCount(usage.fresh!)} fresh, ' + '${formatTokenCount(usage.cacheRead!)} cached', + if (usage.input != null && usage.output != null) + '${formatTokenCount(usage.input!)} in, ' + '${formatTokenCount(usage.output!)} out', + if (usage.partial && usage.stages != null) + '${usage.tokenStages} of ${usage.stages} stages reported', +].join(' · '); -/// The reported cost, or null when no stage reported one: `$1.75`, `$0.10`, -/// `$0.0042` (four places below a cent), -/// `≥$0.10` when some stages reported no cost. -String? tokenUsageCost(TokenUsage usage) { +/// The wording for a cost no stage reported as a positive amount. +const costNotReported = 'Not reported'; + +/// The reported cost: `$1.75`, `$0.10`, `$0.0042` (four places below a cent), +/// `≥$0.10` when some stages reported no cost, and [costNotReported] when no +/// positive cost was reported. +String tokenUsageCost(TokenUsage usage) { final cost = usage.cost; - if (cost == null) return null; - final text = cost >= 0.01 || cost == 0 - ? cost.toStringAsFixed(2) - : cost.toStringAsFixed(4); + if (cost == null) return costNotReported; + final text = cost >= 0.01 ? cost.toStringAsFixed(2) : cost.toStringAsFixed(4); return '${usage.costPartial ? '≥' : ''}\$$text'; } int? _int(dynamic value) => value is num && value.isFinite && value >= 0 ? value.round() : null; -double? _double(dynamic value) => - value is num && value.isFinite && value >= 0 ? value.toDouble() : null; +double? _positive(dynamic value) => + value is num && value.isFinite && value > 0 ? value.toDouble() : null; + +/// The cache reads, only when the server marked the cache split complete and +/// the count fits inside the total. +int? _cacheRead(Map raw, int total) { + if (raw['cache_known'] != true) return null; + final read = _int(raw['cache_read']) ?? 0; + return read <= total ? read : null; +} diff --git a/app/lib/features/runs/ui/run_detail_content.dart b/app/lib/features/runs/ui/run_detail_content.dart index d2605a32..bb296eb4 100644 --- a/app/lib/features/runs/ui/run_detail_content.dart +++ b/app/lib/features/runs/ui/run_detail_content.dart @@ -76,6 +76,7 @@ List detailUsageMetrics(dynamic raw) { final tokenStages = _int(usage['tokens_known_stages']) ?? 0; final costStages = _int(usage['cost_known_stages']) ?? 0; final totalTokens = _int(usage['tokens_total']); + final cost = usage['cost_usd'] is num ? usage['cost_usd'] as num : null; final splitKnown = usage['split_known'] == true && tokenStages > 0; final rows = [ if (stages != null) (label: 'Stages', value: '$stages'), @@ -95,11 +96,11 @@ List detailUsageMetrics(dynamic raw) { ), ( label: 'Reported cost', - // cost_usd is omitted by the server when zero, so a known stage count - // makes an absent value a real $0.000000 rather than an unknown cost. - value: costStages == 0 - ? 'Unavailable' - : '\$${((usage['cost_usd'] as num?) ?? 0).toStringAsFixed(6)}', + // A cost of zero (or none) is what subscription models report even + // with a "known" cost: it is not reported, never a measured zero. + value: costStages == 0 || cost == null || cost <= 0 + ? 'Not reported' + : '\$${cost.toStringAsFixed(6)}', ), ]; if (stages != null && tokenStages < stages) { diff --git a/app/lib/features/runs/ui/run_detail_visit_facts.dart b/app/lib/features/runs/ui/run_detail_visit_facts.dart index 17767453..8a11d4d3 100644 --- a/app/lib/features/runs/ui/run_detail_visit_facts.dart +++ b/app/lib/features/runs/ui/run_detail_visit_facts.dart @@ -84,8 +84,7 @@ List detailVisitFacts(JsonMap step) { final usage = stageTokenUsage(step); if (usage != null) { facts.add((label: 'Tokens', value: tokenUsageDetail(usage))); - final cost = tokenUsageCost(usage); - if (cost != null) facts.add((label: 'Cost', value: cost)); + facts.add((label: 'Cost', value: tokenUsageCost(usage))); } final target = _text(step['target']); if (target != null) facts.add((label: 'Target', value: target)); diff --git a/app/notes/run-tokens.md b/app/notes/run-tokens.md new file mode 100644 index 00000000..96041afb --- /dev/null +++ b/app/notes/run-tokens.md @@ -0,0 +1,48 @@ +# Run token usage beside size + +Runs that spend 20M+ tokens now show it the way they show disk size. + +## Server (payload) + +- `RunSummary.usage` (`protocol.RunUsage`) moved up from `RunDetail`, so + `list_runs` rows carry the run total too. `get_run` still sends `usage` at + the same JSON key, now through the embedded summary. +- The list fills it with one grouped `step_usage` aggregate per 100-run batch + (`internal/pipeline/run_usage.go`), using the same column sums as the + history-wide stats (`usageSumColumns` in `stats.go`). It is joined to + `run_steps`, so orphaned usage rows (reported: about 650 rows, 1.47B tokens, + left by an Oct 19 history wipe) no longer count in stats or per-run totals. + The rows are not deleted. If a wiped run id was reused, an orphan row that + matches a new run's (run_id, sequence) still cannot be told apart. +- `StageUsage` now keeps `tokens_reasoning`, `cache_read` and `cache_write` + with `cache_known` (the runner dropped them before). They are JSON keys in + `usage_json`, so old rows need no migration: they lack `cache_known`, so + their cache share is unknown, not zero. Run totals add `cache_known_stages` + and `cache_known` (every token-reporting stage recorded the split). + +## App + +- `lib/features/runs/domain/run_tokens.dart`: parsing and formatting. + `22.4M tokens`, `≥9.1M tokens` when `tokens_known_stages < stages`. Nothing + is estimated, and a row without `usage` (an older server) shows nothing. +- List row: tokens after the size, in both the full and the compact subtitle. +- Sort: `Most tokens` (`most_tokens`) after `Largest first`, flat like it. + Runs without a reported total sort last. +- Summary: `Size on disk`, `Tokens` (`22.4M · 2.1M fresh, 20.3M cached · + 1.8M in, 240K out`; fresh is total minus cache reads, shown only when + `cache_known`) and `Cost`. +- Stage rows: the Timeline row subtitle adds `· 20.1M tokens`. The visit facts + add `Tokens` and `Cost`. +- Cost is shown only when a positive `cost_usd` was reported. Subscription + models report `cost_known` with no amount, which shows as `Not reported`. + The stats page's `Reported cost` uses the same rule. +- `usage` is no longer treated as detail-only in + `RunsController.acceptRunDetail`, so opening a run refreshes its row's total. + +## Evidence + +Fixture: `tool/probe_usage_fixture.dart`, opt-in `?usage=1` on the capture +entry (run 7: 340 MB, 22.4M tokens, mostly cached, $1.42; run 6: partial +≥9.1M). Captures were taken with `tool/capture` at 560x1000 and 1400x900: +the list, the sort sheet, Most tokens, the run 7 Summary and Timeline, the +build and plan visits, and the no-usage (old-server) list and Summary. diff --git a/app/test/features/runs/domain/run_tokens_test.dart b/app/test/features/runs/domain/run_tokens_test.dart index 2bd5097a..7877e796 100644 --- a/app/test/features/runs/domain/run_tokens_test.dart +++ b/app/test/features/runs/domain/run_tokens_test.dart @@ -82,14 +82,17 @@ void main() { ), ), )!; - expect(tokenUsageDetail(usage), '22.4M (20M in, 2.4M out)'); + expect(tokenUsageDetail(usage), '22.4M · 20M in, 2.4M out'); expect(tokenUsageCost(usage), '\$1.75'); final partialCost = runTokenUsage( _run(1, usage: _usage(9, stages: 2, costStages: 1, cost: 0.1)), )!; expect(tokenUsageCost(partialCost), '≥\$0.10'); - expect(tokenUsageCost(runTokenUsage(_run(1, usage: _usage(9)))!), isNull); + expect( + tokenUsageCost(runTokenUsage(_run(1, usage: _usage(9)))!), + 'Not reported', + ); expect( tokenUsageCost( runTokenUsage(_run(1, usage: _usage(9, costStages: 1, cost: 0.0042)))!, @@ -98,6 +101,52 @@ void main() { ); }); + test('the detail splits fresh from cached when every stage reported it', () { + final usage = runTokenUsage( + _run( + 1, + usage: { + ..._usage(22400000, split: true, input: 900000, output: 150000), + 'cache_known': true, + 'cache_read': 21300000, + 'cache_write': 50000, + }, + ), + )!; + expect(tokenUsageLabel(usage), '22.4M tokens'); + expect(usage.fresh, 1100000); + expect( + tokenUsageDetail(usage), + '22.4M · 1.1M fresh, 21.3M cached · 900K in, 150K out', + ); + // An older row (or a partial cache split) has no cache figure: just the + // total and what else it reported. + final old = runTokenUsage( + _run( + 1, + usage: _usage(22400000, split: true, input: 900000, output: 150000), + ), + )!; + expect(old.fresh, isNull); + expect(tokenUsageDetail(old), '22.4M · 900K in, 150K out'); + }); + + test('a zero or missing "known" cost is not reported, never \$0', () { + // Subscription models report cost_known with no positive cost_usd. + for (final usage in [ + _usage(9, costStages: 1), + _usage(9, costStages: 1, cost: 0), + ]) { + final tokens = runTokenUsage(_run(1, usage: usage))!; + expect(tokens.cost, isNull); + expect(tokenUsageCost(tokens), 'Not reported'); + } + final stage = stageTokenUsage({ + 'usage': {'tokens_total': 9, 'tokens_known': true, 'cost_known': true}, + })!; + expect(tokenUsageCost(stage), 'Not reported'); + }); + test('a stage reports its own tokens only when it counted them', () { final stage = stageTokenUsage({ 'usage': { @@ -110,7 +159,7 @@ void main() { 'cost_usd': 0.02, }, })!; - expect(tokenUsageDetail(stage), '1.5K (1.2K in, 300 out)'); + expect(tokenUsageDetail(stage), '1.5K · 1.2K in, 300 out'); expect(tokenUsageCost(stage), '\$0.02'); expect( stageTokenUsage({ diff --git a/app/test/features/runs/ui/run_detail_summary_fields_test.dart b/app/test/features/runs/ui/run_detail_summary_fields_test.dart index edac3ee9..d083a9ca 100644 --- a/app/test/features/runs/ui/run_detail_summary_fields_test.dart +++ b/app/test/features/runs/ui/run_detail_summary_fields_test.dart @@ -98,7 +98,7 @@ void main() { expect(rows.firstWhere((row) => row.label == 'Size on disk').value, '5 MB'); expect( rows.firstWhere((row) => row.label == 'Tokens').value, - '123 (100 in, 23 out)', + '123 · 100 in, 23 out', ); expect(rows.firstWhere((row) => row.label == 'Cost').value, '\$0.50'); expect(rows.firstWhere((row) => row.label == 'Retry of').value, 'run 12'); diff --git a/app/test/features/runs/ui/run_stage_tokens_test.dart b/app/test/features/runs/ui/run_stage_tokens_test.dart index 4c077c4e..ae6e12ac 100644 --- a/app/test/features/runs/ui/run_stage_tokens_test.dart +++ b/app/test/features/runs/ui/run_stage_tokens_test.dart @@ -30,7 +30,7 @@ void main() { final facts = detailVisitFacts(_reported); expect( facts, - contains((label: 'Tokens', value: '1.5M (1.2M in, 300K out)')), + contains((label: 'Tokens', value: '1.5M · 1.2M in, 300K out')), ); expect(facts, contains((label: 'Cost', value: '\$0.42'))); final none = detailVisitFacts(_unreported); diff --git a/app/tool/probe_usage_fixture.dart b/app/tool/probe_usage_fixture.dart new file mode 100644 index 00000000..5b912f77 --- /dev/null +++ b/app/tool/probe_usage_fixture.dart @@ -0,0 +1,92 @@ +import 'package:pipelines_app/core/api_gateway.dart'; + +/// Opt-in (`?usage=1`) size and reported token usage for the capture +/// fixture's runs, shaped exactly as the server sends them: a run-level +/// `usage` total (list and detail) and a per-step `usage`. Without the +/// parameter the fixture keeps the old-server shape with neither. +bool probeUsage = false; + +/// Per-step usage of run 7, by recorded sequence. The build visit is mostly +/// cache reads, as a caching harness reports; plan 1 predates the cache split. +const _stepUsage = { + 5: { + 'tokens_total': 1200000, + 'tokens_known': true, + 'split_known': true, + 'tokens_in': 300000, + 'tokens_out': 60000, + 'tokens_reasoning': 40000, + 'cache_read': 780000, + 'cache_write': 20000, + 'cache_known': true, + 'cost_known': true, + }, + 2: { + 'tokens_total': 20100000, + 'tokens_known': true, + 'split_known': true, + 'tokens_in': 500000, + 'tokens_out': 80000, + 'tokens_reasoning': 20000, + 'cache_read': 19480000, + 'cache_write': 20000, + 'cache_known': true, + 'cost_usd': 1.42, + 'cost_known': true, + }, + 1: { + 'tokens_total': 1100000, + 'tokens_known': true, + 'split_known': true, + 'tokens_in': 1000000, + 'tokens_out': 100000, + 'cache_known': true, + 'cache_read': 0, + 'cost_known': true, + }, +}; + +/// Run 7's total over [_stepUsage], as the server's aggregate sums it. +const _run7Usage = { + 'stages': 3, + 'tokens_total': 22400000, + 'tokens_known_stages': 3, + 'split_known': true, + 'tokens_in': 1800000, + 'tokens_out': 240000, + 'tokens_reasoning': 60000, + 'cache_read': 20260000, + 'cache_write': 40000, + 'cache_known': true, + 'cache_known_stages': 3, + 'cost_usd': 1.42, + 'cost_known_stages': 3, +}; + +/// Run 6: one of its two stages reported no token count, so its total is a +/// lower bound; no stage reported a positive cost. +const _run6Usage = { + 'stages': 2, + 'tokens_total': 9100000, + 'tokens_known_stages': 1, + 'split_known': false, + 'split_unknown_stages': 1, + 'cost_known_stages': 1, + 'cost_unknown_stages': 1, +}; + +/// Adds the run's size and usage to a list row or detail when enabled. +JsonMap withProbeUsage(JsonMap run) { + if (!probeUsage) return run; + return switch (run['id']) { + 7 => {...run, 'size_bytes': 340 * 1024 * 1024, 'usage': _run7Usage}, + 6 => {...run, 'size_bytes': 4404019, 'usage': _run6Usage}, + _ => run, + }; +} + +/// Adds the recorded per-step usage to run 7's steps when enabled. +JsonMap withProbeStepUsage(JsonMap step) { + final usage = _stepUsage[step['sequence']]; + return probeUsage && usage != null ? {...step, 'usage': usage} : step; +} diff --git a/app/tool/search_selection_probe_entry.dart b/app/tool/search_selection_probe_entry.dart index 15365dee..423f1212 100644 --- a/app/tool/search_selection_probe_entry.dart +++ b/app/tool/search_selection_probe_entry.dart @@ -31,6 +31,7 @@ import 'perf/chat_search_fixture.dart'; import 'perf/chat_scroll_fixture.dart'; import 'perf/perf_fixture.dart'; import 'probe_folder_fixture.dart'; +import 'probe_usage_fixture.dart'; /// The run every run-scoped hit names. Its recorded sequences are deliberately /// not its row indexes (plan 5, build 2, plan 1), exactly like the unit @@ -414,12 +415,28 @@ JsonMap? probeReply(JsonMap request) { 'time': '2026-10-20T09:00:00Z', }; case 'list_runs': - return {'id': id, 'type': 'runs', 'runs': probeRuns(), 'usage': null}; + return { + 'id': id, + 'type': 'runs', + 'runs': probeRuns().map(withProbeUsage).toList(), + 'usage': null, + }; case 'get_run': if (request['run_id'] != 7) { return error('Fixture run ${request['run_id']} is unavailable'); } - return {'id': id, 'type': 'run', 'run_detail': probeRunDetail()}; + final detail = withProbeUsage(probeRunDetail()); + return { + 'id': id, + 'type': 'run', + 'run_detail': { + ...detail, + 'steps': [ + for (final step in detail['steps'] as List) + withProbeStepUsage(step as JsonMap), + ], + }, + }; case 'list_pipelines': return {'id': id, 'type': 'pipelines', 'pipelines': probePipelines()}; case 'get_pipeline': @@ -606,6 +623,7 @@ void main() { configurePerfFixture(params); _fillerRunCount = int.tryParse(params['runs'] ?? '') ?? 0; _fillerStepCount = int.tryParse(params['steps'] ?? '') ?? 0; + probeUsage = params['usage'] == '1'; final cupertino = params['platform'] == 'cupertino'; final route = params['route']; final opensRoute = route != null && route.startsWith('/'); diff --git a/docs/cli.md b/docs/cli.md index 3c97a604..4be82346 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -71,8 +71,9 @@ token-reporting stage reported one, and the summed cost, plus counts of stages that did or did not report a cost. A stage whose provider reported no cost is counted as cost-unknown and never contributes a fabricated or zero cost, and no cost/token count is inferred from output length or transcript text. The totals are -the sum of the per-run totals `inspect` shows. `pipelines stats --json` exposes -these aggregates. +the sum of the per-run totals `inspect` shows. Only usage rows whose recorded +step still exists are counted, so rows orphaned by an old history wipe are ignored +(not deleted). `pipelines stats --json` exposes these aggregates. Use `:status` on any TUI page to open server and ntfy information. Scroll with arrows or `j/k`, PgUp/PgDn; `r` refreshes and Esc closes without changing the page. @@ -349,8 +350,23 @@ Every run summary (`list_runs` rows, `get_run`, subscribe snapshots and terminal with turn documents and session exports, server-created Pi session directories, plus an estimate of its database rows). It is cached in `pipeline_runs.size_bytes`, invalidated by triggers when the run changes, measured on completion, and -backfilled in the background; 0 means not measured yet. `delete_runs` removes -finished runs to free space: +backfilled in the background; 0 means not measured yet. + +`list_runs` rows (and `get_run`) also carry `usage`, the run's totalled reported +model usage, in the same shape as the run detail's total: `tokens_total`, +`stages` and `tokens_known_stages` (the total is partial when fewer stages +reported tokens than recorded usage), the `tokens_in`/`tokens_out` split with +`split_known`, the `tokens_reasoning`/`cache_read`/`cache_write` components with +`cache_known` (true only when every token-reporting stage recorded them; +`tokens_total` includes cache reads, which dominate for a caching harness), and +`cost_usd` with `cost_known_stages`/`cost_unknown_stages`. The list computes it +with one grouped SQL aggregate per batch of 100 runs over the stored per-stage +usage rows joined to their recorded steps, never a step load or transcript parse. +It is omitted for a run with no recorded model usage; an older server omits it on +every row, and clients then show nothing. Subscribe snapshots and `run_event` +frames do not carry it, so a running run's total refreshes with the next list. + +`delete_runs` removes finished runs to free space: ```json {"type":"delete_runs","id":"1","run_ids":[12,13,99]} @@ -627,8 +643,12 @@ per-step `provider`/`model`, and the run's graph shows the effective pair). Each model step also records the usage the harness reported, carried from the same accounting the stage already used for its budget: `usage.tokens_total`, the `tokens_in`/`tokens_out` split when the harness reported it (`split_known`), -`cost_usd` when reported, and the explicit `tokens_known`/`cost_known`/ -`split_known` markers. The run detail totals those stages as `usage`, so a +`cost_usd` when reported, the `tokens_reasoning`/`cache_read`/`cache_write` +components when the harness reported the component split (`cache_known`; rows +recorded before these were kept lack it, so their cache share is unknown), and +the explicit `tokens_known`/`cost_known`/`split_known` markers. Some subscription +providers report `cost_known` with no positive `cost_usd`; clients show that cost +as not reported, never as $0. The run detail totals those stages as `usage`, so a mixed-model run shows each stage's model, tokens and cost and the run total. A stage whose provider reported no cost shows its tokens with `cost_known: false` (never a fabricated or zero cost), and a harness that reports only a total (Pi) diff --git a/internal/cli/commands.go b/internal/cli/commands.go index b9136240..ab87ea9e 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -52,7 +52,7 @@ var notes = map[string]string{ "submit": "Confirm publication at the prompt, or use --yes for explicit automation; schedules activate on acceptance. Managed drafts provide the name and base-version guard. --force never bypasses validation.", "run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. --model PROVIDER/MODEL overrides the model for every stage of that run; --stage-model STAGE=PROVIDER/MODEL overrides one stage and is repeatable, with a repeated key last-wins. Precedence is per-stage > whole-run > the model pinned in the pipeline version; overrides apply to that run only and never change the pinned definition. An unknown stage key or a malformed provider/model is refused before the run starts, naming the valid stage keys. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).", "retry": "Resume a failed stage by default, restoring the completed prefix handoff; use --mode=restart to start at entry. --stage selects a recorded stage (use --visit for repeated stages). --input-file accepts a bounded JSON object, merged with replacement at the selected stage; '-' reads stdin. Original history and pinned identity are retained; --version selects an explicit published version of the same pipeline only. --wait-timeout leaves an active retry for later read-only inspection or reattachment with the original --request-key, rather than starting a duplicate retry.", - "runs": "Lists runs newest first; --json includes each run's size_bytes (bytes on disk: artifacts, session exports, server-created Pi session directories and an estimate of its database rows; 0 until first measured). `runs rm ` permanently deletes finished runs (succeeded, failed, canceled, interrupted) with every database row and server-owned file for them, after confirmation (--yes for scripts). Active runs, runs an unfinished retry depends on and unknown ids are refused individually with a reason, and the command then exits 1. User workspaces are never touched. Database pages are reused by SQLite; `retention apply` additionally VACUUMs to shrink the file.", + "runs": "Lists runs newest first; --json includes each run's size_bytes (bytes on disk: artifacts, session exports, server-created Pi session directories and an estimate of its database rows; 0 until first measured) and, when any model stage recorded usage, its usage total (tokens_total, split and cache components, cost). `runs rm ` permanently deletes finished runs (succeeded, failed, canceled, interrupted) with every database row and server-owned file for them, after confirmation (--yes for scripts). Active runs, runs an unfinished retry depends on and unknown ids are refused individually with a reason, and the command then exits 1. User workspaces are never touched. Database pages are reused by SQLite; `retention apply` additionally VACUUMs to shrink the file.", "title": "Rename a run's operator-facing title after the fact without touching its input, steps or execution. The title is collapsed to a single bounded line and recorded as explicitly set. `pipelines runs` and `pipelines inspect` then show it.", "wait": "Block until each named run reaches a terminal status - succeeded, failed, canceled or interrupted, and also through a run waiting on a person - then print one line per run with its id, status and outcome, its stage and how long it took, and exit 0 even for a run that finished unsuccessfully. Several ids wait together in one command. The exit is non-zero only for a client-side failure: an unknown run id, a connection that could not be established, or --timeout expiring while a run is still active; a timeout says so plainly, never cancels, retries or otherwise touches a run, and is never reported as its outcome. Read-only: it needs no workspace, is not refused by the workspace guard and writes nothing. --json prints the final run envelope instead of the line (one envelope for one id, an array for several).", "decide": "Choose an option for a run waiting on a human stage. The run must be `waiting`; the option must be one of that stage's non-error edges. The stage is inferred from the run's current stage unless named. The choice and optional --feedback are recorded as the step's evidence (visible in inspect and the TUI) and the run resumes along the chosen edge. An unknown option is refused with the valid list; a repeated decision for an already-decided visit does not double-apply. Works across a server restart because the wait is persisted.", diff --git a/internal/pipeline/run_usage.go b/internal/pipeline/run_usage.go index 2bfe5e26..4050e433 100644 --- a/internal/pipeline/run_usage.go +++ b/internal/pipeline/run_usage.go @@ -11,7 +11,8 @@ import ( // per bounded batch over the stored per-stage usage rows: no GetRun, step load // or transcript parse per row. It uses the same column sums as the // history-wide totals, so a listed run's total equals its detail's TotalUsage -// over the same rows. A run with no recorded usage keeps Usage nil, and a +// over the same rows; a usage row without its recorded step is never counted. +// A run with no recorded usage keeps Usage nil, and a // database without the step_usage table lists runs without usage. func loadRunUsage(ctx context.Context, tx runGraphQuerier, runs []protocol.RunSummary) error { const batchSize = 100 @@ -23,8 +24,8 @@ func loadRunUsage(ctx context.Context, tx runGraphQuerier, runs []protocol.RunSu byID[runs[i].ID] = &runs[i] args = append(args, runs[i].ID) } - rows, err := tx.QueryContext(ctx, `SELECT run_id,`+usageSumColumns+` -FROM step_usage WHERE run_id IN (`+strings.TrimSuffix(strings.Repeat("?,", len(args)), ",")+`) GROUP BY run_id`, args...) + rows, err := tx.QueryContext(ctx, `SELECT u.run_id,`+usageSumColumns+` `+usageFrom+` +WHERE u.run_id IN (`+strings.TrimSuffix(strings.Repeat("?,", len(args)), ",")+`) GROUP BY u.run_id`, args...) if err != nil { if strings.Contains(err.Error(), "no such table") { return nil @@ -34,7 +35,7 @@ FROM step_usage WHERE run_id IN (`+strings.TrimSuffix(strings.Repeat("?,", len(a for rows.Next() { var id int64 var u protocol.RunUsage - if err := rows.Scan(&id, &u.Stages, &u.TokensTotal, &u.TokensKnownStages, &u.TokensIn, &u.TokensOut, &u.CostUSD, &u.CostKnownStages, &u.SplitUnknownStages); err != nil { + if err := rows.Scan(&id, &u.Stages, &u.TokensTotal, &u.TokensKnownStages, &u.TokensIn, &u.TokensOut, &u.CostUSD, &u.CostKnownStages, &u.SplitUnknownStages, &u.TokensReasoning, &u.CacheRead, &u.CacheWrite, &u.CacheKnownStages); err != nil { rows.Close() return err } @@ -43,6 +44,7 @@ FROM step_usage WHERE run_id IN (`+strings.TrimSuffix(strings.Repeat("?,", len(a } u.CostUnknownStages = u.Stages - u.CostKnownStages u.SplitKnown = u.TokensKnownStages > 0 && u.SplitUnknownStages == 0 + u.CacheKnown = u.TokensKnownStages > 0 && u.CacheKnownStages == u.TokensKnownStages byID[id].Usage = &u } err = rows.Err() diff --git a/internal/pipeline/run_usage_test.go b/internal/pipeline/run_usage_test.go index 72f4ba8b..d80a760c 100644 --- a/internal/pipeline/run_usage_test.go +++ b/internal/pipeline/run_usage_test.go @@ -89,3 +89,88 @@ func TestListRunsCarriesRunUsageAggregate(t *testing.T) { t.Fatalf("filler totals = %d", fillers) } } + +// TestRunUsageCountsCacheAndOnlyRecordedSteps proves the cache/reasoning +// components are summed per run (equal in list, detail and stats), a run mixing +// a pre-cache row with a cache-reporting one keeps CacheKnown false, and a usage +// row whose step or run no longer exists (an old history wipe) is counted +// nowhere and left in place. +func TestRunUsageCountsCacheAndOnlyRecordedSteps(t *testing.T) { + ctx := context.Background() + store, err := Open(filepath.Join(t.TempDir(), "cache-usage.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, err := store.Submit(ctx, "cache-usage", Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + items, _ := store.List(ctx) + definition, _ := store.Get(ctx, items[0].ID, 1) + cached, _, err := store.CreateRun(ctx, definition, "cached", "/repo", false, nil, nil) + if err != nil { + t.Fatal(err) + } + finishUsageStep(t, store, cached, 1, "one", &protocol.StageUsage{TokensIn: 900_000, TokensOut: 150_000, TokensTotal: 22_400_000, TokensKnown: true, SplitKnown: true, TokensReasoning: 50_000, CacheRead: 21_250_000, CacheWrite: 50_000, CacheKnown: true}) + mixed, _, err := store.CreateRun(ctx, definition, "mixed", "/repo", false, nil, nil) + if err != nil { + t.Fatal(err) + } + finishUsageStep(t, store, mixed, 1, "one", &protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true}) + finishUsageStep(t, store, mixed, 2, "one", &protocol.StageUsage{TokensIn: 1, TokensOut: 1, TokensTotal: 10, TokensKnown: true, SplitKnown: true, CacheRead: 8, CacheKnown: true}) + + // Orphans: one for a run that no longer exists and one for a step the + // listed run never recorded. Foreign keys are off only on this connection. + conn, err := store.db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=OFF"); err != nil { + t.Fatal(err) + } + orphan := `{"tokens_total":1000000000,"tokens_known":true,"cost_usd":15.67,"cost_known":true}` + for _, key := range [][2]int64{{9999, 1}, {cached, 99}} { + if _, err := conn.ExecContext(ctx, "INSERT INTO step_usage(run_id,sequence,usage_json) VALUES(?,?,?)", key[0], key[1], orphan); err != nil { + t.Fatal(err) + } + } + if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=ON"); err != nil { + t.Fatal(err) + } + conn.Close() + + runs, err := store.ListRuns(ctx) + if err != nil { + t.Fatal(err) + } + listed := map[int64]*protocol.RunUsage{} + for _, run := range runs { + listed[run.ID] = run.Usage + } + for _, id := range []int64{cached, mixed} { + detail, err := store.GetRun(ctx, id) + if err != nil { + t.Fatal(err) + } + if listed[id] == nil || detail.Usage == nil || *listed[id] != *detail.Usage { + t.Fatalf("run %d list usage %+v != detail usage %+v", id, listed[id], detail.Usage) + } + } + if got := listed[cached]; got.TokensTotal != 22_400_000 || got.CacheRead != 21_250_000 || got.CacheWrite != 50_000 || got.TokensReasoning != 50_000 || !got.CacheKnown || got.CacheKnownStages != 1 || got.Stages != 1 || got.CostKnownStages != 0 { + t.Fatalf("cached run usage counted an orphan or lost the cache split: %+v", got) + } + if got := listed[mixed]; got.CacheKnown || got.CacheKnownStages != 1 || got.CacheRead != 8 || got.TokensTotal != 17 { + t.Fatalf("mixed run presented a partial cache split as complete: %+v", got) + } + stats, err := store.UsageStats(ctx) + if err != nil || stats.Usage == nil { + t.Fatalf("stats = %+v err=%v", stats, err) + } + if got := stats.Usage; got.Stages != 3 || got.TokensTotal != 22_400_017 || got.CostUSD != 0 || got.CostKnownStages != 0 || got.CacheRead != 21_250_008 || got.CacheKnown { + t.Fatalf("history totals counted orphan rows: %+v", got) + } + var remaining int + if err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM step_usage").Scan(&remaining); err != nil || remaining != 5 { + t.Fatalf("usage rows were deleted: %d %v", remaining, err) + } +} diff --git a/internal/pipeline/stats.go b/internal/pipeline/stats.go index 15ffecce..4931e7e1 100644 --- a/internal/pipeline/stats.go +++ b/internal/pipeline/stats.go @@ -66,7 +66,8 @@ func (s *Store) UsageStats(ctx context.Context) (*protocol.UsageStats, error) { // usageSumColumns sums step_usage rows into the RunUsage/UsageTotals shape: // stages, reported total, token-reporting stages, split in/out, cost, -// cost-reporting stages and token stages without a split, in that order. It is +// cost-reporting stages, token stages without a split, then reasoning, cache +// read and cache write sums and the stages that recorded them, in that order. It is // the one SQL definition of the sum, shared by the history-wide totals and the // per-run list aggregate, and mirrors protocol.TotalUsage over the same rows. const usageSumColumns = `COALESCE(COUNT(*),0), @@ -76,7 +77,17 @@ COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.split_known') THEN json_extrac COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.split_known') THEN json_extract(usage_json,'$.tokens_out') ELSE 0 END),0), COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.cost_known') THEN json_extract(usage_json,'$.cost_usd') ELSE 0 END),0), COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.cost_known') THEN 1 ELSE 0 END),0), -COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND NOT COALESCE(json_extract(usage_json,'$.split_known'),0) THEN 1 ELSE 0 END),0)` +COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND NOT COALESCE(json_extract(usage_json,'$.split_known'),0) THEN 1 ELSE 0 END),0), +COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND json_extract(usage_json,'$.cache_known') THEN COALESCE(json_extract(usage_json,'$.tokens_reasoning'),0) ELSE 0 END),0), +COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND json_extract(usage_json,'$.cache_known') THEN COALESCE(json_extract(usage_json,'$.cache_read'),0) ELSE 0 END),0), +COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND json_extract(usage_json,'$.cache_known') THEN COALESCE(json_extract(usage_json,'$.cache_write'),0) ELSE 0 END),0), +COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND json_extract(usage_json,'$.cache_known') THEN 1 ELSE 0 END),0)` + +// usageFrom is the step_usage rows that belong to a recorded step. A usage row +// whose step (or run) no longer exists, such as one left behind by an old +// history wipe, is never counted, and is not deleted here either. It is the +// same row set the run detail reads through its step loop. +const usageFrom = `FROM step_usage u JOIN run_steps s ON s.run_id=u.run_id AND s.sequence=u.sequence` // usageTotals is the one extra aggregate query the history-wide usage totals add, // over the same step_usage rows the per-run figures read. It sums reported totals @@ -85,8 +96,7 @@ COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND NOT COALESC // read-only database with no step_usage table yields no totals, not an error. func usageTotals(ctx context.Context, tx *sql.Tx) (*protocol.UsageTotals, error) { var totals protocol.UsageTotals - err := tx.QueryRowContext(ctx, `SELECT `+usageSumColumns+` -FROM step_usage`).Scan(&totals.Stages, &totals.TokensTotal, &totals.TokensKnownStages, &totals.TokensIn, &totals.TokensOut, &totals.CostUSD, &totals.CostKnownStages, &totals.SplitUnknownStages) + err := tx.QueryRowContext(ctx, `SELECT `+usageSumColumns+` `+usageFrom).Scan(&totals.Stages, &totals.TokensTotal, &totals.TokensKnownStages, &totals.TokensIn, &totals.TokensOut, &totals.CostUSD, &totals.CostKnownStages, &totals.SplitUnknownStages, &totals.TokensReasoning, &totals.CacheRead, &totals.CacheWrite, &totals.CacheKnownStages) if err != nil { if strings.Contains(err.Error(), "no such table") { return nil, nil @@ -98,5 +108,6 @@ FROM step_usage`).Scan(&totals.Stages, &totals.TokensTotal, &totals.TokensKnownS } totals.CostUnknownStages = totals.Stages - totals.CostKnownStages totals.SplitKnown = totals.TokensKnownStages > 0 && totals.SplitUnknownStages == 0 + totals.CacheKnown = totals.TokensKnownStages > 0 && totals.CacheKnownStages == totals.TokensKnownStages return &totals, nil } diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index 86a130de..49a91844 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -325,6 +325,14 @@ type StageUsage struct { SplitKnown bool `json:"split_known,omitempty"` CostUSD float64 `json:"cost_usd,omitempty"` CostKnown bool `json:"cost_known"` + // TokensReasoning, CacheRead and CacheWrite are the remaining components of + // a split report (TokensTotal = in + out + reasoning + cache read + cache + // write). They are recorded only when CacheKnown; a row stored before they + // were kept has CacheKnown=false, so its absent counts are unknown, not zero. + TokensReasoning int64 `json:"tokens_reasoning,omitempty"` + CacheRead int64 `json:"cache_read,omitempty"` + CacheWrite int64 `json:"cache_write,omitempty"` + CacheKnown bool `json:"cache_known,omitempty"` } // RunUsage is the totalled reported usage of a run's model stages. It is summed @@ -344,6 +352,15 @@ type RunUsage struct { SplitUnknownStages int `json:"split_unknown_stages,omitempty"` CostKnownStages int `json:"cost_known_stages"` CostUnknownStages int `json:"cost_unknown_stages,omitempty"` + // TokensReasoning, CacheRead and CacheWrite sum the stages that recorded the + // component split (CacheKnownStages). CacheKnown is true only when every + // token-reporting stage did, so fresh (total minus cache read) versus cached + // is never presented from a partial split. + TokensReasoning int64 `json:"tokens_reasoning,omitempty"` + CacheRead int64 `json:"cache_read,omitempty"` + CacheWrite int64 `json:"cache_write,omitempty"` + CacheKnown bool `json:"cache_known,omitempty"` + CacheKnownStages int `json:"cache_known_stages,omitempty"` } // TotalUsage sums the per-stage usage of a run's steps into one run total. It is @@ -368,6 +385,12 @@ func TotalUsage(steps []RunStep) *RunUsage { splitComplete = false total.SplitUnknownStages++ } + if usage.CacheKnown { + total.CacheKnownStages++ + total.TokensReasoning += usage.TokensReasoning + total.CacheRead += usage.CacheRead + total.CacheWrite += usage.CacheWrite + } } if usage.CostKnown { total.CostKnownStages++ @@ -380,6 +403,7 @@ func TotalUsage(steps []RunStep) *RunUsage { return nil } total.SplitKnown = total.TokensKnownStages > 0 && splitComplete + total.CacheKnown = total.TokensKnownStages > 0 && total.CacheKnownStages == total.TokensKnownStages return total } diff --git a/internal/protocol/stats.go b/internal/protocol/stats.go index 8c8c11a5..2d4a576c 100644 --- a/internal/protocol/stats.go +++ b/internal/protocol/stats.go @@ -37,6 +37,12 @@ type UsageTotals struct { SplitUnknownStages int `json:"split_unknown_stages,omitempty"` CostKnownStages int `json:"cost_known_stages"` CostUnknownStages int `json:"cost_unknown_stages,omitempty"` + // The cache/reasoning components, as in RunUsage. + TokensReasoning int64 `json:"tokens_reasoning,omitempty"` + CacheRead int64 `json:"cache_read,omitempty"` + CacheWrite int64 `json:"cache_write,omitempty"` + CacheKnown bool `json:"cache_known,omitempty"` + CacheKnownStages int `json:"cache_known_stages,omitempty"` } type UsageStats struct { diff --git a/internal/server/runner.go b/internal/server/runner.go index 6c4dd2bb..3e3f7292 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -1448,6 +1448,10 @@ func stageUsage(snapshot stage.BudgetSnapshot) *protocol.StageUsage { usage.TokensTotal = snapshot.UsedTokens if snapshot.SplitKnown { usage.TokensIn, usage.TokensOut, usage.SplitKnown = snapshot.TokensInput, snapshot.TokensOutput, true + // The rest of the component split is kept too: for a caching harness + // most of the total is cache reads, which a client shows as cached + // rather than fresh tokens. + usage.TokensReasoning, usage.CacheRead, usage.CacheWrite, usage.CacheKnown = snapshot.TokensReasoning, snapshot.CacheRead, snapshot.CacheWrite, true } } if snapshot.USDKnown { diff --git a/internal/server/usage_test.go b/internal/server/usage_test.go index 200bce17..f5200719 100644 --- a/internal/server/usage_test.go +++ b/internal/server/usage_test.go @@ -31,11 +31,13 @@ func TestStageUsageCarriesHonestUnknowns(t *testing.T) { }{ {"no usage stays absent", stage.BudgetSnapshot{}, nil}, {"split and cost", stage.BudgetSnapshot{UsedTokens: 7, TokensKnown: true, SplitKnown: true, TokensInput: 3, TokensOutput: 4, UsedUSD: .5, USDKnown: true}, - &protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true, CostUSD: .5, CostKnown: true}}, + &protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true, CostUSD: .5, CostKnown: true, CacheKnown: true}}, + {"cache and reasoning components carry through", stage.BudgetSnapshot{UsedTokens: 22_400_000, TokensKnown: true, SplitKnown: true, TokensInput: 900_000, TokensOutput: 150_000, TokensReasoning: 50_000, CacheRead: 21_250_000, CacheWrite: 50_000}, + &protocol.StageUsage{TokensIn: 900_000, TokensOut: 150_000, TokensTotal: 22_400_000, TokensKnown: true, SplitKnown: true, TokensReasoning: 50_000, CacheRead: 21_250_000, CacheWrite: 50_000, CacheKnown: true}}, {"total only (Pi)", stage.BudgetSnapshot{UsedTokens: 17, TokensKnown: true}, &protocol.StageUsage{TokensTotal: 17, TokensKnown: true}}, {"no cost", stage.BudgetSnapshot{UsedTokens: 5, TokensKnown: true, SplitKnown: true, TokensInput: 5}, - &protocol.StageUsage{TokensIn: 5, TokensTotal: 5, TokensKnown: true, SplitKnown: true}}, + &protocol.StageUsage{TokensIn: 5, TokensTotal: 5, TokensKnown: true, SplitKnown: true, CacheKnown: true}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) {