From aa948bdfe4eee0d7b732263fb2e9a4874a6fe880 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Tue, 3 Nov 2026 22:04:55 +0000 Subject: [PATCH] Runs list carries per-run token usage; app shows tokens beside size, Most tokens sort, Summary and stage tokens --- app/lib/features/runs/domain/run_models.dart | 14 ++ app/lib/features/runs/domain/run_tokens.dart | 145 ++++++++++++++++++ .../features/runs/ui/run_detail_content.dart | 6 + .../runs/ui/run_detail_summary_fields.dart | 18 ++- .../runs/ui/run_detail_tab_bodies.dart | 2 +- .../features/runs/ui/run_detail_timeline.dart | 13 +- .../runs/ui/run_detail_visit_facts.dart | 7 + app/lib/features/runs/ui/run_list_row.dart | 6 + app/lib/features/runs/ui/runs_controller.dart | 9 +- .../features/runs/ui/runs_filter_options.dart | 5 +- .../features/runs/domain/run_tokens_test.dart | 142 +++++++++++++++++ .../ui/run_detail_summary_fields_test.dart | 10 ++ .../features/runs/ui/run_list_row_test.dart | 42 +++++ .../runs/ui/run_stage_tokens_test.dart | 46 ++++++ internal/pipeline/run_usage.go | 55 +++++++ internal/pipeline/run_usage_test.go | 91 +++++++++++ internal/pipeline/runs.go | 3 + internal/pipeline/stats.go | 24 +-- internal/protocol/runs.go | 12 +- 19 files changed, 620 insertions(+), 30 deletions(-) create mode 100644 app/lib/features/runs/domain/run_tokens.dart create mode 100644 app/test/features/runs/domain/run_tokens_test.dart create mode 100644 app/test/features/runs/ui/run_stage_tokens_test.dart create mode 100644 internal/pipeline/run_usage.go create mode 100644 internal/pipeline/run_usage_test.go diff --git a/app/lib/features/runs/domain/run_models.dart b/app/lib/features/runs/domain/run_models.dart index 118966b2..90bc23b7 100644 --- a/app/lib/features/runs/domain/run_models.dart +++ b/app/lib/features/runs/domain/run_models.dart @@ -1,5 +1,6 @@ import '../../pipelines/pipeline_format.dart'; import '../../../core/api_gateway.dart'; +import 'run_tokens.dart'; const runStates = [ 'queued', @@ -268,6 +269,19 @@ List sortRuns(List runs, {String sort = 'newest_first'}) { primary = cmp(bt ?? -_maxMillis, at ?? -_maxMillis); } break; + case 'most_tokens': + // Runs with no reported tokens (older servers, no model stage) sort + // after every reported total; a partial total sorts by what was + // reported. + final ta = runTokenUsage(a)?.total, tb = runTokenUsage(b)?.total; + primary = cmp(ta == null ? 1 : 0, tb == null ? 1 : 0); + if (primary == 0 && ta != null && tb != null) { + primary = tb.compareTo(ta); + } + if (primary == 0) { + primary = cmp(bt ?? -_maxMillis, at ?? -_maxMillis); + } + break; case 'pipeline_name': primary = '${a['name'] ?? ''}'.toLowerCase().compareTo( '${b['name'] ?? ''}'.toLowerCase(), diff --git a/app/lib/features/runs/domain/run_tokens.dart b/app/lib/features/runs/domain/run_tokens.dart new file mode 100644 index 00000000..034d7677 --- /dev/null +++ b/app/lib/features/runs/domain/run_tokens.dart @@ -0,0 +1,145 @@ +import '../../../core/api_gateway.dart'; + +/// Reported token and cost usage of a run or one stage. No widgets here. +/// +/// Every figure is what the server recorded from the harness: nothing is +/// estimated. A run total is partial when some stages that recorded usage +/// reported no token count, and it is then labelled as a lower bound (`≥`). +class TokenUsage { + const TokenUsage({ + required this.total, + required this.partial, + this.input, + this.output, + this.cost, + this.costPartial = false, + this.stages, + this.tokenStages, + }); + + /// The summed reported total. + final int total; + + /// Some stages that recorded usage reported no token count. + final bool partial; + + /// The input/output split, only when every counted stage reported it. + final int? input; + final int? output; + + /// The reported cost in USD, only when some stage reported one. + final double? cost; + + /// Some stages reported no cost, so [cost] is a lower bound. + final bool costPartial; + + /// Run totals only: the stages that recorded usage, and those of them that + /// reported tokens. + final int? stages; + final int? tokenStages; +} + +/// The run's reported token usage from its `usage` total, or null when the +/// server sent none (an older server's list, or a run with no model usage) or +/// no stage reported a token count. +TokenUsage? runTokenUsage(JsonMap run) { + final raw = run['usage']; + if (raw is! Map) return null; + final stages = _int(raw['stages']); + final tokenStages = _int(raw['tokens_known_stages']) ?? 0; + final total = _int(raw['tokens_total']); + if (tokenStages <= 0 || total == null) return null; + final costStages = _int(raw['cost_known_stages']) ?? 0; + final split = raw['split_known'] == true; + return TokenUsage( + total: total, + partial: stages != null && tokenStages < stages, + // The server omits a zero in/out, so a known split makes absence zero. + input: split ? _int(raw['tokens_in']) ?? 0 : null, + output: split ? _int(raw['tokens_out']) ?? 0 : null, + cost: costStages > 0 ? _double(raw['cost_usd']) ?? 0 : null, + costPartial: costStages > 0 && stages != null && costStages < stages, + stages: stages, + tokenStages: tokenStages, + ); +} + +/// One stage's reported token usage from its step `usage`, or null when the +/// stage reported no token count. +TokenUsage? stageTokenUsage(JsonMap step) { + final raw = step['usage']; + if (raw is! Map || raw['tokens_known'] != true) return null; + final total = _int(raw['tokens_total']); + if (total == null) return null; + final split = raw['split_known'] == true; + return TokenUsage( + total: total, + partial: false, + input: split ? _int(raw['tokens_in']) ?? 0 : null, + output: split ? _int(raw['tokens_out']) ?? 0 : null, + cost: raw['cost_known'] == true ? _double(raw['cost_usd']) ?? 0 : null, + ); +} + +/// A compact token count: `950`, `1.2K`, `340K`, `22.4M`, `1.3B`. +/// +/// Decimal multiples, one decimal below a hundred units, whole units above. +String formatTokenCount(int tokens) { + if (tokens < 1000) return '$tokens'; + const units = ['K', 'M', 'B', 'T']; + var value = tokens / 1000; + var unit = 0; + while (value >= 1000 && unit < units.length - 1) { + value /= 1000; + unit++; + } + // Rounding 999.6K up would print "1000K"; promote it instead. + if (value >= 999.5 && unit < units.length - 1) { + value /= 1000; + unit++; + } + final text = value < 99.95 + ? value.toStringAsFixed(1).replaceFirst(RegExp(r'\.0$'), '') + : value.round().toString(); + return '$text${units[unit]}'; +} + +/// The list row's compact label: `22.4M tokens`, or `≥22.4M tokens` when some +/// stages did not report a count. +String tokenUsageLabel(TokenUsage usage) => + '${usage.partial ? '≥' : ''}${formatTokenCount(usage.total)} tokens'; + +/// The detail's token value: the compact total, the in/out split when +/// reported, and how many stages reported when the total is partial. +/// `22.4M (20M in, 2.4M out)`, `≥9 · 1 of 2 stages reported`. +String tokenUsageDetail(TokenUsage usage) { + final parts = [ + [ + '${usage.partial ? '≥' : ''}${formatTokenCount(usage.total)}', + if (usage.input != null && usage.output != null) + '(${formatTokenCount(usage.input!)} in, ' + '${formatTokenCount(usage.output!)} out)', + ].join(' '), + if (usage.partial && usage.stages != null) + '${usage.tokenStages} of ${usage.stages} stages reported', + ]; + return parts.join(' · '); +} + +/// The reported cost, or null when no stage reported one: `$1.75`, `$0.10`, +/// `$0.0042` (four places below a cent), +/// `≥$0.10` when some stages reported no cost. +String? tokenUsageCost(TokenUsage usage) { + final cost = usage.cost; + if (cost == null) return null; + final text = cost >= 0.01 || cost == 0 + ? cost.toStringAsFixed(2) + : cost.toStringAsFixed(4); + return '${usage.costPartial ? '≥' : ''}\$$text'; +} + +int? _int(dynamic value) => + value is num && value.isFinite && value >= 0 ? value.round() : null; + +double? _double(dynamic value) => + value is num && value.isFinite && value >= 0 ? value.toDouble() : null; diff --git a/app/lib/features/runs/ui/run_detail_content.dart b/app/lib/features/runs/ui/run_detail_content.dart index 1412fc13..d2605a32 100644 --- a/app/lib/features/runs/ui/run_detail_content.dart +++ b/app/lib/features/runs/ui/run_detail_content.dart @@ -3,6 +3,7 @@ import 'dart:convert'; import '../../../core/api_gateway.dart'; import '../../pipelines/pipeline_format.dart' show formatDuration; import '../domain/run_models.dart'; +import '../domain/run_tokens.dart'; /// One labelled value in the run detail's metrics block. typedef DetailMetric = ({String label, String value}); @@ -23,6 +24,7 @@ typedef DetailVisit = ({ String outcome, String target, String? duration, + String? tokens, }); /// The run's primary label: the stored title, else the pipeline name, and only @@ -149,6 +151,10 @@ List detailTimeline(dynamic steps) { outcome: '${step['outcome'] ?? ''}'.trim(), target: '${step['target'] ?? ''}'.trim(), duration: _visitDuration(step), + tokens: switch (stageTokenUsage(JsonMap.from(step))) { + final usage? => tokenUsageLabel(usage), + null => null, + }, ); }(), ]; diff --git a/app/lib/features/runs/ui/run_detail_summary_fields.dart b/app/lib/features/runs/ui/run_detail_summary_fields.dart index 0234eb57..3124c529 100644 --- a/app/lib/features/runs/ui/run_detail_summary_fields.dart +++ b/app/lib/features/runs/ui/run_detail_summary_fields.dart @@ -1,5 +1,7 @@ import '../../../core/api_gateway.dart'; import '../../pipelines/pipeline_format.dart'; +import '../domain/run_deletion.dart' show formatByteSize, runSizeBytes; +import '../domain/run_tokens.dart'; import 'run_detail_visit_facts.dart'; /// One recorded Summary row: the web run summary's key/value shape. @@ -14,9 +16,9 @@ typedef DetailSummaryRow = ({ bool block, }); -/// The run Summary's rows, in the web summary's order: Pipeline, Stage, -/// Workspace, Content hash, Started, Finished, Duration, Retry of and the -/// Diagnostic block. +/// The run Summary's rows: Pipeline, Stage, Workspace, Content hash, Started, +/// Finished, Duration, Size on disk, Tokens, Cost, Retry of and the Diagnostic +/// block. /// /// Only values the server actually recorded are rows: a blank field is omitted /// rather than drawn as an empty or invented placeholder, exactly as @@ -53,6 +55,16 @@ List detailSummaryRows(JsonMap run) { _text(run['finished_at']), ); if (seconds != null) add('Duration', formatDuration(seconds)); + // What the run occupies on disk and what it spent, as the server measured + // and the harness reported; an unmeasured size or unreported usage has no + // row. + final bytes = runSizeBytes(run); + if (bytes != null) add('Size on disk', formatByteSize(bytes)); + final usage = runTokenUsage(run); + if (usage != null) { + add('Tokens', tokenUsageDetail(usage)); + add('Cost', tokenUsageCost(usage)); + } final retryOf = _number(run['retry_of'])?.toInt() ?? 0; if (retryOf > 0) add('Retry of', 'run $retryOf'); // A diagnostic is a recorded document: its own whitespace is preserved, but a diff --git a/app/lib/features/runs/ui/run_detail_tab_bodies.dart b/app/lib/features/runs/ui/run_detail_tab_bodies.dart index fd5ae2c6..753f62f8 100644 --- a/app/lib/features/runs/ui/run_detail_tab_bodies.dart +++ b/app/lib/features/runs/ui/run_detail_tab_bodies.dart @@ -16,7 +16,7 @@ import '../../../ui/adaptive/floating_bar_layout.dart'; /// the stored original prompt with its copy control and the collapsible Input /// and Output sections. /// -/// The web run detail has no usage block, so no metrics are drawn here. A fact +/// The run's size and reported token usage are facts like the others. A fact /// the server did not record is omitted rather than shown as an invented or /// `Unavailable` row; only the Input/Output empty sentences remain, and only /// when that section recorded nothing. diff --git a/app/lib/features/runs/ui/run_detail_timeline.dart b/app/lib/features/runs/ui/run_detail_timeline.dart index 60372328..00a68093 100644 --- a/app/lib/features/runs/ui/run_detail_timeline.dart +++ b/app/lib/features/runs/ui/run_detail_timeline.dart @@ -135,8 +135,8 @@ class RunTimelineRow extends StatelessWidget { @override Widget build(BuildContext context) { final state = _state; - // The outcome chip leads the second line, then where the visit went; the - // duration is the row's time. + // The outcome chip leads the second line, then where the visit went and + // the tokens it reported; the duration is the row's time. return AppListRow( cupertino: cupertino, selected: selected, @@ -152,7 +152,14 @@ class RunTimelineRow extends StatelessWidget { title: Text(visit.title), time: visit.duration, subtitleLeading: _outcome(context), - subtitle: visit.target.isEmpty ? null : Text('→ ${visit.target}'), + subtitle: visit.target.isEmpty && visit.tokens == null + ? null + : Text( + [ + if (visit.target.isNotEmpty) '→ ${visit.target}', + ?visit.tokens, + ].join(' · '), + ), ); } } diff --git a/app/lib/features/runs/ui/run_detail_visit_facts.dart b/app/lib/features/runs/ui/run_detail_visit_facts.dart index a631d183..17767453 100644 --- a/app/lib/features/runs/ui/run_detail_visit_facts.dart +++ b/app/lib/features/runs/ui/run_detail_visit_facts.dart @@ -1,6 +1,7 @@ import '../../../core/api_gateway.dart'; import '../../graph/domain/stage_definition.dart'; import '../../pipelines/pipeline_format.dart'; +import '../domain/run_tokens.dart'; import 'run_detail_content.dart'; /// The exact empty wording for a visit's recorded Input and Output. They are @@ -80,6 +81,12 @@ List detailVisitFacts(JsonMap step) { if (duration != null) facts.add((label: 'Duration', value: duration)); final model = detailVisitProviderModel(step); if (model != null) facts.add((label: 'Provider/model', value: model)); + final usage = stageTokenUsage(step); + if (usage != null) { + facts.add((label: 'Tokens', value: tokenUsageDetail(usage))); + final cost = tokenUsageCost(usage); + if (cost != null) facts.add((label: 'Cost', value: cost)); + } final target = _text(step['target']); if (target != null) facts.add((label: 'Target', value: target)); return facts; diff --git a/app/lib/features/runs/ui/run_list_row.dart b/app/lib/features/runs/ui/run_list_row.dart index 36f1d065..e662fb5a 100644 --- a/app/lib/features/runs/ui/run_list_row.dart +++ b/app/lib/features/runs/ui/run_list_row.dart @@ -9,6 +9,7 @@ import '../../../ui/status/run_status_icon.dart'; import '../../../ui/text/relative_time_label.dart'; import '../domain/run_deletion.dart'; import '../domain/run_models.dart'; +import '../domain/run_tokens.dart'; import 'run_more_menu.dart'; /// One run in the list: a status avatar, title/time, and one muted fact @@ -77,6 +78,9 @@ class RunListRow extends StatelessWidget { final scheduled = run['scheduled'] == true; final bytes = runSizeBytes(run); final size = bytes == null ? null : formatByteSize(bytes); + // Reported tokens sit beside the size; an older server sends none. + final usage = runTokenUsage(run); + final tokens = usage == null ? null : tokenUsageLabel(usage); final createdAt = DateTime.tryParse('${run['created_at'] ?? ''}'); final subtitle = [ ?pipelineLabel, @@ -84,6 +88,7 @@ class RunListRow extends StatelessWidget { ?facts.duration, ?facts.progress, ?size, + ?tokens, ].join(' · '); // A narrow pane keeps the decision facts: the pipeline, and progress // while executing (duration only when there is no pipeline to show). @@ -93,6 +98,7 @@ class RunListRow extends StatelessWidget { if (run['executing'] == true && facts.progress != null) facts.progress!, if (pipelineLabel == null && facts.duration != null) facts.duration!, ?size, + ?tokens, ].join(' · '); final state = runStateOf(run['status']); return AppListRow( diff --git a/app/lib/features/runs/ui/runs_controller.dart b/app/lib/features/runs/ui/runs_controller.dart index 3ac6854d..8f0e2fa8 100644 --- a/app/lib/features/runs/ui/runs_controller.dart +++ b/app/lib/features/runs/ui/runs_controller.dart @@ -162,14 +162,7 @@ class RunsController extends ChangeNotifier { // A cancel/decision reply changes lifecycle fields as well as title. // Keep sparse/unknown summary fields, without copying bulky detail-only // payloads into the held list or inventing missing values. - const detailOnly = { - 'steps', - 'input', - 'output', - 'graph', - 'usage', - 'notifications', - }; + const detailOnly = {'steps', 'input', 'output', 'graph', 'notifications'}; runs = [...runs] ..[index] = { ...runs[index], diff --git a/app/lib/features/runs/ui/runs_filter_options.dart b/app/lib/features/runs/ui/runs_filter_options.dart index 87a57a8f..266dd986 100644 --- a/app/lib/features/runs/ui/runs_filter_options.dart +++ b/app/lib/features/runs/ui/runs_filter_options.dart @@ -7,6 +7,7 @@ const runSorts = [ ('newest_first', 'Newest first'), ('oldest_first', 'Oldest first'), ('largest_first', 'Largest first'), + ('most_tokens', 'Most tokens'), ('longest_duration', 'Longest duration'), ('shortest_duration', 'Shortest duration'), ('pipeline_name', 'Pipeline name'), @@ -25,8 +26,8 @@ const runTimeRanges = [ ]; /// Sorts that list runs flat instead of under day headings: an order by -/// size has no meaningful day grouping. -const flatRunSorts = {'largest_first'}; +/// size or token count has no meaningful day grouping. +const flatRunSorts = {'largest_first', 'most_tokens'}; /// The pipelines the loaded runs belong to, by name then id. List<({int id, String name})> runFilterPipelines(RunsController controller) { diff --git a/app/test/features/runs/domain/run_tokens_test.dart b/app/test/features/runs/domain/run_tokens_test.dart new file mode 100644 index 00000000..2bd5097a --- /dev/null +++ b/app/test/features/runs/domain/run_tokens_test.dart @@ -0,0 +1,142 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/core/api_gateway.dart'; +import 'package:pipelines_app/features/runs/domain/run_models.dart'; +import 'package:pipelines_app/features/runs/domain/run_tokens.dart'; +import 'package:pipelines_app/features/runs/ui/runs_filter_options.dart'; + +JsonMap _run(int id, {Map? usage, String? created}) => { + 'id': id, + 'created_at': created ?? '2026-10-2${id % 10}T12:00:00Z', + 'usage': ?usage, +}; + +Map _usage( + int total, { + int stages = 1, + int? tokenStages, + bool split = false, + int input = 0, + int output = 0, + int costStages = 0, + double? cost, +}) => { + 'stages': stages, + 'tokens_total': total, + 'tokens_known_stages': tokenStages ?? stages, + 'split_known': split, + if (input > 0) 'tokens_in': input, + if (output > 0) 'tokens_out': output, + 'cost_known_stages': costStages, + 'cost_usd': ?cost, +}; + +void main() { + test('token counts are compact decimal multiples', () { + expect(formatTokenCount(0), '0'); + expect(formatTokenCount(950), '950'); + expect(formatTokenCount(1000), '1K'); + expect(formatTokenCount(1234), '1.2K'); + expect(formatTokenCount(340000), '340K'); + expect(formatTokenCount(999600), '1M'); + expect(formatTokenCount(22400015), '22.4M'); + expect(formatTokenCount(1300000000), '1.3B'); + }); + + test('a reported run total is labelled like the size', () { + final usage = runTokenUsage(_run(1, usage: _usage(22400015)))!; + expect(usage.partial, isFalse); + expect(tokenUsageLabel(usage), '22.4M tokens'); + }); + + test('a partial total is a lower bound, never an estimate', () { + final usage = runTokenUsage( + _run(1, usage: _usage(9, stages: 2, tokenStages: 1)), + )!; + expect(usage.partial, isTrue); + expect(tokenUsageLabel(usage), '≥9 tokens'); + expect(tokenUsageDetail(usage), '≥9 · 1 of 2 stages reported'); + }); + + test('absent or unreported usage shows nothing', () { + expect(runTokenUsage(_run(1)), isNull); + expect(runTokenUsage({'id': 1, 'usage': 'bogus'}), isNull); + // Only a cost was reported: no token figure is invented. + expect( + runTokenUsage(_run(1, usage: _usage(0, tokenStages: 0, costStages: 1))), + isNull, + ); + }); + + test('the detail adds the split and the reported cost', () { + final usage = runTokenUsage( + _run( + 1, + usage: _usage( + 22400015, + stages: 2, + split: true, + input: 20000010, + output: 2400005, + costStages: 2, + cost: 1.75, + ), + ), + )!; + expect(tokenUsageDetail(usage), '22.4M (20M in, 2.4M out)'); + expect(tokenUsageCost(usage), '\$1.75'); + + final partialCost = runTokenUsage( + _run(1, usage: _usage(9, stages: 2, costStages: 1, cost: 0.1)), + )!; + expect(tokenUsageCost(partialCost), '≥\$0.10'); + expect(tokenUsageCost(runTokenUsage(_run(1, usage: _usage(9)))!), isNull); + expect( + tokenUsageCost( + runTokenUsage(_run(1, usage: _usage(9, costStages: 1, cost: 0.0042)))!, + ), + '\$0.0042', + ); + }); + + test('a stage reports its own tokens only when it counted them', () { + final stage = stageTokenUsage({ + 'usage': { + 'tokens_total': 1500, + 'tokens_known': true, + 'split_known': true, + 'tokens_in': 1200, + 'tokens_out': 300, + 'cost_known': true, + 'cost_usd': 0.02, + }, + })!; + expect(tokenUsageDetail(stage), '1.5K (1.2K in, 300 out)'); + expect(tokenUsageCost(stage), '\$0.02'); + expect( + stageTokenUsage({ + 'usage': {'cost_known': true}, + }), + isNull, + ); + expect(stageTokenUsage({}), isNull); + }); + + test('Most tokens sorts reported totals first, largest first', () { + final runs = [ + _run(1, usage: _usage(5)), + _run(2), + _run(3, usage: _usage(22400000)), + _run(4, usage: _usage(9, stages: 2, tokenStages: 1)), + _run(5), + ]; + final sorted = sortRuns(runs, sort: 'most_tokens'); + expect(sorted.map((run) => run['id']), [3, 4, 1, 5, 2]); + }); + + test('Most tokens is offered beside Largest first and lists flat', () { + final keys = runSorts.map((option) => option.$1).toList(); + expect(keys.indexOf('most_tokens'), keys.indexOf('largest_first') + 1); + expect(runSortLabel('most_tokens'), 'Most tokens'); + expect(flatRunSorts, contains('most_tokens')); + }); +} diff --git a/app/test/features/runs/ui/run_detail_summary_fields_test.dart b/app/test/features/runs/ui/run_detail_summary_fields_test.dart index d38fd4b9..edac3ee9 100644 --- a/app/test/features/runs/ui/run_detail_summary_fields_test.dart +++ b/app/test/features/runs/ui/run_detail_summary_fields_test.dart @@ -24,6 +24,7 @@ JsonMap recordedRunFixture() => { 'retry_of': 12, 'diagnostic': 'Traceback: boom', 'completed_steps': 2, + 'size_bytes': 5 * 1024 * 1024, 'input': {'prompt': 'ship it'}, 'output': {'result': 'ok'}, 'steps': [ @@ -64,6 +65,9 @@ const recordedLabels = [ 'Started', 'Finished', 'Duration', + 'Size on disk', + 'Tokens', + 'Cost', 'Retry of', 'Diagnostic', ]; @@ -91,6 +95,12 @@ void main() { formatLocalTimestamp('2026-10-19T09:32:05Z'), ); expect(rows.firstWhere((row) => row.label == 'Duration').value, '2m 5s'); + expect(rows.firstWhere((row) => row.label == 'Size on disk').value, '5 MB'); + expect( + rows.firstWhere((row) => row.label == 'Tokens').value, + '123 (100 in, 23 out)', + ); + expect(rows.firstWhere((row) => row.label == 'Cost').value, '\$0.50'); expect(rows.firstWhere((row) => row.label == 'Retry of').value, 'run 12'); final diagnostic = rows.firstWhere((row) => row.label == 'Diagnostic'); expect(diagnostic.value, 'Traceback: boom'); diff --git a/app/test/features/runs/ui/run_list_row_test.dart b/app/test/features/runs/ui/run_list_row_test.dart index 6a7346e8..40130ff9 100644 --- a/app/test/features/runs/ui/run_list_row_test.dart +++ b/app/test/features/runs/ui/run_list_row_test.dart @@ -86,6 +86,48 @@ void main() { expect(find.text('Pipeline unavailable · 2m 5s · 3 steps'), findsOneWidget); }); + testWidgets('shows reported tokens beside the size, and nothing without', ( + tester, + ) async { + Future pump(JsonMap run) => tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: RunListRow(run: run, now: DateTime.utc(2026, 10, 22)), + ), + ), + ); + final sized = {..._finished, 'size_bytes': 340 * 1024 * 1024}; + await pump({ + ...sized, + 'usage': { + 'stages': 3, + 'tokens_known_stages': 3, + 'tokens_total': 22400015, + }, + }); + expect( + find.text('release · 2m 5s · 3 steps · 340 MB · 22.4M tokens'), + findsOneWidget, + ); + + await pump({ + ...sized, + 'usage': { + 'stages': 3, + 'tokens_known_stages': 2, + 'tokens_total': 22400015, + }, + }); + expect( + find.text('release · 2m 5s · 3 steps · 340 MB · ≥22.4M tokens'), + findsOneWidget, + ); + + // An older server's list carries no usage: the row is unchanged. + await pump(sized); + expect(find.text('release · 2m 5s · 3 steps · 340 MB'), findsOneWidget); + }); + test('the run action sheet is titled with the run', () { expect(runMoreMenuTitle(_finished), 'Ship the release notes'); expect(runMoreMenuTitle({'id': 5, 'name': 'nightly'}), 'nightly'); diff --git a/app/test/features/runs/ui/run_stage_tokens_test.dart b/app/test/features/runs/ui/run_stage_tokens_test.dart new file mode 100644 index 00000000..4c077c4e --- /dev/null +++ b/app/test/features/runs/ui/run_stage_tokens_test.dart @@ -0,0 +1,46 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/runs/ui/run_detail_content.dart'; +import 'package:pipelines_app/features/runs/ui/run_detail_visit_facts.dart'; + +final _reported = { + 'stage': 'build', + 'sequence': 1, + 'status': 'completed', + 'outcome': 'success', + 'target': 'review', + 'usage': { + 'tokens_total': 1500000, + 'tokens_known': true, + 'split_known': true, + 'tokens_in': 1200000, + 'tokens_out': 300000, + 'cost_known': true, + 'cost_usd': 0.42, + }, +}; + +final _unreported = { + 'stage': 'gate', + 'sequence': 2, + 'status': 'completed', +}; + +void main() { + test('a visit lists its reported tokens and cost as facts', () { + final facts = detailVisitFacts(_reported); + expect( + facts, + contains((label: 'Tokens', value: '1.5M (1.2M in, 300K out)')), + ); + expect(facts, contains((label: 'Cost', value: '\$0.42'))); + final none = detailVisitFacts(_unreported); + expect(none.any((fact) => fact.label == 'Tokens'), isFalse); + expect(none.any((fact) => fact.label == 'Cost'), isFalse); + }); + + test('a stage row carries its compact token total', () { + final visits = detailTimeline([_reported, _unreported]); + expect(visits[0].tokens, '1.5M tokens'); + expect(visits[1].tokens, isNull); + }); +} diff --git a/internal/pipeline/run_usage.go b/internal/pipeline/run_usage.go new file mode 100644 index 00000000..2bfe5e26 --- /dev/null +++ b/internal/pipeline/run_usage.go @@ -0,0 +1,55 @@ +package pipeline + +import ( + "context" + "strings" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// loadRunUsage fills each listed run's Usage with one grouped aggregate query +// per bounded batch over the stored per-stage usage rows: no GetRun, step load +// or transcript parse per row. It uses the same column sums as the +// history-wide totals, so a listed run's total equals its detail's TotalUsage +// over the same rows. A run with no recorded usage keeps Usage nil, and a +// database without the step_usage table lists runs without usage. +func loadRunUsage(ctx context.Context, tx runGraphQuerier, runs []protocol.RunSummary) error { + const batchSize = 100 + for start := 0; start < len(runs); start += batchSize { + end := min(start+batchSize, len(runs)) + args := make([]any, 0, end-start) + byID := make(map[int64]*protocol.RunSummary, end-start) + for i := start; i < end; i++ { + byID[runs[i].ID] = &runs[i] + args = append(args, runs[i].ID) + } + rows, err := tx.QueryContext(ctx, `SELECT run_id,`+usageSumColumns+` +FROM step_usage WHERE run_id IN (`+strings.TrimSuffix(strings.Repeat("?,", len(args)), ",")+`) GROUP BY run_id`, args...) + if err != nil { + if strings.Contains(err.Error(), "no such table") { + return nil + } + return err + } + for rows.Next() { + var id int64 + var u protocol.RunUsage + if err := rows.Scan(&id, &u.Stages, &u.TokensTotal, &u.TokensKnownStages, &u.TokensIn, &u.TokensOut, &u.CostUSD, &u.CostKnownStages, &u.SplitUnknownStages); err != nil { + rows.Close() + return err + } + if u.Stages == 0 || byID[id] == nil { + continue + } + u.CostUnknownStages = u.Stages - u.CostKnownStages + u.SplitKnown = u.TokensKnownStages > 0 && u.SplitUnknownStages == 0 + byID[id].Usage = &u + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + } + return nil +} diff --git a/internal/pipeline/run_usage_test.go b/internal/pipeline/run_usage_test.go new file mode 100644 index 00000000..72f4ba8b --- /dev/null +++ b/internal/pipeline/run_usage_test.go @@ -0,0 +1,91 @@ +package pipeline + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// TestListRunsCarriesRunUsageAggregate proves the run list carries each run's +// usage total from the grouped aggregate, equal to the detail's TotalUsage over +// the same rows, marks a partial total by its stage counts, leaves a run with +// no recorded usage absent rather than zero, and stays correct across list +// batches. +func TestListRunsCarriesRunUsageAggregate(t *testing.T) { + ctx := context.Background() + store, err := Open(filepath.Join(t.TempDir(), "list-usage.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, err := store.Submit(ctx, "list-usage", Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + items, _ := store.List(ctx) + definition, _ := store.Get(ctx, items[0].ID, 1) + create := func(key string) int64 { + id, _, err := store.CreateRun(ctx, definition, key, "/repo", false, nil, nil) + if err != nil { + t.Fatal(err) + } + return id + } + full := create("full") + finishUsageStep(t, store, full, 1, "one", &protocol.StageUsage{TokensIn: 20_000_000, TokensOut: 2_400_000, TokensTotal: 22_400_000, TokensKnown: true, SplitKnown: true, CostUSD: 1.5, CostKnown: true}) + finishUsageStep(t, store, full, 2, "one", &protocol.StageUsage{TokensIn: 10, TokensOut: 5, TokensTotal: 15, TokensKnown: true, SplitKnown: true, CostUSD: .25, CostKnown: true}) + partial := create("partial") + finishUsageStep(t, store, partial, 1, "one", &protocol.StageUsage{TokensTotal: 9, TokensKnown: true}) + finishUsageStep(t, store, partial, 2, "one", &protocol.StageUsage{CostUSD: .1, CostKnown: true}) + none := create("none") + finishUsageStep(t, store, none, 1, "one", nil) + // Enough further runs to push the first ones past one list batch. + const fillerRuns = 105 + for i := range fillerRuns { + filler := create(fmt.Sprintf("filler-%d", i)) + finishUsageStep(t, store, filler, 1, "one", &protocol.StageUsage{TokensTotal: int64(i + 1), TokensKnown: true}) + } + + runs, err := store.ListRuns(ctx) + if err != nil { + t.Fatal(err) + } + byID := map[int64]protocol.RunSummary{} + for _, run := range runs { + byID[run.ID] = run + } + for _, id := range []int64{full, partial} { + detail, err := store.GetRun(ctx, id) + if err != nil { + t.Fatal(err) + } + listed := byID[id].Usage + if listed == nil || detail.Usage == nil || *listed != *detail.Usage { + t.Fatalf("run %d list usage %+v != detail usage %+v", id, listed, detail.Usage) + } + } + if got := byID[full].Usage; got.TokensTotal != 22_400_015 || got.TokensIn != 20_000_010 || got.TokensOut != 2_400_005 || !got.SplitKnown || got.CostUSD != 1.75 || got.TokensKnownStages != 2 || got.Stages != 2 { + t.Fatalf("full run usage = %+v", got) + } + if got := byID[partial].Usage; got.TokensTotal != 9 || got.TokensKnownStages != 1 || got.Stages != 2 || got.SplitKnown || got.CostKnownStages != 1 || got.CostUnknownStages != 1 { + t.Fatalf("partial run usage = %+v", got) + } + if got := byID[none].Usage; got != nil { + t.Fatalf("run without usage listed a total: %+v", got) + } + var fillers int64 + for _, run := range runs { + if run.ID == full || run.ID == partial || run.ID == none { + continue + } + if run.Usage == nil { + t.Fatalf("filler run %d lost its usage", run.ID) + } + fillers += run.Usage.TokensTotal + } + if fillers != fillerRuns*(fillerRuns+1)/2 { + t.Fatalf("filler totals = %d", fillers) + } +} diff --git a/internal/pipeline/runs.go b/internal/pipeline/runs.go index 45fb973c..d0f1a8e9 100644 --- a/internal/pipeline/runs.go +++ b/internal/pipeline/runs.go @@ -76,6 +76,9 @@ func (s *Store) ListRuns(ctx context.Context) ([]protocol.RunSummary, error) { if err := loadRunGraphEvidence(ctx, tx, items); err != nil { return nil, err } + if err := loadRunUsage(ctx, tx, items); err != nil { + return nil, err + } if err := tx.Commit(); err != nil { return nil, err } diff --git a/internal/pipeline/stats.go b/internal/pipeline/stats.go index cfa63607..15ffecce 100644 --- a/internal/pipeline/stats.go +++ b/internal/pipeline/stats.go @@ -64,6 +64,20 @@ func (s *Store) UsageStats(ctx context.Context) (*protocol.UsageStats, error) { return stats, tx.Commit() } +// usageSumColumns sums step_usage rows into the RunUsage/UsageTotals shape: +// stages, reported total, token-reporting stages, split in/out, cost, +// cost-reporting stages and token stages without a split, in that order. It is +// the one SQL definition of the sum, shared by the history-wide totals and the +// per-run list aggregate, and mirrors protocol.TotalUsage over the same rows. +const usageSumColumns = `COALESCE(COUNT(*),0), +COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') THEN json_extract(usage_json,'$.tokens_total') ELSE 0 END),0), +COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') THEN 1 ELSE 0 END),0), +COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.split_known') THEN json_extract(usage_json,'$.tokens_in') ELSE 0 END),0), +COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.split_known') THEN json_extract(usage_json,'$.tokens_out') ELSE 0 END),0), +COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.cost_known') THEN json_extract(usage_json,'$.cost_usd') ELSE 0 END),0), +COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.cost_known') THEN 1 ELSE 0 END),0), +COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND NOT COALESCE(json_extract(usage_json,'$.split_known'),0) THEN 1 ELSE 0 END),0)` + // usageTotals is the one extra aggregate query the history-wide usage totals add, // over the same step_usage rows the per-run figures read. It sums reported totals // with json_extract, counts token/cost-reporting stages and leaves a stage that @@ -71,15 +85,7 @@ func (s *Store) UsageStats(ctx context.Context) (*protocol.UsageStats, error) { // read-only database with no step_usage table yields no totals, not an error. func usageTotals(ctx context.Context, tx *sql.Tx) (*protocol.UsageTotals, error) { var totals protocol.UsageTotals - err := tx.QueryRowContext(ctx, `SELECT -COALESCE(COUNT(*),0), -COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') THEN json_extract(usage_json,'$.tokens_total') ELSE 0 END),0), -COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') THEN 1 ELSE 0 END),0), -COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.split_known') THEN json_extract(usage_json,'$.tokens_in') ELSE 0 END),0), -COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.split_known') THEN json_extract(usage_json,'$.tokens_out') ELSE 0 END),0), -COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.cost_known') THEN json_extract(usage_json,'$.cost_usd') ELSE 0 END),0), -COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.cost_known') THEN 1 ELSE 0 END),0), -COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND NOT COALESCE(json_extract(usage_json,'$.split_known'),0) THEN 1 ELSE 0 END),0) + err := tx.QueryRowContext(ctx, `SELECT `+usageSumColumns+` FROM step_usage`).Scan(&totals.Stages, &totals.TokensTotal, &totals.TokensKnownStages, &totals.TokensIn, &totals.TokensOut, &totals.CostUSD, &totals.CostKnownStages, &totals.SplitUnknownStages) if err != nil { if strings.Contains(err.Error(), "no such table") { diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index 2e02910e..86a130de 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -197,6 +197,14 @@ type RunSummary struct { // session directories and an estimate of its database rows. It is cached // server-side; 0 means not measured yet (a background backfill fills it). SizeBytes int64 `json:"size_bytes"` + // Usage is the totalled reported usage of the run's model stages. The run + // list sums it with one SQL aggregate per batch over the stored per-stage + // usage rows; the run detail sums the same rows in one pass over its loaded + // steps (TotalUsage). It is absent for a run with no model usage recorded + // (including historical runs and servers predating the list aggregate), so + // a client shows nothing rather than zero. A total is partial when + // TokensKnownStages < Stages. + Usage *RunUsage `json:"usage,omitempty"` } // RunGraphVisit contains only routing and lifecycle facts, never private step @@ -433,10 +441,6 @@ type RunDetail struct { Input map[string]any `json:"input"` Output map[string]any `json:"output"` Steps []RunStep `json:"steps"` - // Usage is the totalled reported usage of the run's model stages, summed in - // one pass over Steps. It is absent for a run with no model usage recorded - // (including historical runs). - Usage *RunUsage `json:"usage,omitempty"` // TitleSource is "model" when Title came from the shared summariser or // "fallback" when it was derived from the prompt; TitleModel names the model // that produced a model title. They are retained so a diagnosis can tell the