Merge: run token usage beside size, fresh vs cached, orphan-free stats

This commit is contained in:
2026-11-03 22:22:04 +00:00
28 changed files with 1025 additions and 50 deletions
+5 -3
View File
@@ -83,9 +83,11 @@ pipelines harness list # status of all four
## What you can do
**Runs**
- Runs list with filters, sorting and multi-select delete.
- Run detail: Summary, Timeline, and Artifacts and Changes when the run has
any.
- Runs list with filters, sorting (including largest on disk and most tokens)
and multi-select delete. Each row shows the run's size and reported token
total (`≥` when some stages did not report).
- Run detail: Summary (with size, tokens split fresh versus cached, and the
reported cost), Timeline, and Artifacts and Changes when the run has any.
- Artifacts viewer for files a stage produced, including the recorded session
export (`session.html`), with a full-screen view where HTML pages scroll,
zoom and open their links.
@@ -1,5 +1,6 @@
import '../../pipelines/pipeline_format.dart';
import '../../../core/api_gateway.dart';
import 'run_tokens.dart';
const runStates = [
'queued',
@@ -268,6 +269,19 @@ List<JsonMap> sortRuns(List<JsonMap> runs, {String sort = 'newest_first'}) {
primary = cmp(bt ?? -_maxMillis, at ?? -_maxMillis);
}
break;
case 'most_tokens':
// Runs with no reported tokens (older servers, no model stage) sort
// after every reported total; a partial total sorts by what was
// reported.
final ta = runTokenUsage(a)?.total, tb = runTokenUsage(b)?.total;
primary = cmp(ta == null ? 1 : 0, tb == null ? 1 : 0);
if (primary == 0 && ta != null && tb != null) {
primary = tb.compareTo(ta);
}
if (primary == 0) {
primary = cmp(bt ?? -_maxMillis, at ?? -_maxMillis);
}
break;
case 'pipeline_name':
primary = '${a['name'] ?? ''}'.toLowerCase().compareTo(
'${b['name'] ?? ''}'.toLowerCase(),
@@ -0,0 +1,167 @@
import '../../../core/api_gateway.dart';
/// Reported token and cost usage of a run or one stage. No widgets here.
///
/// Every figure is what the server recorded from the harness: nothing is
/// estimated. A run total is partial when some stages that recorded usage
/// reported no token count, and it is then labelled as a lower bound (`≥`).
class TokenUsage {
const TokenUsage({
required this.total,
required this.partial,
this.input,
this.output,
this.cacheRead,
this.cost,
this.costPartial = false,
this.stages,
this.tokenStages,
});
/// The summed reported total.
final int total;
/// Some stages that recorded usage reported no token count.
final bool partial;
/// The input/output split, only when every counted stage reported it.
final int? input;
final int? output;
/// The cache-read share of [total], only when every counted stage reported
/// its cache components. Older rows never recorded them, so they have none.
final int? cacheRead;
/// The tokens that were not cache reads, when [cacheRead] is known.
int? get fresh => cacheRead == null ? null : total - cacheRead!;
/// The reported cost in USD, only when a positive cost was reported. Some
/// providers (subscription models) report a cost of zero or none at all;
/// that is "not reported", never a known zero.
final double? cost;
/// Some stages reported no cost, so [cost] is a lower bound.
final bool costPartial;
/// Run totals only: the stages that recorded usage, and those of them that
/// reported tokens.
final int? stages;
final int? tokenStages;
}
/// The run's reported token usage from its `usage` total, or null when the
/// server sent none (an older server's list, or a run with no model usage) or
/// no stage reported a token count.
TokenUsage? runTokenUsage(JsonMap run) {
final raw = run['usage'];
if (raw is! Map) return null;
final stages = _int(raw['stages']);
final tokenStages = _int(raw['tokens_known_stages']) ?? 0;
final total = _int(raw['tokens_total']);
if (tokenStages <= 0 || total == null) return null;
final costStages = _int(raw['cost_known_stages']) ?? 0;
final split = raw['split_known'] == true;
final cost = costStages > 0 ? _positive(raw['cost_usd']) : null;
return TokenUsage(
total: total,
partial: stages != null && tokenStages < stages,
// The server omits a zero in/out, so a known split makes absence zero.
input: split ? _int(raw['tokens_in']) ?? 0 : null,
output: split ? _int(raw['tokens_out']) ?? 0 : null,
cacheRead: _cacheRead(raw, total),
cost: cost,
costPartial: cost != null && stages != null && costStages < stages,
stages: stages,
tokenStages: tokenStages,
);
}
/// One stage's reported token usage from its step `usage`, or null when the
/// stage reported no token count.
TokenUsage? stageTokenUsage(JsonMap step) {
final raw = step['usage'];
if (raw is! Map || raw['tokens_known'] != true) return null;
final total = _int(raw['tokens_total']);
if (total == null) return null;
final split = raw['split_known'] == true;
return TokenUsage(
total: total,
partial: false,
input: split ? _int(raw['tokens_in']) ?? 0 : null,
output: split ? _int(raw['tokens_out']) ?? 0 : null,
cacheRead: _cacheRead(raw, total),
cost: raw['cost_known'] == true ? _positive(raw['cost_usd']) : null,
);
}
/// A compact token count: `950`, `1.2K`, `340K`, `22.4M`, `1.3B`.
///
/// Decimal multiples, one decimal below a hundred units, whole units above.
String formatTokenCount(int tokens) {
if (tokens < 1000) return '$tokens';
const units = ['K', 'M', 'B', 'T'];
var value = tokens / 1000;
var unit = 0;
while (value >= 1000 && unit < units.length - 1) {
value /= 1000;
unit++;
}
// Rounding 999.6K up would print "1000K"; promote it instead.
if (value >= 999.5 && unit < units.length - 1) {
value /= 1000;
unit++;
}
final text = value < 99.95
? value.toStringAsFixed(1).replaceFirst(RegExp(r'\.0$'), '')
: value.round().toString();
return '$text${units[unit]}';
}
/// The list row's compact label: `22.4M tokens`, or `≥22.4M tokens` when some
/// stages did not report a count.
String tokenUsageLabel(TokenUsage usage) =>
'${usage.partial ? '≥' : ''}${formatTokenCount(usage.total)} tokens';
/// The detail's token value: the compact total, then fresh versus cached
/// when every stage reported its cache components, the in/out split when
/// reported, and how many stages reported when the total is partial.
/// `22.4M · 1.1M fresh, 21.3M cached · 900K in, 150K out`,
/// `≥9 · 1 of 2 stages reported`.
String tokenUsageDetail(TokenUsage usage) => [
'${usage.partial ? '≥' : ''}${formatTokenCount(usage.total)}',
if (usage.fresh != null)
'${formatTokenCount(usage.fresh!)} fresh, '
'${formatTokenCount(usage.cacheRead!)} cached',
if (usage.input != null && usage.output != null)
'${formatTokenCount(usage.input!)} in, '
'${formatTokenCount(usage.output!)} out',
if (usage.partial && usage.stages != null)
'${usage.tokenStages} of ${usage.stages} stages reported',
].join(' · ');
/// The wording for a cost no stage reported as a positive amount.
const costNotReported = 'Not reported';
/// The reported cost: `$1.75`, `$0.10`, `$0.0042` (four places below a cent),
/// `≥$0.10` when some stages reported no cost, and [costNotReported] when no
/// positive cost was reported.
String tokenUsageCost(TokenUsage usage) {
final cost = usage.cost;
if (cost == null) return costNotReported;
final text = cost >= 0.01 ? cost.toStringAsFixed(2) : cost.toStringAsFixed(4);
return '${usage.costPartial ? '≥' : ''}\$$text';
}
int? _int(dynamic value) =>
value is num && value.isFinite && value >= 0 ? value.round() : null;
double? _positive(dynamic value) =>
value is num && value.isFinite && value > 0 ? value.toDouble() : null;
/// The cache reads, only when the server marked the cache split complete and
/// the count fits inside the total.
int? _cacheRead(Map raw, int total) {
if (raw['cache_known'] != true) return null;
final read = _int(raw['cache_read']) ?? 0;
return read <= total ? read : null;
}
@@ -3,6 +3,7 @@ import 'dart:convert';
import '../../../core/api_gateway.dart';
import '../../pipelines/pipeline_format.dart' show formatDuration;
import '../domain/run_models.dart';
import '../domain/run_tokens.dart';
/// One labelled value in the run detail's metrics block.
typedef DetailMetric = ({String label, String value});
@@ -23,6 +24,7 @@ typedef DetailVisit = ({
String outcome,
String target,
String? duration,
String? tokens,
});
/// The run's primary label: the stored title, else the pipeline name, and only
@@ -74,6 +76,7 @@ List<DetailMetric> detailUsageMetrics(dynamic raw) {
final tokenStages = _int(usage['tokens_known_stages']) ?? 0;
final costStages = _int(usage['cost_known_stages']) ?? 0;
final totalTokens = _int(usage['tokens_total']);
final cost = usage['cost_usd'] is num ? usage['cost_usd'] as num : null;
final splitKnown = usage['split_known'] == true && tokenStages > 0;
final rows = <DetailMetric>[
if (stages != null) (label: 'Stages', value: '$stages'),
@@ -93,11 +96,11 @@ List<DetailMetric> detailUsageMetrics(dynamic raw) {
),
(
label: 'Reported cost',
// cost_usd is omitted by the server when zero, so a known stage count
// makes an absent value a real $0.000000 rather than an unknown cost.
value: costStages == 0
? 'Unavailable'
: '\$${((usage['cost_usd'] as num?) ?? 0).toStringAsFixed(6)}',
// A cost of zero (or none) is what subscription models report even
// with a "known" cost: it is not reported, never a measured zero.
value: costStages == 0 || cost == null || cost <= 0
? 'Not reported'
: '\$${cost.toStringAsFixed(6)}',
),
];
if (stages != null && tokenStages < stages) {
@@ -149,6 +152,10 @@ List<DetailVisit> detailTimeline(dynamic steps) {
outcome: '${step['outcome'] ?? ''}'.trim(),
target: '${step['target'] ?? ''}'.trim(),
duration: _visitDuration(step),
tokens: switch (stageTokenUsage(JsonMap.from(step))) {
final usage? => tokenUsageLabel(usage),
null => null,
},
);
}(),
];
@@ -1,5 +1,7 @@
import '../../../core/api_gateway.dart';
import '../../pipelines/pipeline_format.dart';
import '../domain/run_deletion.dart' show formatByteSize, runSizeBytes;
import '../domain/run_tokens.dart';
import 'run_detail_visit_facts.dart';
/// One recorded Summary row: the web run summary's key/value shape.
@@ -14,9 +16,9 @@ typedef DetailSummaryRow = ({
bool block,
});
/// The run Summary's rows, in the web summary's order: Pipeline, Stage,
/// Workspace, Content hash, Started, Finished, Duration, Retry of and the
/// Diagnostic block.
/// The run Summary's rows: Pipeline, Stage, Workspace, Content hash, Started,
/// Finished, Duration, Size on disk, Tokens, Cost, Retry of and the Diagnostic
/// block.
///
/// Only values the server actually recorded are rows: a blank field is omitted
/// rather than drawn as an empty or invented placeholder, exactly as
@@ -53,6 +55,16 @@ List<DetailSummaryRow> detailSummaryRows(JsonMap run) {
_text(run['finished_at']),
);
if (seconds != null) add('Duration', formatDuration(seconds));
// What the run occupies on disk and what it spent, as the server measured
// and the harness reported; an unmeasured size or unreported usage has no
// row.
final bytes = runSizeBytes(run);
if (bytes != null) add('Size on disk', formatByteSize(bytes));
final usage = runTokenUsage(run);
if (usage != null) {
add('Tokens', tokenUsageDetail(usage));
add('Cost', tokenUsageCost(usage));
}
final retryOf = _number(run['retry_of'])?.toInt() ?? 0;
if (retryOf > 0) add('Retry of', 'run $retryOf');
// A diagnostic is a recorded document: its own whitespace is preserved, but a
@@ -16,7 +16,7 @@ import '../../../ui/adaptive/floating_bar_layout.dart';
/// the stored original prompt with its copy control and the collapsible Input
/// and Output sections.
///
/// The web run detail has no usage block, so no metrics are drawn here. A fact
/// The run's size and reported token usage are facts like the others. A fact
/// the server did not record is omitted rather than shown as an invented or
/// `Unavailable` row; only the Input/Output empty sentences remain, and only
/// when that section recorded nothing.
@@ -135,8 +135,8 @@ class RunTimelineRow extends StatelessWidget {
@override
Widget build(BuildContext context) {
final state = _state;
// The outcome chip leads the second line, then where the visit went; the
// duration is the row's time.
// The outcome chip leads the second line, then where the visit went and
// the tokens it reported; the duration is the row's time.
return AppListRow(
cupertino: cupertino,
selected: selected,
@@ -152,7 +152,14 @@ class RunTimelineRow extends StatelessWidget {
title: Text(visit.title),
time: visit.duration,
subtitleLeading: _outcome(context),
subtitle: visit.target.isEmpty ? null : Text('→ ${visit.target}'),
subtitle: visit.target.isEmpty && visit.tokens == null
? null
: Text(
[
if (visit.target.isNotEmpty) '→ ${visit.target}',
?visit.tokens,
].join(' · '),
),
);
}
}
@@ -1,6 +1,7 @@
import '../../../core/api_gateway.dart';
import '../../graph/domain/stage_definition.dart';
import '../../pipelines/pipeline_format.dart';
import '../domain/run_tokens.dart';
import 'run_detail_content.dart';
/// The exact empty wording for a visit's recorded Input and Output. They are
@@ -80,6 +81,11 @@ List<VisitFact> detailVisitFacts(JsonMap step) {
if (duration != null) facts.add((label: 'Duration', value: duration));
final model = detailVisitProviderModel(step);
if (model != null) facts.add((label: 'Provider/model', value: model));
final usage = stageTokenUsage(step);
if (usage != null) {
facts.add((label: 'Tokens', value: tokenUsageDetail(usage)));
facts.add((label: 'Cost', value: tokenUsageCost(usage)));
}
final target = _text(step['target']);
if (target != null) facts.add((label: 'Target', value: target));
return facts;
@@ -9,6 +9,7 @@ import '../../../ui/status/run_status_icon.dart';
import '../../../ui/text/relative_time_label.dart';
import '../domain/run_deletion.dart';
import '../domain/run_models.dart';
import '../domain/run_tokens.dart';
import 'run_more_menu.dart';
/// One run in the list: a status avatar, title/time, and one muted fact
@@ -77,6 +78,9 @@ class RunListRow extends StatelessWidget {
final scheduled = run['scheduled'] == true;
final bytes = runSizeBytes(run);
final size = bytes == null ? null : formatByteSize(bytes);
// Reported tokens sit beside the size; an older server sends none.
final usage = runTokenUsage(run);
final tokens = usage == null ? null : tokenUsageLabel(usage);
final createdAt = DateTime.tryParse('${run['created_at'] ?? ''}');
final subtitle = [
?pipelineLabel,
@@ -84,6 +88,7 @@ class RunListRow extends StatelessWidget {
?facts.duration,
?facts.progress,
?size,
?tokens,
].join(' · ');
// A narrow pane keeps the decision facts: the pipeline, and progress
// while executing (duration only when there is no pipeline to show).
@@ -93,6 +98,7 @@ class RunListRow extends StatelessWidget {
if (run['executing'] == true && facts.progress != null) facts.progress!,
if (pipelineLabel == null && facts.duration != null) facts.duration!,
?size,
?tokens,
].join(' · ');
final state = runStateOf(run['status']);
return AppListRow(
@@ -162,14 +162,7 @@ class RunsController extends ChangeNotifier {
// A cancel/decision reply changes lifecycle fields as well as title.
// Keep sparse/unknown summary fields, without copying bulky detail-only
// payloads into the held list or inventing missing values.
const detailOnly = {
'steps',
'input',
'output',
'graph',
'usage',
'notifications',
};
const detailOnly = {'steps', 'input', 'output', 'graph', 'notifications'};
runs = [...runs]
..[index] = {
...runs[index],
@@ -7,6 +7,7 @@ const runSorts = [
('newest_first', 'Newest first'),
('oldest_first', 'Oldest first'),
('largest_first', 'Largest first'),
('most_tokens', 'Most tokens'),
('longest_duration', 'Longest duration'),
('shortest_duration', 'Shortest duration'),
('pipeline_name', 'Pipeline name'),
@@ -25,8 +26,8 @@ const runTimeRanges = [
];
/// Sorts that list runs flat instead of under day headings: an order by
/// size has no meaningful day grouping.
const flatRunSorts = {'largest_first'};
/// size or token count has no meaningful day grouping.
const flatRunSorts = {'largest_first', 'most_tokens'};
/// The pipelines the loaded runs belong to, by name then id.
List<({int id, String name})> runFilterPipelines(RunsController controller) {
+48
View File
@@ -0,0 +1,48 @@
# Run token usage beside size
Runs that spend 20M+ tokens now show it the way they show disk size.
## Server (payload)
- `RunSummary.usage` (`protocol.RunUsage`) moved up from `RunDetail`, so
`list_runs` rows carry the run total too. `get_run` still sends `usage` at
the same JSON key, now through the embedded summary.
- The list fills it with one grouped `step_usage` aggregate per 100-run batch
(`internal/pipeline/run_usage.go`), using the same column sums as the
history-wide stats (`usageSumColumns` in `stats.go`). It is joined to
`run_steps`, so orphaned usage rows (reported: about 650 rows, 1.47B tokens,
left by an Oct 19 history wipe) no longer count in stats or per-run totals.
The rows are not deleted. If a wiped run id was reused, an orphan row that
matches a new run's (run_id, sequence) still cannot be told apart.
- `StageUsage` now keeps `tokens_reasoning`, `cache_read` and `cache_write`
with `cache_known` (the runner dropped them before). They are JSON keys in
`usage_json`, so old rows need no migration: they lack `cache_known`, so
their cache share is unknown, not zero. Run totals add `cache_known_stages`
and `cache_known` (every token-reporting stage recorded the split).
## App
- `lib/features/runs/domain/run_tokens.dart`: parsing and formatting.
`22.4M tokens`, `≥9.1M tokens` when `tokens_known_stages < stages`. Nothing
is estimated, and a row without `usage` (an older server) shows nothing.
- List row: tokens after the size, in both the full and the compact subtitle.
- Sort: `Most tokens` (`most_tokens`) after `Largest first`, flat like it.
Runs without a reported total sort last.
- Summary: `Size on disk`, `Tokens` (`22.4M · 2.1M fresh, 20.3M cached ·
1.8M in, 240K out`; fresh is total minus cache reads, shown only when
`cache_known`) and `Cost`.
- Stage rows: the Timeline row subtitle adds `· 20.1M tokens`. The visit facts
add `Tokens` and `Cost`.
- Cost is shown only when a positive `cost_usd` was reported. Subscription
models report `cost_known` with no amount, which shows as `Not reported`.
The stats page's `Reported cost` uses the same rule.
- `usage` is no longer treated as detail-only in
`RunsController.acceptRunDetail`, so opening a run refreshes its row's total.
## Evidence
Fixture: `tool/probe_usage_fixture.dart`, opt-in `?usage=1` on the capture
entry (run 7: 340 MB, 22.4M tokens, mostly cached, $1.42; run 6: partial
≥9.1M). Captures were taken with `tool/capture` at 560x1000 and 1400x900:
the list, the sort sheet, Most tokens, the run 7 Summary and Timeline, the
build and plan visits, and the no-usage (old-server) list and Summary.
@@ -0,0 +1,191 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:pipelines_app/core/api_gateway.dart';
import 'package:pipelines_app/features/runs/domain/run_models.dart';
import 'package:pipelines_app/features/runs/domain/run_tokens.dart';
import 'package:pipelines_app/features/runs/ui/runs_filter_options.dart';
JsonMap _run(int id, {Map<String, dynamic>? usage, String? created}) => {
'id': id,
'created_at': created ?? '2026-10-2${id % 10}T12:00:00Z',
'usage': ?usage,
};
Map<String, dynamic> _usage(
int total, {
int stages = 1,
int? tokenStages,
bool split = false,
int input = 0,
int output = 0,
int costStages = 0,
double? cost,
}) => {
'stages': stages,
'tokens_total': total,
'tokens_known_stages': tokenStages ?? stages,
'split_known': split,
if (input > 0) 'tokens_in': input,
if (output > 0) 'tokens_out': output,
'cost_known_stages': costStages,
'cost_usd': ?cost,
};
void main() {
test('token counts are compact decimal multiples', () {
expect(formatTokenCount(0), '0');
expect(formatTokenCount(950), '950');
expect(formatTokenCount(1000), '1K');
expect(formatTokenCount(1234), '1.2K');
expect(formatTokenCount(340000), '340K');
expect(formatTokenCount(999600), '1M');
expect(formatTokenCount(22400015), '22.4M');
expect(formatTokenCount(1300000000), '1.3B');
});
test('a reported run total is labelled like the size', () {
final usage = runTokenUsage(_run(1, usage: _usage(22400015)))!;
expect(usage.partial, isFalse);
expect(tokenUsageLabel(usage), '22.4M tokens');
});
test('a partial total is a lower bound, never an estimate', () {
final usage = runTokenUsage(
_run(1, usage: _usage(9, stages: 2, tokenStages: 1)),
)!;
expect(usage.partial, isTrue);
expect(tokenUsageLabel(usage), '≥9 tokens');
expect(tokenUsageDetail(usage), '≥9 · 1 of 2 stages reported');
});
test('absent or unreported usage shows nothing', () {
expect(runTokenUsage(_run(1)), isNull);
expect(runTokenUsage({'id': 1, 'usage': 'bogus'}), isNull);
// Only a cost was reported: no token figure is invented.
expect(
runTokenUsage(_run(1, usage: _usage(0, tokenStages: 0, costStages: 1))),
isNull,
);
});
test('the detail adds the split and the reported cost', () {
final usage = runTokenUsage(
_run(
1,
usage: _usage(
22400015,
stages: 2,
split: true,
input: 20000010,
output: 2400005,
costStages: 2,
cost: 1.75,
),
),
)!;
expect(tokenUsageDetail(usage), '22.4M · 20M in, 2.4M out');
expect(tokenUsageCost(usage), '\$1.75');
final partialCost = runTokenUsage(
_run(1, usage: _usage(9, stages: 2, costStages: 1, cost: 0.1)),
)!;
expect(tokenUsageCost(partialCost), '≥\$0.10');
expect(
tokenUsageCost(runTokenUsage(_run(1, usage: _usage(9)))!),
'Not reported',
);
expect(
tokenUsageCost(
runTokenUsage(_run(1, usage: _usage(9, costStages: 1, cost: 0.0042)))!,
),
'\$0.0042',
);
});
test('the detail splits fresh from cached when every stage reported it', () {
final usage = runTokenUsage(
_run(
1,
usage: {
..._usage(22400000, split: true, input: 900000, output: 150000),
'cache_known': true,
'cache_read': 21300000,
'cache_write': 50000,
},
),
)!;
expect(tokenUsageLabel(usage), '22.4M tokens');
expect(usage.fresh, 1100000);
expect(
tokenUsageDetail(usage),
'22.4M · 1.1M fresh, 21.3M cached · 900K in, 150K out',
);
// An older row (or a partial cache split) has no cache figure: just the
// total and what else it reported.
final old = runTokenUsage(
_run(
1,
usage: _usage(22400000, split: true, input: 900000, output: 150000),
),
)!;
expect(old.fresh, isNull);
expect(tokenUsageDetail(old), '22.4M · 900K in, 150K out');
});
test('a zero or missing "known" cost is not reported, never \$0', () {
// Subscription models report cost_known with no positive cost_usd.
for (final usage in [
_usage(9, costStages: 1),
_usage(9, costStages: 1, cost: 0),
]) {
final tokens = runTokenUsage(_run(1, usage: usage))!;
expect(tokens.cost, isNull);
expect(tokenUsageCost(tokens), 'Not reported');
}
final stage = stageTokenUsage({
'usage': {'tokens_total': 9, 'tokens_known': true, 'cost_known': true},
})!;
expect(tokenUsageCost(stage), 'Not reported');
});
test('a stage reports its own tokens only when it counted them', () {
final stage = stageTokenUsage({
'usage': {
'tokens_total': 1500,
'tokens_known': true,
'split_known': true,
'tokens_in': 1200,
'tokens_out': 300,
'cost_known': true,
'cost_usd': 0.02,
},
})!;
expect(tokenUsageDetail(stage), '1.5K · 1.2K in, 300 out');
expect(tokenUsageCost(stage), '\$0.02');
expect(
stageTokenUsage({
'usage': {'cost_known': true},
}),
isNull,
);
expect(stageTokenUsage({}), isNull);
});
test('Most tokens sorts reported totals first, largest first', () {
final runs = [
_run(1, usage: _usage(5)),
_run(2),
_run(3, usage: _usage(22400000)),
_run(4, usage: _usage(9, stages: 2, tokenStages: 1)),
_run(5),
];
final sorted = sortRuns(runs, sort: 'most_tokens');
expect(sorted.map((run) => run['id']), [3, 4, 1, 5, 2]);
});
test('Most tokens is offered beside Largest first and lists flat', () {
final keys = runSorts.map((option) => option.$1).toList();
expect(keys.indexOf('most_tokens'), keys.indexOf('largest_first') + 1);
expect(runSortLabel('most_tokens'), 'Most tokens');
expect(flatRunSorts, contains('most_tokens'));
});
}
@@ -24,6 +24,7 @@ JsonMap recordedRunFixture() => <String, dynamic>{
'retry_of': 12,
'diagnostic': 'Traceback: boom',
'completed_steps': 2,
'size_bytes': 5 * 1024 * 1024,
'input': <String, dynamic>{'prompt': 'ship it'},
'output': <String, dynamic>{'result': 'ok'},
'steps': <dynamic>[
@@ -64,6 +65,9 @@ const recordedLabels = [
'Started',
'Finished',
'Duration',
'Size on disk',
'Tokens',
'Cost',
'Retry of',
'Diagnostic',
];
@@ -91,6 +95,12 @@ void main() {
formatLocalTimestamp('2026-10-19T09:32:05Z'),
);
expect(rows.firstWhere((row) => row.label == 'Duration').value, '2m 5s');
expect(rows.firstWhere((row) => row.label == 'Size on disk').value, '5 MB');
expect(
rows.firstWhere((row) => row.label == 'Tokens').value,
'123 · 100 in, 23 out',
);
expect(rows.firstWhere((row) => row.label == 'Cost').value, '\$0.50');
expect(rows.firstWhere((row) => row.label == 'Retry of').value, 'run 12');
final diagnostic = rows.firstWhere((row) => row.label == 'Diagnostic');
expect(diagnostic.value, 'Traceback: boom');
@@ -86,6 +86,48 @@ void main() {
expect(find.text('Pipeline unavailable · 2m 5s · 3 steps'), findsOneWidget);
});
testWidgets('shows reported tokens beside the size, and nothing without', (
tester,
) async {
Future<void> pump(JsonMap run) => tester.pumpWidget(
MaterialApp(
home: Scaffold(
body: RunListRow(run: run, now: DateTime.utc(2026, 10, 22)),
),
),
);
final sized = {..._finished, 'size_bytes': 340 * 1024 * 1024};
await pump({
...sized,
'usage': {
'stages': 3,
'tokens_known_stages': 3,
'tokens_total': 22400015,
},
});
expect(
find.text('release · 2m 5s · 3 steps · 340 MB · 22.4M tokens'),
findsOneWidget,
);
await pump({
...sized,
'usage': {
'stages': 3,
'tokens_known_stages': 2,
'tokens_total': 22400015,
},
});
expect(
find.text('release · 2m 5s · 3 steps · 340 MB · ≥22.4M tokens'),
findsOneWidget,
);
// An older server's list carries no usage: the row is unchanged.
await pump(sized);
expect(find.text('release · 2m 5s · 3 steps · 340 MB'), findsOneWidget);
});
test('the run action sheet is titled with the run', () {
expect(runMoreMenuTitle(_finished), 'Ship the release notes');
expect(runMoreMenuTitle({'id': 5, 'name': 'nightly'}), 'nightly');
@@ -0,0 +1,46 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:pipelines_app/features/runs/ui/run_detail_content.dart';
import 'package:pipelines_app/features/runs/ui/run_detail_visit_facts.dart';
final _reported = <String, dynamic>{
'stage': 'build',
'sequence': 1,
'status': 'completed',
'outcome': 'success',
'target': 'review',
'usage': {
'tokens_total': 1500000,
'tokens_known': true,
'split_known': true,
'tokens_in': 1200000,
'tokens_out': 300000,
'cost_known': true,
'cost_usd': 0.42,
},
};
final _unreported = <String, dynamic>{
'stage': 'gate',
'sequence': 2,
'status': 'completed',
};
void main() {
test('a visit lists its reported tokens and cost as facts', () {
final facts = detailVisitFacts(_reported);
expect(
facts,
contains((label: 'Tokens', value: '1.5M · 1.2M in, 300K out')),
);
expect(facts, contains((label: 'Cost', value: '\$0.42')));
final none = detailVisitFacts(_unreported);
expect(none.any((fact) => fact.label == 'Tokens'), isFalse);
expect(none.any((fact) => fact.label == 'Cost'), isFalse);
});
test('a stage row carries its compact token total', () {
final visits = detailTimeline([_reported, _unreported]);
expect(visits[0].tokens, '1.5M tokens');
expect(visits[1].tokens, isNull);
});
}
+92
View File
@@ -0,0 +1,92 @@
import 'package:pipelines_app/core/api_gateway.dart';
/// Opt-in (`?usage=1`) size and reported token usage for the capture
/// fixture's runs, shaped exactly as the server sends them: a run-level
/// `usage` total (list and detail) and a per-step `usage`. Without the
/// parameter the fixture keeps the old-server shape with neither.
bool probeUsage = false;
/// Per-step usage of run 7, by recorded sequence. The build visit is mostly
/// cache reads, as a caching harness reports; plan 1 predates the cache split.
const _stepUsage = <int, JsonMap>{
5: {
'tokens_total': 1200000,
'tokens_known': true,
'split_known': true,
'tokens_in': 300000,
'tokens_out': 60000,
'tokens_reasoning': 40000,
'cache_read': 780000,
'cache_write': 20000,
'cache_known': true,
'cost_known': true,
},
2: {
'tokens_total': 20100000,
'tokens_known': true,
'split_known': true,
'tokens_in': 500000,
'tokens_out': 80000,
'tokens_reasoning': 20000,
'cache_read': 19480000,
'cache_write': 20000,
'cache_known': true,
'cost_usd': 1.42,
'cost_known': true,
},
1: {
'tokens_total': 1100000,
'tokens_known': true,
'split_known': true,
'tokens_in': 1000000,
'tokens_out': 100000,
'cache_known': true,
'cache_read': 0,
'cost_known': true,
},
};
/// Run 7's total over [_stepUsage], as the server's aggregate sums it.
const _run7Usage = <String, dynamic>{
'stages': 3,
'tokens_total': 22400000,
'tokens_known_stages': 3,
'split_known': true,
'tokens_in': 1800000,
'tokens_out': 240000,
'tokens_reasoning': 60000,
'cache_read': 20260000,
'cache_write': 40000,
'cache_known': true,
'cache_known_stages': 3,
'cost_usd': 1.42,
'cost_known_stages': 3,
};
/// Run 6: one of its two stages reported no token count, so its total is a
/// lower bound; no stage reported a positive cost.
const _run6Usage = <String, dynamic>{
'stages': 2,
'tokens_total': 9100000,
'tokens_known_stages': 1,
'split_known': false,
'split_unknown_stages': 1,
'cost_known_stages': 1,
'cost_unknown_stages': 1,
};
/// Adds the run's size and usage to a list row or detail when enabled.
JsonMap withProbeUsage(JsonMap run) {
if (!probeUsage) return run;
return switch (run['id']) {
7 => {...run, 'size_bytes': 340 * 1024 * 1024, 'usage': _run7Usage},
6 => {...run, 'size_bytes': 4404019, 'usage': _run6Usage},
_ => run,
};
}
/// Adds the recorded per-step usage to run 7's steps when enabled.
JsonMap withProbeStepUsage(JsonMap step) {
final usage = _stepUsage[step['sequence']];
return probeUsage && usage != null ? {...step, 'usage': usage} : step;
}
+20 -2
View File
@@ -32,6 +32,7 @@ import 'perf/chat_question_fixture.dart';
import 'perf/chat_scroll_fixture.dart';
import 'perf/perf_fixture.dart';
import 'probe_folder_fixture.dart';
import 'probe_usage_fixture.dart';
/// The run every run-scoped hit names. Its recorded sequences are deliberately
/// not its row indexes (plan 5, build 2, plan 1), exactly like the unit
@@ -426,12 +427,28 @@ JsonMap? probeReply(JsonMap request) {
'time': '2026-10-20T09:00:00Z',
};
case 'list_runs':
return {'id': id, 'type': 'runs', 'runs': probeRuns(), 'usage': null};
return {
'id': id,
'type': 'runs',
'runs': probeRuns().map(withProbeUsage).toList(),
'usage': null,
};
case 'get_run':
if (request['run_id'] != 7) {
return error('Fixture run ${request['run_id']} is unavailable');
}
return {'id': id, 'type': 'run', 'run_detail': probeRunDetail()};
final detail = withProbeUsage(probeRunDetail());
return {
'id': id,
'type': 'run',
'run_detail': {
...detail,
'steps': [
for (final step in detail['steps'] as List)
withProbeStepUsage(step as JsonMap),
],
},
};
case 'list_pipelines':
return {'id': id, 'type': 'pipelines', 'pipelines': probePipelines()};
case 'get_pipeline':
@@ -622,6 +639,7 @@ void main() {
configurePerfFixture(params);
_fillerRunCount = int.tryParse(params['runs'] ?? '') ?? 0;
_fillerStepCount = int.tryParse(params['steps'] ?? '') ?? 0;
probeUsage = params['usage'] == '1';
_htmlArtifact = params['html'] == '1';
final cupertino = params['platform'] == 'cupertino';
final route = params['route'];
+26 -6
View File
@@ -71,8 +71,9 @@ token-reporting stage reported one, and the summed cost, plus counts of stages
that did or did not report a cost. A stage whose provider reported no cost is
counted as cost-unknown and never contributes a fabricated or zero cost, and no
cost/token count is inferred from output length or transcript text. The totals are
the sum of the per-run totals `inspect` shows. `pipelines stats --json` exposes
these aggregates.
the sum of the per-run totals `inspect` shows. Only usage rows whose recorded
step still exists are counted, so rows orphaned by an old history wipe are ignored
(not deleted). `pipelines stats --json` exposes these aggregates.
Use `:status` on any TUI page to open server and ntfy information. Scroll with
arrows or `j/k`, PgUp/PgDn; `r` refreshes and Esc closes without changing the page.
@@ -343,8 +344,23 @@ Every run summary (`list_runs` rows, `get_run`, subscribe snapshots and terminal
with turn documents and session exports, server-created Pi session directories,
plus an estimate of its database rows). It is cached in `pipeline_runs.size_bytes`,
invalidated by triggers when the run changes, measured on completion, and
backfilled in the background; 0 means not measured yet. `delete_runs` removes
finished runs to free space:
backfilled in the background; 0 means not measured yet.
`list_runs` rows (and `get_run`) also carry `usage`, the run's totalled reported
model usage, in the same shape as the run detail's total: `tokens_total`,
`stages` and `tokens_known_stages` (the total is partial when fewer stages
reported tokens than recorded usage), the `tokens_in`/`tokens_out` split with
`split_known`, the `tokens_reasoning`/`cache_read`/`cache_write` components with
`cache_known` (true only when every token-reporting stage recorded them;
`tokens_total` includes cache reads, which dominate for a caching harness), and
`cost_usd` with `cost_known_stages`/`cost_unknown_stages`. The list computes it
with one grouped SQL aggregate per batch of 100 runs over the stored per-stage
usage rows joined to their recorded steps, never a step load or transcript parse.
It is omitted for a run with no recorded model usage; an older server omits it on
every row, and clients then show nothing. Subscribe snapshots and `run_event`
frames do not carry it, so a running run's total refreshes with the next list.
`delete_runs` removes finished runs to free space:
```json
{"type":"delete_runs","id":"1","run_ids":[12,13,99]}
@@ -635,8 +651,12 @@ per-step `provider`/`model`, and the run's graph shows the effective pair). Each
model step also records the usage the harness reported, carried from the same
accounting the stage already used for its budget: `usage.tokens_total`, the
`tokens_in`/`tokens_out` split when the harness reported it (`split_known`),
`cost_usd` when reported, and the explicit `tokens_known`/`cost_known`/
`split_known` markers. The run detail totals those stages as `usage`, so a
`cost_usd` when reported, the `tokens_reasoning`/`cache_read`/`cache_write`
components when the harness reported the component split (`cache_known`; rows
recorded before these were kept lack it, so their cache share is unknown), and
the explicit `tokens_known`/`cost_known`/`split_known` markers. Some subscription
providers report `cost_known` with no positive `cost_usd`; clients show that cost
as not reported, never as $0. The run detail totals those stages as `usage`, so a
mixed-model run shows each stage's model, tokens and cost and the run total. A
stage whose provider reported no cost shows its tokens with `cost_known: false`
(never a fabricated or zero cost), and a harness that reports only a total (Pi)
+1 -1
View File
@@ -52,7 +52,7 @@ var notes = map[string]string{
"submit": "Confirm publication at the prompt, or use --yes for explicit automation; schedules activate on acceptance. Managed drafts provide the name and base-version guard. --force never bypasses validation.",
"run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. --model PROVIDER/MODEL overrides the model for every stage of that run; --stage-model STAGE=PROVIDER/MODEL overrides one stage and is repeatable, with a repeated key last-wins. Precedence is per-stage > whole-run > the model pinned in the pipeline version; overrides apply to that run only and never change the pinned definition. An unknown stage key or a malformed provider/model is refused before the run starts, naming the valid stage keys. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect <run-id> --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).",
"retry": "Resume a failed stage by default, restoring the completed prefix handoff; use --mode=restart to start at entry. --stage selects a recorded stage (use --visit for repeated stages). --input-file accepts a bounded JSON object, merged with replacement at the selected stage; '-' reads stdin. Original history and pinned identity are retained; --version selects an explicit published version of the same pipeline only. --wait-timeout leaves an active retry for later read-only inspection or reattachment with the original --request-key, rather than starting a duplicate retry.",
"runs": "Lists runs newest first; --json includes each run's size_bytes (bytes on disk: artifacts, session exports, server-created Pi session directories and an estimate of its database rows; 0 until first measured). `runs rm <id...>` permanently deletes finished runs (succeeded, failed, canceled, interrupted) with every database row and server-owned file for them, after confirmation (--yes for scripts). Active runs, runs an unfinished retry depends on and unknown ids are refused individually with a reason, and the command then exits 1. User workspaces are never touched. Database pages are reused by SQLite; `retention apply` additionally VACUUMs to shrink the file.",
"runs": "Lists runs newest first; --json includes each run's size_bytes (bytes on disk: artifacts, session exports, server-created Pi session directories and an estimate of its database rows; 0 until first measured) and, when any model stage recorded usage, its usage total (tokens_total, split and cache components, cost). `runs rm <id...>` permanently deletes finished runs (succeeded, failed, canceled, interrupted) with every database row and server-owned file for them, after confirmation (--yes for scripts). Active runs, runs an unfinished retry depends on and unknown ids are refused individually with a reason, and the command then exits 1. User workspaces are never touched. Database pages are reused by SQLite; `retention apply` additionally VACUUMs to shrink the file.",
"title": "Rename a run's operator-facing title after the fact without touching its input, steps or execution. The title is collapsed to a single bounded line and recorded as explicitly set. `pipelines runs` and `pipelines inspect` then show it.",
"wait": "Block until each named run reaches a terminal status - succeeded, failed, canceled or interrupted, and also through a run waiting on a person - then print one line per run with its id, status and outcome, its stage and how long it took, and exit 0 even for a run that finished unsuccessfully. Several ids wait together in one command. The exit is non-zero only for a client-side failure: an unknown run id, a connection that could not be established, or --timeout expiring while a run is still active; a timeout says so plainly, never cancels, retries or otherwise touches a run, and is never reported as its outcome. Read-only: it needs no workspace, is not refused by the workspace guard and writes nothing. --json prints the final run envelope instead of the line (one envelope for one id, an array for several).",
"decide": "Choose an option for a run waiting on a human stage. The run must be `waiting`; the option must be one of that stage's non-error edges. The stage is inferred from the run's current stage unless named. The choice and optional --feedback are recorded as the step's evidence (visible in inspect and the TUI) and the run resumes along the chosen edge. An unknown option is refused with the valid list; a repeated decision for an already-decided visit does not double-apply. Works across a server restart because the wait is persisted.",
+57
View File
@@ -0,0 +1,57 @@
package pipeline
import (
"context"
"strings"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// loadRunUsage fills each listed run's Usage with one grouped aggregate query
// per bounded batch over the stored per-stage usage rows: no GetRun, step load
// or transcript parse per row. It uses the same column sums as the
// history-wide totals, so a listed run's total equals its detail's TotalUsage
// over the same rows; a usage row without its recorded step is never counted.
// A run with no recorded usage keeps Usage nil, and a
// database without the step_usage table lists runs without usage.
func loadRunUsage(ctx context.Context, tx runGraphQuerier, runs []protocol.RunSummary) error {
const batchSize = 100
for start := 0; start < len(runs); start += batchSize {
end := min(start+batchSize, len(runs))
args := make([]any, 0, end-start)
byID := make(map[int64]*protocol.RunSummary, end-start)
for i := start; i < end; i++ {
byID[runs[i].ID] = &runs[i]
args = append(args, runs[i].ID)
}
rows, err := tx.QueryContext(ctx, `SELECT u.run_id,`+usageSumColumns+` `+usageFrom+`
WHERE u.run_id IN (`+strings.TrimSuffix(strings.Repeat("?,", len(args)), ",")+`) GROUP BY u.run_id`, args...)
if err != nil {
if strings.Contains(err.Error(), "no such table") {
return nil
}
return err
}
for rows.Next() {
var id int64
var u protocol.RunUsage
if err := rows.Scan(&id, &u.Stages, &u.TokensTotal, &u.TokensKnownStages, &u.TokensIn, &u.TokensOut, &u.CostUSD, &u.CostKnownStages, &u.SplitUnknownStages, &u.TokensReasoning, &u.CacheRead, &u.CacheWrite, &u.CacheKnownStages); err != nil {
rows.Close()
return err
}
if u.Stages == 0 || byID[id] == nil {
continue
}
u.CostUnknownStages = u.Stages - u.CostKnownStages
u.SplitKnown = u.TokensKnownStages > 0 && u.SplitUnknownStages == 0
u.CacheKnown = u.TokensKnownStages > 0 && u.CacheKnownStages == u.TokensKnownStages
byID[id].Usage = &u
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
}
return nil
}
+176
View File
@@ -0,0 +1,176 @@
package pipeline
import (
"context"
"fmt"
"path/filepath"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// TestListRunsCarriesRunUsageAggregate proves the run list carries each run's
// usage total from the grouped aggregate, equal to the detail's TotalUsage over
// the same rows, marks a partial total by its stage counts, leaves a run with
// no recorded usage absent rather than zero, and stays correct across list
// batches.
func TestListRunsCarriesRunUsageAggregate(t *testing.T) {
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), "list-usage.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "list-usage", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, _ := store.Get(ctx, items[0].ID, 1)
create := func(key string) int64 {
id, _, err := store.CreateRun(ctx, definition, key, "/repo", false, nil, nil)
if err != nil {
t.Fatal(err)
}
return id
}
full := create("full")
finishUsageStep(t, store, full, 1, "one", &protocol.StageUsage{TokensIn: 20_000_000, TokensOut: 2_400_000, TokensTotal: 22_400_000, TokensKnown: true, SplitKnown: true, CostUSD: 1.5, CostKnown: true})
finishUsageStep(t, store, full, 2, "one", &protocol.StageUsage{TokensIn: 10, TokensOut: 5, TokensTotal: 15, TokensKnown: true, SplitKnown: true, CostUSD: .25, CostKnown: true})
partial := create("partial")
finishUsageStep(t, store, partial, 1, "one", &protocol.StageUsage{TokensTotal: 9, TokensKnown: true})
finishUsageStep(t, store, partial, 2, "one", &protocol.StageUsage{CostUSD: .1, CostKnown: true})
none := create("none")
finishUsageStep(t, store, none, 1, "one", nil)
// Enough further runs to push the first ones past one list batch.
const fillerRuns = 105
for i := range fillerRuns {
filler := create(fmt.Sprintf("filler-%d", i))
finishUsageStep(t, store, filler, 1, "one", &protocol.StageUsage{TokensTotal: int64(i + 1), TokensKnown: true})
}
runs, err := store.ListRuns(ctx)
if err != nil {
t.Fatal(err)
}
byID := map[int64]protocol.RunSummary{}
for _, run := range runs {
byID[run.ID] = run
}
for _, id := range []int64{full, partial} {
detail, err := store.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
listed := byID[id].Usage
if listed == nil || detail.Usage == nil || *listed != *detail.Usage {
t.Fatalf("run %d list usage %+v != detail usage %+v", id, listed, detail.Usage)
}
}
if got := byID[full].Usage; got.TokensTotal != 22_400_015 || got.TokensIn != 20_000_010 || got.TokensOut != 2_400_005 || !got.SplitKnown || got.CostUSD != 1.75 || got.TokensKnownStages != 2 || got.Stages != 2 {
t.Fatalf("full run usage = %+v", got)
}
if got := byID[partial].Usage; got.TokensTotal != 9 || got.TokensKnownStages != 1 || got.Stages != 2 || got.SplitKnown || got.CostKnownStages != 1 || got.CostUnknownStages != 1 {
t.Fatalf("partial run usage = %+v", got)
}
if got := byID[none].Usage; got != nil {
t.Fatalf("run without usage listed a total: %+v", got)
}
var fillers int64
for _, run := range runs {
if run.ID == full || run.ID == partial || run.ID == none {
continue
}
if run.Usage == nil {
t.Fatalf("filler run %d lost its usage", run.ID)
}
fillers += run.Usage.TokensTotal
}
if fillers != fillerRuns*(fillerRuns+1)/2 {
t.Fatalf("filler totals = %d", fillers)
}
}
// TestRunUsageCountsCacheAndOnlyRecordedSteps proves the cache/reasoning
// components are summed per run (equal in list, detail and stats), a run mixing
// a pre-cache row with a cache-reporting one keeps CacheKnown false, and a usage
// row whose step or run no longer exists (an old history wipe) is counted
// nowhere and left in place.
func TestRunUsageCountsCacheAndOnlyRecordedSteps(t *testing.T) {
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), "cache-usage.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "cache-usage", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, _ := store.Get(ctx, items[0].ID, 1)
cached, _, err := store.CreateRun(ctx, definition, "cached", "/repo", false, nil, nil)
if err != nil {
t.Fatal(err)
}
finishUsageStep(t, store, cached, 1, "one", &protocol.StageUsage{TokensIn: 900_000, TokensOut: 150_000, TokensTotal: 22_400_000, TokensKnown: true, SplitKnown: true, TokensReasoning: 50_000, CacheRead: 21_250_000, CacheWrite: 50_000, CacheKnown: true})
mixed, _, err := store.CreateRun(ctx, definition, "mixed", "/repo", false, nil, nil)
if err != nil {
t.Fatal(err)
}
finishUsageStep(t, store, mixed, 1, "one", &protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true})
finishUsageStep(t, store, mixed, 2, "one", &protocol.StageUsage{TokensIn: 1, TokensOut: 1, TokensTotal: 10, TokensKnown: true, SplitKnown: true, CacheRead: 8, CacheKnown: true})
// Orphans: one for a run that no longer exists and one for a step the
// listed run never recorded. Foreign keys are off only on this connection.
conn, err := store.db.Conn(ctx)
if err != nil {
t.Fatal(err)
}
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=OFF"); err != nil {
t.Fatal(err)
}
orphan := `{"tokens_total":1000000000,"tokens_known":true,"cost_usd":15.67,"cost_known":true}`
for _, key := range [][2]int64{{9999, 1}, {cached, 99}} {
if _, err := conn.ExecContext(ctx, "INSERT INTO step_usage(run_id,sequence,usage_json) VALUES(?,?,?)", key[0], key[1], orphan); err != nil {
t.Fatal(err)
}
}
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=ON"); err != nil {
t.Fatal(err)
}
conn.Close()
runs, err := store.ListRuns(ctx)
if err != nil {
t.Fatal(err)
}
listed := map[int64]*protocol.RunUsage{}
for _, run := range runs {
listed[run.ID] = run.Usage
}
for _, id := range []int64{cached, mixed} {
detail, err := store.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
if listed[id] == nil || detail.Usage == nil || *listed[id] != *detail.Usage {
t.Fatalf("run %d list usage %+v != detail usage %+v", id, listed[id], detail.Usage)
}
}
if got := listed[cached]; got.TokensTotal != 22_400_000 || got.CacheRead != 21_250_000 || got.CacheWrite != 50_000 || got.TokensReasoning != 50_000 || !got.CacheKnown || got.CacheKnownStages != 1 || got.Stages != 1 || got.CostKnownStages != 0 {
t.Fatalf("cached run usage counted an orphan or lost the cache split: %+v", got)
}
if got := listed[mixed]; got.CacheKnown || got.CacheKnownStages != 1 || got.CacheRead != 8 || got.TokensTotal != 17 {
t.Fatalf("mixed run presented a partial cache split as complete: %+v", got)
}
stats, err := store.UsageStats(ctx)
if err != nil || stats.Usage == nil {
t.Fatalf("stats = %+v err=%v", stats, err)
}
if got := stats.Usage; got.Stages != 3 || got.TokensTotal != 22_400_017 || got.CostUSD != 0 || got.CostKnownStages != 0 || got.CacheRead != 21_250_008 || got.CacheKnown {
t.Fatalf("history totals counted orphan rows: %+v", got)
}
var remaining int
if err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM step_usage").Scan(&remaining); err != nil || remaining != 5 {
t.Fatalf("usage rows were deleted: %d %v", remaining, err)
}
}
+3
View File
@@ -76,6 +76,9 @@ func (s *Store) ListRuns(ctx context.Context) ([]protocol.RunSummary, error) {
if err := loadRunGraphEvidence(ctx, tx, items); err != nil {
return nil, err
}
if err := loadRunUsage(ctx, tx, items); err != nil {
return nil, err
}
if err := tx.Commit(); err != nil {
return nil, err
}
+27 -10
View File
@@ -64,6 +64,31 @@ func (s *Store) UsageStats(ctx context.Context) (*protocol.UsageStats, error) {
return stats, tx.Commit()
}
// usageSumColumns sums step_usage rows into the RunUsage/UsageTotals shape:
// stages, reported total, token-reporting stages, split in/out, cost,
// cost-reporting stages, token stages without a split, then reasoning, cache
// read and cache write sums and the stages that recorded them, in that order. It is
// the one SQL definition of the sum, shared by the history-wide totals and the
// per-run list aggregate, and mirrors protocol.TotalUsage over the same rows.
const usageSumColumns = `COALESCE(COUNT(*),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') THEN json_extract(usage_json,'$.tokens_total') ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') THEN 1 ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.split_known') THEN json_extract(usage_json,'$.tokens_in') ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.split_known') THEN json_extract(usage_json,'$.tokens_out') ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.cost_known') THEN json_extract(usage_json,'$.cost_usd') ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.cost_known') THEN 1 ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND NOT COALESCE(json_extract(usage_json,'$.split_known'),0) THEN 1 ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND json_extract(usage_json,'$.cache_known') THEN COALESCE(json_extract(usage_json,'$.tokens_reasoning'),0) ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND json_extract(usage_json,'$.cache_known') THEN COALESCE(json_extract(usage_json,'$.cache_read'),0) ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND json_extract(usage_json,'$.cache_known') THEN COALESCE(json_extract(usage_json,'$.cache_write'),0) ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND json_extract(usage_json,'$.cache_known') THEN 1 ELSE 0 END),0)`
// usageFrom is the step_usage rows that belong to a recorded step. A usage row
// whose step (or run) no longer exists, such as one left behind by an old
// history wipe, is never counted, and is not deleted here either. It is the
// same row set the run detail reads through its step loop.
const usageFrom = `FROM step_usage u JOIN run_steps s ON s.run_id=u.run_id AND s.sequence=u.sequence`
// usageTotals is the one extra aggregate query the history-wide usage totals add,
// over the same step_usage rows the per-run figures read. It sums reported totals
// with json_extract, counts token/cost-reporting stages and leaves a stage that
@@ -71,16 +96,7 @@ func (s *Store) UsageStats(ctx context.Context) (*protocol.UsageStats, error) {
// read-only database with no step_usage table yields no totals, not an error.
func usageTotals(ctx context.Context, tx *sql.Tx) (*protocol.UsageTotals, error) {
var totals protocol.UsageTotals
err := tx.QueryRowContext(ctx, `SELECT
COALESCE(COUNT(*),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') THEN json_extract(usage_json,'$.tokens_total') ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') THEN 1 ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.split_known') THEN json_extract(usage_json,'$.tokens_in') ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.split_known') THEN json_extract(usage_json,'$.tokens_out') ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.cost_known') THEN json_extract(usage_json,'$.cost_usd') ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.cost_known') THEN 1 ELSE 0 END),0),
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND NOT COALESCE(json_extract(usage_json,'$.split_known'),0) THEN 1 ELSE 0 END),0)
FROM step_usage`).Scan(&totals.Stages, &totals.TokensTotal, &totals.TokensKnownStages, &totals.TokensIn, &totals.TokensOut, &totals.CostUSD, &totals.CostKnownStages, &totals.SplitUnknownStages)
err := tx.QueryRowContext(ctx, `SELECT `+usageSumColumns+` `+usageFrom).Scan(&totals.Stages, &totals.TokensTotal, &totals.TokensKnownStages, &totals.TokensIn, &totals.TokensOut, &totals.CostUSD, &totals.CostKnownStages, &totals.SplitUnknownStages, &totals.TokensReasoning, &totals.CacheRead, &totals.CacheWrite, &totals.CacheKnownStages)
if err != nil {
if strings.Contains(err.Error(), "no such table") {
return nil, nil
@@ -92,5 +108,6 @@ FROM step_usage`).Scan(&totals.Stages, &totals.TokensTotal, &totals.TokensKnownS
}
totals.CostUnknownStages = totals.Stages - totals.CostKnownStages
totals.SplitKnown = totals.TokensKnownStages > 0 && totals.SplitUnknownStages == 0
totals.CacheKnown = totals.TokensKnownStages > 0 && totals.CacheKnownStages == totals.TokensKnownStages
return &totals, nil
}
+32 -4
View File
@@ -197,6 +197,14 @@ type RunSummary struct {
// session directories and an estimate of its database rows. It is cached
// server-side; 0 means not measured yet (a background backfill fills it).
SizeBytes int64 `json:"size_bytes"`
// Usage is the totalled reported usage of the run's model stages. The run
// list sums it with one SQL aggregate per batch over the stored per-stage
// usage rows; the run detail sums the same rows in one pass over its loaded
// steps (TotalUsage). It is absent for a run with no model usage recorded
// (including historical runs and servers predating the list aggregate), so
// a client shows nothing rather than zero. A total is partial when
// TokensKnownStages < Stages.
Usage *RunUsage `json:"usage,omitempty"`
}
// RunGraphVisit contains only routing and lifecycle facts, never private step
@@ -317,6 +325,14 @@ type StageUsage struct {
SplitKnown bool `json:"split_known,omitempty"`
CostUSD float64 `json:"cost_usd,omitempty"`
CostKnown bool `json:"cost_known"`
// TokensReasoning, CacheRead and CacheWrite are the remaining components of
// a split report (TokensTotal = in + out + reasoning + cache read + cache
// write). They are recorded only when CacheKnown; a row stored before they
// were kept has CacheKnown=false, so its absent counts are unknown, not zero.
TokensReasoning int64 `json:"tokens_reasoning,omitempty"`
CacheRead int64 `json:"cache_read,omitempty"`
CacheWrite int64 `json:"cache_write,omitempty"`
CacheKnown bool `json:"cache_known,omitempty"`
}
// RunUsage is the totalled reported usage of a run's model stages. It is summed
@@ -336,6 +352,15 @@ type RunUsage struct {
SplitUnknownStages int `json:"split_unknown_stages,omitempty"`
CostKnownStages int `json:"cost_known_stages"`
CostUnknownStages int `json:"cost_unknown_stages,omitempty"`
// TokensReasoning, CacheRead and CacheWrite sum the stages that recorded the
// component split (CacheKnownStages). CacheKnown is true only when every
// token-reporting stage did, so fresh (total minus cache read) versus cached
// is never presented from a partial split.
TokensReasoning int64 `json:"tokens_reasoning,omitempty"`
CacheRead int64 `json:"cache_read,omitempty"`
CacheWrite int64 `json:"cache_write,omitempty"`
CacheKnown bool `json:"cache_known,omitempty"`
CacheKnownStages int `json:"cache_known_stages,omitempty"`
}
// TotalUsage sums the per-stage usage of a run's steps into one run total. It is
@@ -360,6 +385,12 @@ func TotalUsage(steps []RunStep) *RunUsage {
splitComplete = false
total.SplitUnknownStages++
}
if usage.CacheKnown {
total.CacheKnownStages++
total.TokensReasoning += usage.TokensReasoning
total.CacheRead += usage.CacheRead
total.CacheWrite += usage.CacheWrite
}
}
if usage.CostKnown {
total.CostKnownStages++
@@ -372,6 +403,7 @@ func TotalUsage(steps []RunStep) *RunUsage {
return nil
}
total.SplitKnown = total.TokensKnownStages > 0 && splitComplete
total.CacheKnown = total.TokensKnownStages > 0 && total.CacheKnownStages == total.TokensKnownStages
return total
}
@@ -433,10 +465,6 @@ type RunDetail struct {
Input map[string]any `json:"input"`
Output map[string]any `json:"output"`
Steps []RunStep `json:"steps"`
// Usage is the totalled reported usage of the run's model stages, summed in
// one pass over Steps. It is absent for a run with no model usage recorded
// (including historical runs).
Usage *RunUsage `json:"usage,omitempty"`
// TitleSource is "model" when Title came from the shared summariser or
// "fallback" when it was derived from the prompt; TitleModel names the model
// that produced a model title. They are retained so a diagnosis can tell the
+6
View File
@@ -37,6 +37,12 @@ type UsageTotals struct {
SplitUnknownStages int `json:"split_unknown_stages,omitempty"`
CostKnownStages int `json:"cost_known_stages"`
CostUnknownStages int `json:"cost_unknown_stages,omitempty"`
// The cache/reasoning components, as in RunUsage.
TokensReasoning int64 `json:"tokens_reasoning,omitempty"`
CacheRead int64 `json:"cache_read,omitempty"`
CacheWrite int64 `json:"cache_write,omitempty"`
CacheKnown bool `json:"cache_known,omitempty"`
CacheKnownStages int `json:"cache_known_stages,omitempty"`
}
type UsageStats struct {
+4
View File
@@ -1448,6 +1448,10 @@ func stageUsage(snapshot stage.BudgetSnapshot) *protocol.StageUsage {
usage.TokensTotal = snapshot.UsedTokens
if snapshot.SplitKnown {
usage.TokensIn, usage.TokensOut, usage.SplitKnown = snapshot.TokensInput, snapshot.TokensOutput, true
// The rest of the component split is kept too: for a caching harness
// most of the total is cache reads, which a client shows as cached
// rather than fresh tokens.
usage.TokensReasoning, usage.CacheRead, usage.CacheWrite, usage.CacheKnown = snapshot.TokensReasoning, snapshot.CacheRead, snapshot.CacheWrite, true
}
}
if snapshot.USDKnown {
+4 -2
View File
@@ -31,11 +31,13 @@ func TestStageUsageCarriesHonestUnknowns(t *testing.T) {
}{
{"no usage stays absent", stage.BudgetSnapshot{}, nil},
{"split and cost", stage.BudgetSnapshot{UsedTokens: 7, TokensKnown: true, SplitKnown: true, TokensInput: 3, TokensOutput: 4, UsedUSD: .5, USDKnown: true},
&protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true, CostUSD: .5, CostKnown: true}},
&protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true, CostUSD: .5, CostKnown: true, CacheKnown: true}},
{"cache and reasoning components carry through", stage.BudgetSnapshot{UsedTokens: 22_400_000, TokensKnown: true, SplitKnown: true, TokensInput: 900_000, TokensOutput: 150_000, TokensReasoning: 50_000, CacheRead: 21_250_000, CacheWrite: 50_000},
&protocol.StageUsage{TokensIn: 900_000, TokensOut: 150_000, TokensTotal: 22_400_000, TokensKnown: true, SplitKnown: true, TokensReasoning: 50_000, CacheRead: 21_250_000, CacheWrite: 50_000, CacheKnown: true}},
{"total only (Pi)", stage.BudgetSnapshot{UsedTokens: 17, TokensKnown: true},
&protocol.StageUsage{TokensTotal: 17, TokensKnown: true}},
{"no cost", stage.BudgetSnapshot{UsedTokens: 5, TokensKnown: true, SplitKnown: true, TokensInput: 5},
&protocol.StageUsage{TokensIn: 5, TokensTotal: 5, TokensKnown: true, SplitKnown: true}},
&protocol.StageUsage{TokensIn: 5, TokensTotal: 5, TokensKnown: true, SplitKnown: true, CacheKnown: true}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {