Merge lane-retry: record and expose per-run model usage (tokens in/out, cost, model)
This commit is contained in:
+28
-4
@@ -65,8 +65,14 @@ The usage summary is aggregated over all stored history, including retry attempt
|
|||||||
total/active runs, outcomes, success rate, average completed duration, pipeline
|
total/active runs, outcomes, success rate, average completed duration, pipeline
|
||||||
usage and a zero-filled 14-day daily activity chart (UTC). Success rate is
|
usage and a zero-filled 14-day daily activity chart (UTC). Success rate is
|
||||||
succeeded / (succeeded + failed); active, canceled and interrupted runs are excluded.
|
succeeded / (succeeded + failed); active, canceled and interrupted runs are excluded.
|
||||||
The average includes all runs with a finish timestamp. No model cost/token counts
|
The average includes all runs with a finish timestamp. It also totals the reported
|
||||||
are inferred from missing data. `pipelines stats --json` exposes these aggregates.
|
model usage across history: `tokens_total`, an in/out split only when every
|
||||||
|
token-reporting stage reported one, and the summed cost, plus counts of stages
|
||||||
|
that did or did not report a cost. A stage whose provider reported no cost is
|
||||||
|
counted as cost-unknown and never contributes a fabricated or zero cost, and no
|
||||||
|
cost/token count is inferred from output length or transcript text. The totals are
|
||||||
|
the sum of the per-run totals `inspect` shows. `pipelines stats --json` exposes
|
||||||
|
these aggregates.
|
||||||
|
|
||||||
Use `:status` on any TUI page to open server and ntfy information. Scroll with
|
Use `:status` on any TUI page to open server and ntfy information. Scroll with
|
||||||
arrows or `j/k`, PgUp/PgDn; `r` refreshes and Esc closes without changing the page.
|
arrows or `j/k`, PgUp/PgDn; `r` refreshes and Esc closes without changing the page.
|
||||||
@@ -404,7 +410,17 @@ because that stage never runs a model; `--model` on a pipeline with no model sta
|
|||||||
is accepted and inert. The effective model reaches the stage and its
|
is accepted and inert. The effective model reaches the stage and its
|
||||||
repair/formatter path from the same resolved definition, and each step's evidence
|
repair/formatter path from the same resolved definition, and each step's evidence
|
||||||
records the provider/model it actually ran with (`inspect <id> --json` exposes
|
records the provider/model it actually ran with (`inspect <id> --json` exposes
|
||||||
per-step `provider`/`model`, and the run's graph shows the effective pair). The
|
per-step `provider`/`model`, and the run's graph shows the effective pair). Each
|
||||||
|
model step also records the usage the harness reported, carried from the same
|
||||||
|
accounting the stage already used for its budget: `usage.tokens_total`, the
|
||||||
|
`tokens_in`/`tokens_out` split when the harness reported it (`split_known`),
|
||||||
|
`cost_usd` when reported, and the explicit `tokens_known`/`cost_known`/
|
||||||
|
`split_known` markers. The run detail totals those stages as `usage`, so a
|
||||||
|
mixed-model run shows each stage's model, tokens and cost and the run total. A
|
||||||
|
stage whose provider reported no cost shows its tokens with `cost_known: false`
|
||||||
|
(never a fabricated or zero cost), and a harness that reports only a total (Pi)
|
||||||
|
shows `tokens_total` with the split marked unavailable rather than presenting the
|
||||||
|
total as input tokens. The
|
||||||
CLI is the deliverable; a TUI picker for choosing overrides is a documented
|
CLI is the deliverable; a TUI picker for choosing overrides is a documented
|
||||||
follow-up, not half-built.
|
follow-up, not half-built.
|
||||||
|
|
||||||
@@ -622,6 +638,13 @@ retry potentially delivered notifications and do not accept model budgets.
|
|||||||
|
|
||||||
Pinned definitions contain the configured limits. Run transcripts retain limits,
|
Pinned definitions contain the configured limits. Run transcripts retain limits,
|
||||||
attempt counts and reported budget usage, also shown in the TUI's Evidence pane.
|
attempt counts and reported budget usage, also shown in the TUI's Evidence pane.
|
||||||
|
The same reported usage is recorded on the run as typed per-step evidence and
|
||||||
|
totalled for the run, so `inspect <id> --json` exposes tokens in/out, the total,
|
||||||
|
the cost when reported and the model each stage ran with. A stage revisited within
|
||||||
|
a run shares one budget, so each step records only that visit's increment of it
|
||||||
|
and the run total is not the cumulative budget counted again. Missing or unknown cost
|
||||||
|
or split is reported as unavailable (never zero or inferred), and a historical or
|
||||||
|
asked-for run recorded before this evidence existed simply omits `usage`.
|
||||||
Exact stage outputs are authoritative; a transcript over the 32 KiB retained
|
Exact stage outputs are authoritative; a transcript over the 32 KiB retained
|
||||||
bound is stored as a valid, credential-redacted preview envelope that preserves
|
bound is stored as a valid, credential-redacted preview envelope that preserves
|
||||||
available stage limits and failure/format diagnostics and names the retained
|
available stage limits and failure/format diagnostics and names the retained
|
||||||
@@ -709,7 +732,8 @@ anything already renamed. A removal never half-completes.
|
|||||||
`artifacts/.retention-staging-<timestamp>-<pid>/` directory (a same-filesystem
|
`artifacts/.retention-staging-<timestamp>-<pid>/` directory (a same-filesystem
|
||||||
rename), deletes the rows for those runs in one immediate write transaction
|
rename), deletes the rows for those runs in one immediate write transaction
|
||||||
(`step_notifications`, `step_file_changes`, `step_control`, `step_artifacts`,
|
(`step_notifications`, `step_file_changes`, `step_control`, `step_artifacts`,
|
||||||
`step_verdicts`, `run_session_owners`, `run_steps`, `run_retries`, then
|
`step_verdicts`, `step_session_exports`, `step_usage`, `run_session_owners`,
|
||||||
|
`run_steps`, `run_retries`, then
|
||||||
`pipeline_runs`, batched at 500 ids), commits, and removes the staging directory.
|
`pipeline_runs`, batched at 500 ids), commits, and removes the staging directory.
|
||||||
A symlink at `<root>/<id>` is removed as a link without touching its target; a
|
A symlink at `<root>/<id>` is removed as a link without touching its target; a
|
||||||
regular file at `<root>/<id>` is refused. A symlinked artifacts root is followed
|
regular file at `<root>/<id>` is refused. A symlinked artifacts root is followed
|
||||||
|
|||||||
@@ -59,7 +59,8 @@ parallel/distributed task engine or a guarantee that model reports are correct.
|
|||||||
|
|
||||||
SQLite retains pipelines/immutable versions, runs, recorded stage visits,
|
SQLite retains pipelines/immutable versions, runs, recorded stage visits,
|
||||||
inputs/outputs, status/errors, audit transcripts, session ownership/references,
|
inputs/outputs, status/errors, audit transcripts, session ownership/references,
|
||||||
retry links, schedules/state, notification receipts and file changes. Not everything
|
reported per-stage model usage (tokens/cost), retry links, schedules/state,
|
||||||
|
notification receipts and file changes. Not everything
|
||||||
is in that database: drafts/settings are client files; complete OpenCode sessions
|
is in that database: drafts/settings are client files; complete OpenCode sessions
|
||||||
and credentials use OpenCode storage; repository artifacts are files; ntfy has its
|
and credentials use OpenCode storage; repository artifacts are files; ntfy has its
|
||||||
own storage. A pipeline ZIP is not a full backup of these components.
|
own storage. A pipeline ZIP is not a full backup of these components.
|
||||||
@@ -98,6 +99,13 @@ Use the authoritative `CLI_PREFIX` from the prompt:
|
|||||||
without changing its input or steps. The stored prompt is unchanged, and
|
without changing its input or steps. The stored prompt is unchanged, and
|
||||||
listing/renaming never calls a model.
|
listing/renaming never calls a model.
|
||||||
- `CLI_PREFIX inspect RUN_ID --json`: selected run, visits and retained evidence.
|
- `CLI_PREFIX inspect RUN_ID --json`: selected run, visits and retained evidence.
|
||||||
|
Each model step carries the usage its harness reported (`usage.tokens_total`,
|
||||||
|
`tokens_in`/`tokens_out` only when `split_known`, `cost_usd` only when
|
||||||
|
`cost_known`, plus the model the step actually ran with); the run carries the
|
||||||
|
totalled `usage` over those stages, so a mixed-model run shows what each stage
|
||||||
|
cost. An unreported cost or split is shown as unavailable, never as zero and
|
||||||
|
never inferred from output length or transcript text; a historical or
|
||||||
|
pre-upgrade run simply omits `usage`.
|
||||||
Exact stage outputs are authoritative; a transcript over the 32 KiB retained
|
Exact stage outputs are authoritative; a transcript over the 32 KiB retained
|
||||||
bound is a valid, credential-redacted preview envelope (with preserved stage
|
bound is a valid, credential-redacted preview envelope (with preserved stage
|
||||||
limits, failure/format diagnostics and the retained session pointer), not the
|
limits, failure/format diagnostics and the retained session pointer), not the
|
||||||
|
|||||||
@@ -34,6 +34,40 @@ Accounting failures stop budget-limited stages. In-flight calls can overshoot;
|
|||||||
these are **not provider-side hard caps or a guaranteed bill ceiling**. Reported
|
these are **not provider-side hard caps or a guaranteed bill ceiling**. Reported
|
||||||
USD may validly be zero; do not fabricate prices/accounting from output length.
|
USD may validly be zero; do not fabricate prices/accounting from output length.
|
||||||
|
|
||||||
|
Per-run usage is recorded as typed step evidence from the usage the stage already
|
||||||
|
measured for its budget (no second harness read and no transcript parse), and
|
||||||
|
totalled for the run and over all history. A stage revisited within a run shares
|
||||||
|
one budget, so each step records only that visit's increment of it and the run
|
||||||
|
total is not the cumulative stage budget counted again; a revisit that reports
|
||||||
|
nothing new (a failure or timeout before any usage poll, or a Pi visit that never
|
||||||
|
reaches a usage stat) records no usage for that visit rather than a known zero.
|
||||||
|
`CLI_PREFIX inspect RUN_ID --json`
|
||||||
|
exposes each model step's `tokens_total`, `tokens_in`/`tokens_out` (only when
|
||||||
|
`split_known`), `cost_usd` (only when `cost_known`) and the model it ran with,
|
||||||
|
plus the totalled `usage`; `CLI_PREFIX stats --json` adds the history-wide totals,
|
||||||
|
which equal the sum of the per-run totals. A stage that reported no cost exposes
|
||||||
|
its tokens and marks the cost unavailable rather than showing zero; a harness that
|
||||||
|
reports only a total (Pi) exposes the total and marks the in/out split
|
||||||
|
unavailable rather than presenting the total as input tokens. An upgraded or
|
||||||
|
historical database gains an empty usage table with no backfill, so old runs omit
|
||||||
|
`usage` instead of showing zero. This is current source, not deployment evidence.
|
||||||
|
|
||||||
|
Dated current-source evidence (2026-10-17, this machine). A throwaway read-only
|
||||||
|
reader (`internal` store API) over a scratch copy of
|
||||||
|
`~/.local/share/pipelines/data/pipelines.db`, taken with
|
||||||
|
`sqlite3 "file:$HOME/.local/share/pipelines/data/pipelines.db?mode=ro" ".backup <scratch>"`
|
||||||
|
so the live DB is never mutated, opened 228 runs and found **no** `step_usage`
|
||||||
|
rows: every stored run predates this change, so per-run figures are unavailable
|
||||||
|
rather than zero. Run 314 (this item's own run) and run 239 (whose implementer
|
||||||
|
step ran `opencode-go/kimi-k2.7-code`, a stage model different from the pinned
|
||||||
|
`opencode-go/deepseek-v4.1-flash`) both report `usage=(none recorded)`, and
|
||||||
|
`stats` omits the history-wide totals. The recorded figures themselves are
|
||||||
|
covered by the passing current-source round trips — `internal/pipeline/usage_test.go`,
|
||||||
|
`internal/server/usage_test.go` (including `inspect --json` over a websocket) and
|
||||||
|
the pre-upgrade/read-bounded/stats-sum cases — not by a live run, because the
|
||||||
|
deployed server predates this change and was not restarted. No live model call,
|
||||||
|
deploy or rebuild produced this evidence.
|
||||||
|
|
||||||
A live V2 trial found `/generate` returned text while its retained session had no
|
A live V2 trial found `/generate` returned text while its retained session had no
|
||||||
messages and zero counters. Current source uses tracked `/prompt` → bounded
|
messages and zero counters. Current source uses tracked `/prompt` → bounded
|
||||||
active-session polling → message readback for **budgeted V2 text-only calls, with
|
active-session polling → message readback for **budgeted V2 text-only calls, with
|
||||||
|
|||||||
+17
-1
@@ -524,7 +524,23 @@ func (a App) Run(ctx context.Context, args []string) error {
|
|||||||
if u.Total > u.Active {
|
if u.Total > u.Active {
|
||||||
average = time.Duration(u.AverageSeconds * float64(time.Second)).Round(time.Second).String()
|
average = time.Duration(u.AverageSeconds * float64(time.Second)).Round(time.Second).String()
|
||||||
}
|
}
|
||||||
fmt.Fprintf(a.Out, "Usage · all history\n%d runs · %d succeeded · %d failed · %d canceled · %d interrupted · %d active\nAverage completed duration: %s\n\nPipeline\tRuns\tSucceeded\tFailed\tActive\n", u.Total, u.Succeeded, u.Failed, u.Canceled, u.Interrupted, u.Active, average)
|
fmt.Fprintf(a.Out, "Usage · all history\n%d runs · %d succeeded · %d failed · %d canceled · %d interrupted · %d active\nAverage completed duration: %s\n", u.Total, u.Succeeded, u.Failed, u.Canceled, u.Interrupted, u.Active, average)
|
||||||
|
if t := u.Usage; t != nil {
|
||||||
|
line := fmt.Sprintf("Reported model usage: %d tokens over %d stage(s)", t.TokensTotal, t.Stages)
|
||||||
|
if t.SplitKnown {
|
||||||
|
line += fmt.Sprintf(" (%d in / %d out)", t.TokensIn, t.TokensOut)
|
||||||
|
} else if t.SplitUnknownStages > 0 {
|
||||||
|
line += " (in/out split unavailable)"
|
||||||
|
}
|
||||||
|
if t.CostKnownStages > 0 {
|
||||||
|
line += fmt.Sprintf(" · $%.6f", t.CostUSD)
|
||||||
|
}
|
||||||
|
if t.CostUnknownStages > 0 {
|
||||||
|
line += fmt.Sprintf(" · cost unavailable for %d stage(s)", t.CostUnknownStages)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(a.Out, line)
|
||||||
|
}
|
||||||
|
fmt.Fprint(a.Out, "\nPipeline\tRuns\tSucceeded\tFailed\tActive\n")
|
||||||
for _, p := range u.Pipelines {
|
for _, p := range u.Pipelines {
|
||||||
fmt.Fprintf(a.Out, "%s\t%d\t%d\t%d\t%d\n", p.Name, p.Total, p.Succeeded, p.Failed, p.Active)
|
fmt.Fprintf(a.Out, "%s\t%d\t%d\t%d\t%d\n", p.Name, p.Total, p.Succeeded, p.Failed, p.Active)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -626,6 +626,7 @@ func deleteRetentionRows(ctx context.Context, tx *sql.Tx, ids []int64) error {
|
|||||||
{"step_artifacts", "run_id"},
|
{"step_artifacts", "run_id"},
|
||||||
{"step_verdicts", "run_id"},
|
{"step_verdicts", "run_id"},
|
||||||
{"step_session_exports", "run_id"},
|
{"step_session_exports", "run_id"},
|
||||||
|
{"step_usage", "run_id"},
|
||||||
{"run_session_owners", "run_id"},
|
{"run_session_owners", "run_id"},
|
||||||
{"run_steps", "run_id"},
|
{"run_steps", "run_id"},
|
||||||
} {
|
} {
|
||||||
|
|||||||
@@ -50,6 +50,9 @@ PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(
|
|||||||
CREATE TABLE IF NOT EXISTS step_session_exports (
|
CREATE TABLE IF NOT EXISTS step_session_exports (
|
||||||
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, export_json TEXT NOT NULL,
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, export_json TEXT NOT NULL,
|
||||||
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
||||||
|
CREATE TABLE IF NOT EXISTS step_usage (
|
||||||
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, usage_json TEXT NOT NULL,
|
||||||
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
||||||
CREATE TABLE IF NOT EXISTS run_session_owners (
|
CREATE TABLE IF NOT EXISTS run_session_owners (
|
||||||
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, reference TEXT NOT NULL DEFAULT '',
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, reference TEXT NOT NULL DEFAULT '',
|
||||||
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
||||||
@@ -347,6 +350,7 @@ COALESCE((SELECT changes_json FROM step_file_changes f WHERE f.run_id=run_steps.
|
|||||||
COALESCE((SELECT artifacts_json FROM step_artifacts a WHERE a.run_id=run_steps.run_id AND a.sequence=run_steps.sequence),'null'),
|
COALESCE((SELECT artifacts_json FROM step_artifacts a WHERE a.run_id=run_steps.run_id AND a.sequence=run_steps.sequence),'null'),
|
||||||
COALESCE((SELECT verdict_json FROM step_verdicts v WHERE v.run_id=run_steps.run_id AND v.sequence=run_steps.sequence),''),
|
COALESCE((SELECT verdict_json FROM step_verdicts v WHERE v.run_id=run_steps.run_id AND v.sequence=run_steps.sequence),''),
|
||||||
COALESCE((SELECT export_json FROM step_session_exports x WHERE x.run_id=run_steps.run_id AND x.sequence=run_steps.sequence),''),
|
COALESCE((SELECT export_json FROM step_session_exports x WHERE x.run_id=run_steps.run_id AND x.sequence=run_steps.sequence),''),
|
||||||
|
COALESCE((SELECT usage_json FROM step_usage u WHERE u.run_id=run_steps.run_id AND u.sequence=run_steps.sequence),''),
|
||||||
strftime('%Y-%m-%dT%H:%M:%SZ',started_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',finished_at),'') FROM run_steps WHERE run_id=? ORDER BY sequence`, id)
|
strftime('%Y-%m-%dT%H:%M:%SZ',started_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',finished_at),'') FROM run_steps WHERE run_id=? ORDER BY sequence`, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -355,8 +359,8 @@ strftime('%Y-%m-%dT%H:%M:%SZ',started_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ'
|
|||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var step protocol.RunStep
|
var step protocol.RunStep
|
||||||
var notifications string
|
var notifications string
|
||||||
var fileChanges, artifacts, storedVerdict, sessionExport string
|
var fileChanges, artifacts, storedVerdict, sessionExport, storedUsage string
|
||||||
if err := rows.Scan(&step.Sequence, &step.Stage, &step.Status, &step.Outcome, &step.Target, &input, &output, &step.Diagnostic, &step.SessionID, &step.AgentActive, &step.Transcript, &step.Provider, &step.Model, ¬ifications, &fileChanges, &artifacts, &storedVerdict, &sessionExport, &step.StartedAt, &step.FinishedAt); err != nil {
|
if err := rows.Scan(&step.Sequence, &step.Stage, &step.Status, &step.Outcome, &step.Target, &input, &output, &step.Diagnostic, &step.SessionID, &step.AgentActive, &step.Transcript, &step.Provider, &step.Model, ¬ifications, &fileChanges, &artifacts, &storedVerdict, &sessionExport, &storedUsage, &step.StartedAt, &step.FinishedAt); err != nil {
|
||||||
rows.Close()
|
rows.Close()
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -404,6 +408,12 @@ strftime('%Y-%m-%dT%H:%M:%SZ',started_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ'
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if storedUsage != "" {
|
||||||
|
if err := json.Unmarshal([]byte(storedUsage), &step.Usage); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
// Loaded below after the rows cursor closes to preserve SQLite connection use.
|
// Loaded below after the rows cursor closes to preserve SQLite connection use.
|
||||||
r.Steps = append(r.Steps, step)
|
r.Steps = append(r.Steps, step)
|
||||||
}
|
}
|
||||||
@@ -425,6 +435,9 @@ strftime('%Y-%m-%dT%H:%M:%SZ',started_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ'
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// The run total is one pass over the steps already loaded above: it adds no
|
||||||
|
// query and no transcript parse.
|
||||||
|
r.Usage = protocol.TotalUsage(r.Steps)
|
||||||
if err := tx.Commit(); err != nil {
|
if err := tx.Commit(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -653,6 +666,19 @@ WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome,
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// The reported usage the stage already measured for budget accounting is
|
||||||
|
// persisted as step-scoped evidence in the same transaction, mirroring the
|
||||||
|
// other step evidence. It is written only when the stage actually reported
|
||||||
|
// usage, so a historical or non-model step stays absent rather than zero.
|
||||||
|
if step.Usage != nil {
|
||||||
|
data, err := json.Marshal(step.Usage)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := tx.ExecContext(ctx, "INSERT INTO step_usage(run_id,sequence,usage_json) VALUES(?,?,?)", id, step.Sequence, string(data)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
return tx.Commit()
|
return tx.Commit()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,9 @@ package pipeline
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -53,5 +56,41 @@ func (s *Store) UsageStats(ctx context.Context) (*protocol.UsageStats, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
rows.Close()
|
rows.Close()
|
||||||
|
usage, err := usageTotals(ctx, tx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
stats.Usage = usage
|
||||||
return stats, tx.Commit()
|
return stats, tx.Commit()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// usageTotals is the one extra aggregate query the history-wide usage totals add,
|
||||||
|
// over the same step_usage rows the per-run figures read. It sums reported totals
|
||||||
|
// with json_extract, counts token/cost-reporting stages and leaves a stage that
|
||||||
|
// reported no cost uncounted rather than adding a fabricated zero; an old or
|
||||||
|
// read-only database with no step_usage table yields no totals, not an error.
|
||||||
|
func usageTotals(ctx context.Context, tx *sql.Tx) (*protocol.UsageTotals, error) {
|
||||||
|
var totals protocol.UsageTotals
|
||||||
|
err := tx.QueryRowContext(ctx, `SELECT
|
||||||
|
COALESCE(COUNT(*),0),
|
||||||
|
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') THEN json_extract(usage_json,'$.tokens_total') ELSE 0 END),0),
|
||||||
|
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') THEN 1 ELSE 0 END),0),
|
||||||
|
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.split_known') THEN json_extract(usage_json,'$.tokens_in') ELSE 0 END),0),
|
||||||
|
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.split_known') THEN json_extract(usage_json,'$.tokens_out') ELSE 0 END),0),
|
||||||
|
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.cost_known') THEN json_extract(usage_json,'$.cost_usd') ELSE 0 END),0),
|
||||||
|
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.cost_known') THEN 1 ELSE 0 END),0),
|
||||||
|
COALESCE(SUM(CASE WHEN json_extract(usage_json,'$.tokens_known') AND NOT COALESCE(json_extract(usage_json,'$.split_known'),0) THEN 1 ELSE 0 END),0)
|
||||||
|
FROM step_usage`).Scan(&totals.Stages, &totals.TokensTotal, &totals.TokensKnownStages, &totals.TokensIn, &totals.TokensOut, &totals.CostUSD, &totals.CostKnownStages, &totals.SplitUnknownStages)
|
||||||
|
if err != nil {
|
||||||
|
if strings.Contains(err.Error(), "no such table") {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if totals.Stages == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
totals.CostUnknownStages = totals.Stages - totals.CostKnownStages
|
||||||
|
totals.SplitKnown = totals.TokensKnownStages > 0 && totals.SplitUnknownStages == 0
|
||||||
|
return &totals, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,192 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||||
|
)
|
||||||
|
|
||||||
|
// finishUsageStep begins and finishes one completed step carrying the given
|
||||||
|
// reported usage, the same store path the runner uses.
|
||||||
|
func finishUsageStep(t *testing.T, store *Store, run int64, sequence int, key string, usage *protocol.StageUsage) {
|
||||||
|
t.Helper()
|
||||||
|
if err := store.BeginStep(context.Background(), run, sequence, key, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := store.FinishStep(context.Background(), run, protocol.RunStep{
|
||||||
|
Sequence: sequence, Stage: key, Status: "completed", Outcome: "success", Target: "#success",
|
||||||
|
Output: map[string]any{"summary": "done"}, Usage: usage,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStepUsageRoundTripRunTotalAndStatsSum proves per-step usage round-trips
|
||||||
|
// through the store, the run total is the single-pass sum of the reported stages,
|
||||||
|
// and the history-wide aggregate equals the sum of the per-run figures over the
|
||||||
|
// same rows.
|
||||||
|
func TestStepUsageRoundTripRunTotalAndStatsSum(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
store, err := Open(filepath.Join(t.TempDir(), "usage.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer store.Close()
|
||||||
|
if _, err := store.Submit(ctx, "usage", Template([]string{"one"}, true)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
items, _ := store.List(ctx)
|
||||||
|
definition, err := store.Get(ctx, items[0].ID, 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run 1: a split-cost stage and a Pi-style total-only stage.
|
||||||
|
run1, _, err := store.CreateRun(ctx, definition, "usage-1", "/repo", false, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
finishUsageStep(t, store, run1, 1, "one", &protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true, CostUSD: .5, CostKnown: true})
|
||||||
|
finishUsageStep(t, store, run1, 2, "one", &protocol.StageUsage{TokensTotal: 17, TokensKnown: true, CostUSD: .25, CostKnown: true})
|
||||||
|
|
||||||
|
// Run 2: tokens with no reported cost.
|
||||||
|
run2, _, err := store.CreateRun(ctx, definition, "usage-2", "/repo", false, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
finishUsageStep(t, store, run2, 1, "one", &protocol.StageUsage{TokensTotal: 5, TokensKnown: true})
|
||||||
|
|
||||||
|
detail, err := store.GetRun(ctx, run1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(detail.Steps) != 2 || detail.Steps[0].Usage == nil || detail.Steps[1].Usage == nil {
|
||||||
|
t.Fatalf("per-step usage did not round-trip: %+v", detail.Steps)
|
||||||
|
}
|
||||||
|
if got := detail.Steps[0].Usage; got.TokensIn != 3 || got.TokensOut != 4 || got.TokensTotal != 7 || !got.SplitKnown || got.CostUSD != .5 || !got.CostKnown {
|
||||||
|
t.Fatalf("split stage usage = %+v", got)
|
||||||
|
}
|
||||||
|
if got := detail.Steps[1].Usage; got.TokensTotal != 17 || got.SplitKnown || got.CostUSD != .25 || !got.CostKnown {
|
||||||
|
t.Fatalf("total-only stage usage = %+v", got)
|
||||||
|
}
|
||||||
|
if detail.Usage == nil || detail.Usage.TokensTotal != 24 || detail.Usage.CostUSD != .75 || detail.Usage.Stages != 2 || detail.Usage.CostKnownStages != 2 || detail.Usage.SplitUnknownStages != 1 || detail.Usage.SplitKnown {
|
||||||
|
t.Fatalf("run total = %+v", detail.Usage)
|
||||||
|
}
|
||||||
|
|
||||||
|
stats, err := store.UsageStats(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if stats.Usage == nil {
|
||||||
|
t.Fatal("history-wide usage totals missing")
|
||||||
|
}
|
||||||
|
var tokens int64
|
||||||
|
var cost float64
|
||||||
|
knownCostStages := 0
|
||||||
|
for _, id := range []int64{run1, run2} {
|
||||||
|
run, err := store.GetRun(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if run.Usage == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
tokens += run.Usage.TokensTotal
|
||||||
|
cost += run.Usage.CostUSD
|
||||||
|
knownCostStages += run.Usage.CostKnownStages
|
||||||
|
}
|
||||||
|
if stats.Usage.TokensTotal != tokens || stats.Usage.CostUSD != cost || stats.Usage.CostKnownStages != knownCostStages {
|
||||||
|
t.Fatalf("stats totals %+v do not equal the summed per-run figures tokens=%d cost=%v costStages=%d", stats.Usage, tokens, cost, knownCostStages)
|
||||||
|
}
|
||||||
|
if stats.Usage.SplitKnown || stats.Usage.SplitUnknownStages != 2 {
|
||||||
|
t.Fatalf("partial split presented as complete: %+v", stats.Usage)
|
||||||
|
}
|
||||||
|
// The run with no cost must never contribute a fabricated cost.
|
||||||
|
if stats.Usage.CostUnknownStages != 1 {
|
||||||
|
t.Fatalf("cost-unknown stage counted as known: %+v", stats.Usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStepUsageReadIsBoundedOverManySteps proves the read path stays a single
|
||||||
|
// per-step indexed projection plus one pass for the run total: a run with many
|
||||||
|
// recorded steps loads them all, totals them and preserves each step's usage,
|
||||||
|
// with no per-step usage query beyond the existing step loop.
|
||||||
|
func TestStepUsageReadIsBoundedOverManySteps(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
store, err := Open(filepath.Join(t.TempDir(), "bounded.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer store.Close()
|
||||||
|
if _, err := store.Submit(ctx, "bounded", Template([]string{"one"}, true)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
items, _ := store.List(ctx)
|
||||||
|
definition, _ := store.Get(ctx, items[0].ID, 1)
|
||||||
|
id, _, err := store.CreateRun(ctx, definition, "bounded", "/repo", false, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
const steps = 120
|
||||||
|
for sequence := 1; sequence <= steps; sequence++ {
|
||||||
|
finishUsageStep(t, store, id, sequence, "one", &protocol.StageUsage{TokensIn: 1, TokensOut: 1, TokensTotal: 2, TokensKnown: true, SplitKnown: true, CostUSD: .001, CostKnown: true})
|
||||||
|
}
|
||||||
|
detail, err := store.GetRun(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(detail.Steps) != steps || detail.Usage == nil || detail.Usage.Stages != steps || detail.Usage.TokensTotal != 2*steps {
|
||||||
|
t.Fatalf("bounded read lost steps or totals: %d %+v", len(detail.Steps), detail.Usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStepUsageStoreUpgrade proves a database created before step_usage exists
|
||||||
|
// opens cleanly, gains the empty table on init with no backfill, keeps listing
|
||||||
|
// runs and, for the historical step, reports usage absent rather than zero.
|
||||||
|
func TestStepUsageStoreUpgrade(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
path := filepath.Join(t.TempDir(), "upgrade-usage.db")
|
||||||
|
store, err := Open(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := store.Submit(ctx, "upgrade-usage", Template([]string{"one"}, true)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
items, _ := store.List(ctx)
|
||||||
|
definition, _ := store.Get(ctx, items[0].ID, 1)
|
||||||
|
id, _, err := store.CreateRun(ctx, definition, "upgrade-usage", "/repo", false, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
finishUsageStep(t, store, id, 1, "one", &protocol.StageUsage{TokensTotal: 9, TokensKnown: true})
|
||||||
|
// Simulate a database created before step_usage existed.
|
||||||
|
if _, err := store.db.ExecContext(ctx, "DROP TABLE step_usage"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := store.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
store, err = Open(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("a pre-usage database did not open cleanly", err)
|
||||||
|
}
|
||||||
|
defer store.Close()
|
||||||
|
runs, err := store.ListRuns(ctx)
|
||||||
|
if err != nil || len(runs) != 1 {
|
||||||
|
t.Fatalf("legacy run list lost history: %v %v", runs, err)
|
||||||
|
}
|
||||||
|
detail, err := store.GetRun(ctx, id)
|
||||||
|
if err != nil || len(detail.Steps) != 1 {
|
||||||
|
t.Fatalf("legacy run/step did not load: %v %v", detail, err)
|
||||||
|
}
|
||||||
|
if detail.Steps[0].Usage != nil || detail.Usage != nil {
|
||||||
|
t.Fatalf("historical usage was backfilled or shown as zero: %+v", detail)
|
||||||
|
}
|
||||||
|
stats, err := store.UsageStats(ctx)
|
||||||
|
if err != nil || stats.Usage != nil {
|
||||||
|
t.Fatalf("post-upgrade stats = %+v err=%v", stats.Usage, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -160,6 +160,7 @@ type RunStep struct {
|
|||||||
FileChanges *StageFileChanges `json:"file_changes,omitempty"`
|
FileChanges *StageFileChanges `json:"file_changes,omitempty"`
|
||||||
Verdict *StageVerdict `json:"verdict,omitempty"`
|
Verdict *StageVerdict `json:"verdict,omitempty"`
|
||||||
SessionExport *SessionExport `json:"session_export,omitempty"`
|
SessionExport *SessionExport `json:"session_export,omitempty"`
|
||||||
|
Usage *StageUsage `json:"usage,omitempty"`
|
||||||
Notifications []NotificationReceipt `json:"notifications,omitempty"`
|
Notifications []NotificationReceipt `json:"notifications,omitempty"`
|
||||||
Artifacts []StageArtifact `json:"artifacts,omitempty"`
|
Artifacts []StageArtifact `json:"artifacts,omitempty"`
|
||||||
Sequence int `json:"sequence"`
|
Sequence int `json:"sequence"`
|
||||||
@@ -196,6 +197,80 @@ type StageArtifact struct {
|
|||||||
URL string `json:"url"`
|
URL string `json:"url"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StageUsage is the usage one model stage actually reported, carried from the
|
||||||
|
// stage's already-measured budget accounting rather than a second measurement or
|
||||||
|
// a transcript parse. TokensTotal/TokensKnown are the reported total; TokensIn
|
||||||
|
// and TokensOut are set only when the harness reported the input/output split
|
||||||
|
// (SplitKnown), so a harness that reports only a total (Pi) never has its total
|
||||||
|
// presented as input tokens. CostUSD is present only when CostKnown: a stage
|
||||||
|
// whose provider reported no cost carries CostKnown=false and no fabricated or
|
||||||
|
// zero cost.
|
||||||
|
type StageUsage struct {
|
||||||
|
TokensIn int64 `json:"tokens_in,omitempty"`
|
||||||
|
TokensOut int64 `json:"tokens_out,omitempty"`
|
||||||
|
TokensTotal int64 `json:"tokens_total"`
|
||||||
|
TokensKnown bool `json:"tokens_known"`
|
||||||
|
SplitKnown bool `json:"split_known,omitempty"`
|
||||||
|
CostUSD float64 `json:"cost_usd,omitempty"`
|
||||||
|
CostKnown bool `json:"cost_known"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunUsage is the totalled reported usage of a run's model stages. It is summed
|
||||||
|
// in one pass over the run's already-loaded steps, so reading it adds no query.
|
||||||
|
// A stage that reported no cost is counted in CostUnknownStages and contributes
|
||||||
|
// no cost; a stage that reported only a total contributes to TokensTotal but not
|
||||||
|
// to the in/out split, and SplitKnown is true only when every stage that
|
||||||
|
// reported tokens also reported the split.
|
||||||
|
type RunUsage struct {
|
||||||
|
TokensTotal int64 `json:"tokens_total"`
|
||||||
|
TokensIn int64 `json:"tokens_in,omitempty"`
|
||||||
|
TokensOut int64 `json:"tokens_out,omitempty"`
|
||||||
|
SplitKnown bool `json:"split_known"`
|
||||||
|
CostUSD float64 `json:"cost_usd,omitempty"`
|
||||||
|
Stages int `json:"stages"`
|
||||||
|
TokensKnownStages int `json:"tokens_known_stages"`
|
||||||
|
SplitUnknownStages int `json:"split_unknown_stages,omitempty"`
|
||||||
|
CostKnownStages int `json:"cost_known_stages"`
|
||||||
|
CostUnknownStages int `json:"cost_unknown_stages,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TotalUsage sums the per-stage usage of a run's steps into one run total. It is
|
||||||
|
// the one definition of the run total, so per-run inspection cannot drift from
|
||||||
|
// the history-wide aggregate over the same rows.
|
||||||
|
func TotalUsage(steps []RunStep) *RunUsage {
|
||||||
|
total := &RunUsage{}
|
||||||
|
splitComplete := true
|
||||||
|
for i := range steps {
|
||||||
|
usage := steps[i].Usage
|
||||||
|
if usage == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
total.Stages++
|
||||||
|
if usage.TokensKnown {
|
||||||
|
total.TokensKnownStages++
|
||||||
|
total.TokensTotal += usage.TokensTotal
|
||||||
|
if usage.SplitKnown {
|
||||||
|
total.TokensIn += usage.TokensIn
|
||||||
|
total.TokensOut += usage.TokensOut
|
||||||
|
} else {
|
||||||
|
splitComplete = false
|
||||||
|
total.SplitUnknownStages++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if usage.CostKnown {
|
||||||
|
total.CostKnownStages++
|
||||||
|
total.CostUSD += usage.CostUSD
|
||||||
|
} else {
|
||||||
|
total.CostUnknownStages++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if total.Stages == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
total.SplitKnown = total.TokensKnownStages > 0 && splitComplete
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
type MemoryRead struct {
|
type MemoryRead struct {
|
||||||
Source string `json:"source"`
|
Source string `json:"source"`
|
||||||
Sequence int `json:"sequence"` // zero denotes original run input
|
Sequence int `json:"sequence"` // zero denotes original run input
|
||||||
@@ -249,6 +324,10 @@ type RunDetail struct {
|
|||||||
Input map[string]any `json:"input"`
|
Input map[string]any `json:"input"`
|
||||||
Output map[string]any `json:"output"`
|
Output map[string]any `json:"output"`
|
||||||
Steps []RunStep `json:"steps"`
|
Steps []RunStep `json:"steps"`
|
||||||
|
// Usage is the totalled reported usage of the run's model stages, summed in
|
||||||
|
// one pass over Steps. It is absent for a run with no model usage recorded
|
||||||
|
// (including historical runs).
|
||||||
|
Usage *RunUsage `json:"usage,omitempty"`
|
||||||
// TitleSource is "model" when Title came from the shared summariser or
|
// TitleSource is "model" when Title came from the shared summariser or
|
||||||
// "fallback" when it was derived from the prompt; TitleModel names the model
|
// "fallback" when it was derived from the prompt; TitleModel names the model
|
||||||
// that produced a model title. They are retained so a diagnosis can tell the
|
// that produced a model title. They are retained so a diagnosis can tell the
|
||||||
|
|||||||
@@ -18,8 +18,32 @@ type DailyUsage struct {
|
|||||||
Date string `json:"date"`
|
Date string `json:"date"`
|
||||||
Runs int `json:"runs"`
|
Runs int `json:"runs"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UsageTotals is the summed reported model usage over all stored run history. It
|
||||||
|
// is summed from the same per-step usage rows the per-run figures use, so the
|
||||||
|
// history-wide totals equal the sum of the per-run totals. A stage that reported
|
||||||
|
// no cost counts in CostUnknownStages and contributes no fabricated or zero cost;
|
||||||
|
// a stage that reported only a total (Pi) contributes to TokensTotal but not to
|
||||||
|
// the in/out split, and SplitKnown is true only when every token-reporting stage
|
||||||
|
// reported a split.
|
||||||
|
type UsageTotals struct {
|
||||||
|
TokensTotal int64 `json:"tokens_total"`
|
||||||
|
TokensIn int64 `json:"tokens_in,omitempty"`
|
||||||
|
TokensOut int64 `json:"tokens_out,omitempty"`
|
||||||
|
SplitKnown bool `json:"split_known"`
|
||||||
|
CostUSD float64 `json:"cost_usd,omitempty"`
|
||||||
|
Stages int `json:"stages"`
|
||||||
|
TokensKnownStages int `json:"tokens_known_stages"`
|
||||||
|
SplitUnknownStages int `json:"split_unknown_stages,omitempty"`
|
||||||
|
CostKnownStages int `json:"cost_known_stages"`
|
||||||
|
CostUnknownStages int `json:"cost_unknown_stages,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
type UsageStats struct {
|
type UsageStats struct {
|
||||||
RunCounts
|
RunCounts
|
||||||
Pipelines []PipelineUsage `json:"pipelines"`
|
Pipelines []PipelineUsage `json:"pipelines"`
|
||||||
Days []DailyUsage `json:"days"`
|
Days []DailyUsage `json:"days"`
|
||||||
|
// Usage is the history-wide sum of the per-run reported usage. It is absent
|
||||||
|
// for a database with no recorded model usage (including a pre-upgrade one).
|
||||||
|
Usage *UsageTotals `json:"usage,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -742,6 +742,28 @@ func completedSteps(steps []protocol.RunStep) int {
|
|||||||
return n
|
return n
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stageUsage carries the usage a model stage already measured for budget
|
||||||
|
// accounting onto the run record. It reports a cost only when the provider
|
||||||
|
// reported one and an in/out split only when the harness reported it, so an
|
||||||
|
// unknown cost or split is never presented as zero. It returns nil when no usage
|
||||||
|
// was observed, so the step stays absent rather than zero.
|
||||||
|
func stageUsage(snapshot stage.BudgetSnapshot) *protocol.StageUsage {
|
||||||
|
if !snapshot.TokensKnown && !snapshot.USDKnown {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
usage := &protocol.StageUsage{TokensKnown: snapshot.TokensKnown, CostKnown: snapshot.USDKnown}
|
||||||
|
if snapshot.TokensKnown {
|
||||||
|
usage.TokensTotal = snapshot.UsedTokens
|
||||||
|
if snapshot.SplitKnown {
|
||||||
|
usage.TokensIn, usage.TokensOut, usage.SplitKnown = snapshot.TokensInput, snapshot.TokensOutput, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if snapshot.USDKnown {
|
||||||
|
usage.CostUSD = snapshot.UsedUSD
|
||||||
|
}
|
||||||
|
return usage
|
||||||
|
}
|
||||||
|
|
||||||
// finishStage persists one step, refreshes the shared memory for a completed
|
// finishStage persists one step, refreshes the shared memory for a completed
|
||||||
// step, notifies and classifies the step's target. Both the executed path and
|
// step, notifies and classifies the step's target. Both the executed path and
|
||||||
// the skipped path use it, so persistence, notification and terminal routing
|
// the skipped path use it, so persistence, notification and terminal routing
|
||||||
@@ -966,6 +988,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
|||||||
var err error
|
var err error
|
||||||
stageCtx := ctx
|
stageCtx := ctx
|
||||||
var modelConfig stage.ModelConfig
|
var modelConfig stage.ModelConfig
|
||||||
|
var budgetBefore stage.BudgetSnapshot
|
||||||
if stage.EffectiveType(def.Type) == stage.ModelType {
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
||||||
spec, specErr := (stage.ModelStage{}).Spec(def)
|
spec, specErr := (stage.ModelStage{}).Spec(def)
|
||||||
if specErr != nil {
|
if specErr != nil {
|
||||||
@@ -976,6 +999,10 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
|||||||
if budgets[key] == nil {
|
if budgets[key] == nil {
|
||||||
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
|
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
|
||||||
}
|
}
|
||||||
|
// The budget is shared across every visit to this stage key, so capture
|
||||||
|
// its cumulative usage before the visit and record only this visit's
|
||||||
|
// increment on the step, never the cumulative total again.
|
||||||
|
budgetBefore = budgets[key].Snapshot()
|
||||||
stageCtx = stage.WithBudget(ctx, budgets[key])
|
stageCtx = stage.WithBudget(ctx, budgets[key])
|
||||||
}
|
}
|
||||||
snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second)
|
snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
@@ -1058,10 +1085,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
|||||||
step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes, SessionExport: sessionExport, Artifacts: artifacts, ArtifactsSkipped: artifactsSkipped, ArtifactsTruncated: artifactsTruncated}
|
step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes, SessionExport: sessionExport, Artifacts: artifacts, ArtifactsSkipped: artifactsSkipped, ArtifactsTruncated: artifactsTruncated}
|
||||||
// Stamp the effective provider/model this visit actually ran with for every
|
// Stamp the effective provider/model this visit actually ran with for every
|
||||||
// model stage, including failures, so the app/TUI evidence reflects any
|
// model stage, including failures, so the app/TUI evidence reflects any
|
||||||
// override. Non-model stages keep empty values.
|
// override. Non-model stages keep empty values. The usage the stage already
|
||||||
|
// measured for budget accounting is carried onto the same step, never
|
||||||
|
// re-read from the harness or parsed from the transcript.
|
||||||
if stage.EffectiveType(def.Type) == stage.ModelType {
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
||||||
step.Provider = modelConfig.Provider
|
step.Provider = modelConfig.Provider
|
||||||
step.Model = modelConfig.Model
|
step.Model = modelConfig.Model
|
||||||
|
if budget := budgets[key]; budget != nil {
|
||||||
|
step.Usage = stageUsage(budget.Snapshot().Since(budgetBefore))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if budget := budgets[key]; budget != nil {
|
if budget := budgets[key]; budget != nil {
|
||||||
step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries)
|
step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries)
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -101,6 +102,18 @@ esac
|
|||||||
if err != nil || stored.Steps[1].Transcript != detail.Steps[1].Transcript {
|
if err != nil || stored.Steps[1].Transcript != detail.Steps[1].Transcript {
|
||||||
t.Fatal("evidence was not retained", err)
|
t.Fatal("evidence was not retained", err)
|
||||||
}
|
}
|
||||||
|
// The usage each visit already measured for budget accounting is recorded per
|
||||||
|
// step as that visit's own increment, so the run total is neither a second
|
||||||
|
// measurement nor the cumulative stage budget counted twice.
|
||||||
|
if stored.Steps[0].Usage == nil || stored.Steps[1].Usage == nil || stored.Steps[0].Usage.TokensTotal != 3 || stored.Steps[1].Usage.TokensTotal != 3 {
|
||||||
|
t.Fatalf("per-visit usage not recorded: %+v", stored.Steps)
|
||||||
|
}
|
||||||
|
if stored.Steps[1].Usage.CostUSD != .4 || !stored.Steps[1].Usage.CostKnown {
|
||||||
|
t.Fatalf("reported cost not recorded: %+v", stored.Steps[1].Usage)
|
||||||
|
}
|
||||||
|
if stored.Usage == nil || stored.Usage.TokensTotal != 6 || math.Abs(stored.Usage.CostUSD-0.8) > 1e-9 {
|
||||||
|
t.Fatalf("run usage total double counted the shared budget: %+v", stored.Usage)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerStageTimeoutAndNoShortOuterDeadline(t *testing.T) {
|
func TestRunnerStageTimeoutAndNoShortOuterDeadline(t *testing.T) {
|
||||||
|
|||||||
@@ -0,0 +1,226 @@
|
|||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.a-wan.net/alan/pipelines/internal/cli"
|
||||||
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||||
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||||
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestStageUsageCarriesHonestUnknowns proves the mapping from the stage's
|
||||||
|
// already-measured budget snapshot to the run record never fabricates a cost or
|
||||||
|
// a split: a reported split and cost carry through, a total-only report (Pi)
|
||||||
|
// keeps the total with the split unavailable, a missing cost stays unknown, and
|
||||||
|
// no observed usage stays absent.
|
||||||
|
func TestStageUsageCarriesHonestUnknowns(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
snapshot stage.BudgetSnapshot
|
||||||
|
want *protocol.StageUsage
|
||||||
|
}{
|
||||||
|
{"no usage stays absent", stage.BudgetSnapshot{}, nil},
|
||||||
|
{"split and cost", stage.BudgetSnapshot{UsedTokens: 7, TokensKnown: true, SplitKnown: true, TokensInput: 3, TokensOutput: 4, UsedUSD: .5, USDKnown: true},
|
||||||
|
&protocol.StageUsage{TokensIn: 3, TokensOut: 4, TokensTotal: 7, TokensKnown: true, SplitKnown: true, CostUSD: .5, CostKnown: true}},
|
||||||
|
{"total only (Pi)", stage.BudgetSnapshot{UsedTokens: 17, TokensKnown: true},
|
||||||
|
&protocol.StageUsage{TokensTotal: 17, TokensKnown: true}},
|
||||||
|
{"no cost", stage.BudgetSnapshot{UsedTokens: 5, TokensKnown: true, SplitKnown: true, TokensInput: 5},
|
||||||
|
&protocol.StageUsage{TokensIn: 5, TokensTotal: 5, TokensKnown: true, SplitKnown: true}},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got := stageUsage(tc.snapshot)
|
||||||
|
if (got == nil) != (tc.want == nil) {
|
||||||
|
t.Fatalf("stageUsage = %+v, want %+v", got, tc.want)
|
||||||
|
}
|
||||||
|
if got != nil && *got != *tc.want {
|
||||||
|
t.Fatalf("stageUsage = %+v, want %+v", got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeBudgetOpencode installs a fake OpenCode CLI that reports per-session usage
|
||||||
|
// for a tracked V2 stage, optionally omitting the cost so a no-cost report can be
|
||||||
|
// exercised. It mirrors the existing stage-limits fake-budget script.
|
||||||
|
func fakeBudgetOpencode(t *testing.T, reportCost bool) {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
usageLine := `printf '{"data":{"id":"%%s","cost":%%s,"tokens":{"input":%%s,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}' "$id" "$cost" "$tokens"`
|
||||||
|
if !reportCost {
|
||||||
|
usageLine = `printf '{"data":{"id":"%%s","tokens":{"input":%%s,"output":0,"reasoning":0,"cache":{"read":0,"write":0}}}}' "$id" "$tokens"`
|
||||||
|
}
|
||||||
|
script := fmt.Sprintf(`#!/bin/sh
|
||||||
|
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
|
||||||
|
case "$3" in
|
||||||
|
/api/session)
|
||||||
|
n=0; if [ -f %q/count ]; then n=$(cat %q/count); fi
|
||||||
|
n=$((n+1)); echo "$n" > %q/count
|
||||||
|
printf '{"data":{"id":"ses_usage_%%s"}}' "$n" ;;
|
||||||
|
*/prompt)
|
||||||
|
id=${3#/api/session/}; id=${id%%/prompt}
|
||||||
|
touch %q/"$id"
|
||||||
|
echo '{}' ;;
|
||||||
|
/api/session/active) echo '{"data":{}}' ;;
|
||||||
|
*/message*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"{\"outcome\":\"success\",\"output\":{\"summary\":\"done\"}}"}]}]}' ;;
|
||||||
|
*/interrupt) echo '{}' ;;
|
||||||
|
/api/session/ses_usage_*)
|
||||||
|
id=${3#/api/session/}; tokens=0; cost=0
|
||||||
|
if [ -f %q/"$id" ]; then tokens=3; cost=0.4; fi
|
||||||
|
`+usageLine+` ;;
|
||||||
|
*) exit 99 ;;
|
||||||
|
esac
|
||||||
|
`, dir, dir, dir, dir, dir)
|
||||||
|
exe := filepath.Join(dir, "opencode")
|
||||||
|
if err := os.WriteFile(exe, []byte(script), 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("PIPELINES_OPENCODE", exe)
|
||||||
|
}
|
||||||
|
|
||||||
|
func budgetPipeline(t *testing.T, ctx context.Context, store *pipeline.Store, names []string, configs string) {
|
||||||
|
t.Helper()
|
||||||
|
source := strings.ReplaceAll(pipeline.Template(names, true), "timeout=1h", "timeout=1h\n"+configs)
|
||||||
|
if _, err := store.Submit(ctx, "budget-usage", source); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunnerRecordsPerStageUsageAcrossMixedModels proves a run with a per-stage
|
||||||
|
// model override records each visit's effective model and the usage that visit
|
||||||
|
// already measured, and that the run total spans those stages. It uses the
|
||||||
|
// existing fake-budget pattern (a fake OpenCode CLI) rather than a second
|
||||||
|
// measurement.
|
||||||
|
func TestRunnerRecordsPerStageUsageAcrossMixedModels(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
fakeBudgetOpencode(t, true)
|
||||||
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "mixed.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer store.Close()
|
||||||
|
budgetPipeline(t, ctx, store, []string{"one", "two"}, "budget_tokens=100")
|
||||||
|
items, _ := store.List(ctx)
|
||||||
|
r, err := newRunner(ctx, store)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mixed", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
r.wg.Wait()
|
||||||
|
detail, err := store.GetRun(ctx, run.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if detail.Status != "succeeded" || len(detail.Steps) != 2 {
|
||||||
|
t.Fatalf("mixed run did not complete: %+v", detail)
|
||||||
|
}
|
||||||
|
if detail.Steps[0].Provider != "acme" || detail.Steps[0].Model != "strong" {
|
||||||
|
t.Fatalf("overridden stage model = %+v", detail.Steps[0])
|
||||||
|
}
|
||||||
|
if detail.Steps[1].Provider != "opencode-go" || detail.Steps[1].Model != "deepseek-v4.1-flash" {
|
||||||
|
t.Fatalf("pinned stage model = %+v", detail.Steps[1])
|
||||||
|
}
|
||||||
|
for i, step := range detail.Steps {
|
||||||
|
if step.Usage == nil || step.Usage.TokensTotal != 3 || !step.Usage.SplitKnown || step.Usage.TokensIn != 3 || step.Usage.CostUSD != .4 || !step.Usage.CostKnown {
|
||||||
|
t.Fatalf("step %d usage = %+v", i, step.Usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if detail.Usage == nil || detail.Usage.TokensTotal != 6 || math.Abs(detail.Usage.CostUSD-.8) > 1e-9 || detail.Usage.Stages != 2 || !detail.Usage.SplitKnown {
|
||||||
|
t.Fatalf("run total = %+v", detail.Usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunnerKeepsCostUnknownWithTokensShown proves a stage whose harness reports
|
||||||
|
// tokens but no cost exposes the tokens and marks the cost unavailable, never
|
||||||
|
// zero, and that the run aggregate counts the cost-unknown stage.
|
||||||
|
func TestRunnerKeepsCostUnknownWithTokensShown(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
fakeBudgetOpencode(t, false)
|
||||||
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "nocost.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer store.Close()
|
||||||
|
budgetPipeline(t, ctx, store, []string{"one"}, "budget_tokens=100")
|
||||||
|
items, _ := store.List(ctx)
|
||||||
|
r, err := newRunner(ctx, store)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "nocost", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
r.wg.Wait()
|
||||||
|
detail, err := store.GetRun(ctx, run.ID)
|
||||||
|
if err != nil || len(detail.Steps) != 1 {
|
||||||
|
t.Fatalf("run did not complete: %+v %v", detail, err)
|
||||||
|
}
|
||||||
|
usage := detail.Steps[0].Usage
|
||||||
|
if usage == nil || usage.TokensTotal != 3 || !usage.TokensKnown || usage.CostKnown || usage.CostUSD != 0 {
|
||||||
|
t.Fatalf("cost-unknown usage = %+v", usage)
|
||||||
|
}
|
||||||
|
if detail.Usage == nil || detail.Usage.CostKnownStages != 0 || detail.Usage.CostUnknownStages != 1 || detail.Usage.TokensTotal != 3 {
|
||||||
|
t.Fatalf("cost-unknown run total = %+v", detail.Usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestInspectJSONExposesUsageThroughCLI proves the real `pipelines inspect
|
||||||
|
// --json` surface carries the per-step usage and the run total over a websocket
|
||||||
|
// round trip, after a stage whose usage came from the harness, not a second read.
|
||||||
|
func TestInspectJSONExposesUsageThroughCLI(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
fakeBudgetOpencode(t, true)
|
||||||
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "inspect.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer store.Close()
|
||||||
|
budgetPipeline(t, ctx, store, []string{"one"}, "budget_tokens=100")
|
||||||
|
items, _ := store.List(ctx)
|
||||||
|
handler := Handler(ctx, store).(*runHandler)
|
||||||
|
defer func() { cancel(); WaitRuns(handler) }()
|
||||||
|
srv := httptest.NewServer(handler)
|
||||||
|
defer srv.Close()
|
||||||
|
run, err := handler.runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "inspect-usage", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
handler.runner.wg.Wait()
|
||||||
|
url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
||||||
|
var out, errOut bytes.Buffer
|
||||||
|
app := cli.App{Out: &out, Err: &errOut, In: strings.NewReader("")}
|
||||||
|
if err := app.Run(ctx, []string{"--url", url, "inspect", fmt.Sprint(run.ID), "--json"}); err != nil {
|
||||||
|
t.Fatal(err, out.String(), errOut.String())
|
||||||
|
}
|
||||||
|
var detail protocol.RunDetail
|
||||||
|
if err := json.Unmarshal(out.Bytes(), &detail); err != nil {
|
||||||
|
t.Fatalf("inspect --json was not run detail JSON: %v %s", err, out.String())
|
||||||
|
}
|
||||||
|
if len(detail.Steps) != 1 || detail.Steps[0].Usage == nil {
|
||||||
|
t.Fatalf("inspect --json carried no per-step usage: %s", out.String())
|
||||||
|
}
|
||||||
|
usage := detail.Steps[0].Usage
|
||||||
|
if usage.TokensTotal != 3 || !usage.TokensKnown || !usage.SplitKnown || usage.TokensIn != 3 || usage.CostUSD != .4 || !usage.CostKnown {
|
||||||
|
t.Fatalf("step usage = %+v", usage)
|
||||||
|
}
|
||||||
|
if detail.Usage == nil || detail.Usage.TokensTotal != 3 || detail.Usage.CostUSD != .4 {
|
||||||
|
t.Fatalf("run usage total = %+v", detail.Usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -57,6 +57,25 @@ type BudgetSnapshot struct {
|
|||||||
USDKnown bool `json:"usd_known"`
|
USDKnown bool `json:"usd_known"`
|
||||||
TokensKnown bool `json:"tokens_known"`
|
TokensKnown bool `json:"tokens_known"`
|
||||||
Attempts int `json:"attempts"`
|
Attempts int `json:"attempts"`
|
||||||
|
// Component token counts the harness reported, accumulated with the same
|
||||||
|
// per-key max/delta accounting as UsedTokens. They are meaningful only when
|
||||||
|
// SplitKnown is true: a harness that reports only a total (Pi's
|
||||||
|
// get_session_stats) leaves them at zero with SplitKnown false, so its total
|
||||||
|
// is never presented as input tokens.
|
||||||
|
TokensInput int64 `json:"tokens_input,omitempty"`
|
||||||
|
TokensOutput int64 `json:"tokens_output,omitempty"`
|
||||||
|
TokensReasoning int64 `json:"tokens_reasoning,omitempty"`
|
||||||
|
CacheRead int64 `json:"cache_read,omitempty"`
|
||||||
|
CacheWrite int64 `json:"cache_write,omitempty"`
|
||||||
|
SplitKnown bool `json:"split_known,omitempty"`
|
||||||
|
// Report counters advance only when a harness report of that kind is
|
||||||
|
// observed. Since uses them to tell a real increment from a visit that
|
||||||
|
// observed nothing: a revisit that reports no new tokens or cost must not
|
||||||
|
// present the budget's unchanged known flags as a known zero for that visit.
|
||||||
|
// They are internal bookkeeping and stay out of the audit snapshot.
|
||||||
|
TokenReports int `json:"-"`
|
||||||
|
CostReports int `json:"-"`
|
||||||
|
SplitReports int `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type reportedUsage struct {
|
type reportedUsage struct {
|
||||||
@@ -64,6 +83,9 @@ type reportedUsage struct {
|
|||||||
Tokens *struct {
|
Tokens *struct {
|
||||||
Input, Output, Reasoning *int64
|
Input, Output, Reasoning *int64
|
||||||
Cache *struct{ Read, Write *int64 }
|
Cache *struct{ Read, Write *int64 }
|
||||||
|
// Total is a reported total with no component split (Pi). Exactly one of
|
||||||
|
// the component set or Total is expected; components win when both appear.
|
||||||
|
Total *int64
|
||||||
} `json:"tokens"`
|
} `json:"tokens"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -79,7 +101,41 @@ func NewBudget(usd float64, tokens int64) *Budget {
|
|||||||
}
|
}
|
||||||
func (b *Budget) Enabled() bool { return b != nil && (b.state.LimitUSD > 0 || b.state.LimitTokens > 0) }
|
func (b *Budget) Enabled() bool { return b != nil && (b.state.LimitUSD > 0 || b.state.LimitTokens > 0) }
|
||||||
func (b *Budget) Snapshot() BudgetSnapshot { b.mu.Lock(); defer b.mu.Unlock(); return b.state }
|
func (b *Budget) Snapshot() BudgetSnapshot { b.mu.Lock(); defer b.mu.Unlock(); return b.state }
|
||||||
func (b *Budget) Attempt() { b.mu.Lock(); b.state.Attempts++; b.mu.Unlock() }
|
|
||||||
|
// Since returns only the usage this budget added after an earlier snapshot of
|
||||||
|
// itself. A budget is shared across every visit to a stage key, so recording its
|
||||||
|
// cumulative Snapshot() on each visit would count the earlier visits' usage
|
||||||
|
// again; a visit records Snapshot().Since(before) so the run total sums to what
|
||||||
|
// the provider actually reported. The counts and cost only accumulate, so the
|
||||||
|
// deltas are non-negative.
|
||||||
|
//
|
||||||
|
// The known flags describe the increment, not the later cumulative snapshot: a
|
||||||
|
// revisit that observed no new report of a kind (a failure or timeout before any
|
||||||
|
// usage poll, or a Pi visit that never reached get_session_stats) must not
|
||||||
|
// present the unchanged known flag as a known zero. Such a kind is reported as
|
||||||
|
// unknown for this visit, so the caller records no usage at all rather than a
|
||||||
|
// fabricated zero cost or token count.
|
||||||
|
func (s BudgetSnapshot) Since(before BudgetSnapshot) BudgetSnapshot {
|
||||||
|
after := s
|
||||||
|
after.UsedUSD -= before.UsedUSD
|
||||||
|
after.UsedTokens -= before.UsedTokens
|
||||||
|
after.TokensInput -= before.TokensInput
|
||||||
|
after.TokensOutput -= before.TokensOutput
|
||||||
|
after.TokensReasoning -= before.TokensReasoning
|
||||||
|
after.CacheRead -= before.CacheRead
|
||||||
|
after.CacheWrite -= before.CacheWrite
|
||||||
|
if after.CostReports == before.CostReports {
|
||||||
|
after.USDKnown = false
|
||||||
|
}
|
||||||
|
if after.TokenReports == before.TokenReports {
|
||||||
|
after.TokensKnown = false
|
||||||
|
}
|
||||||
|
if after.SplitReports == before.SplitReports {
|
||||||
|
after.SplitKnown = false
|
||||||
|
}
|
||||||
|
return after
|
||||||
|
}
|
||||||
|
func (b *Budget) Attempt() { b.mu.Lock(); b.state.Attempts++; b.mu.Unlock() }
|
||||||
func (b *Budget) checkLocked() error {
|
func (b *Budget) checkLocked() error {
|
||||||
if b.state.LimitUSD > 0 && b.state.UsedUSD >= b.state.LimitUSD {
|
if b.state.LimitUSD > 0 && b.state.UsedUSD >= b.state.LimitUSD {
|
||||||
return fmt.Errorf("%w: USD %.6f / %.6f", ErrBudgetExceeded, b.state.UsedUSD, b.state.LimitUSD)
|
return fmt.Errorf("%w: USD %.6f / %.6f", ErrBudgetExceeded, b.state.UsedUSD, b.state.LimitUSD)
|
||||||
@@ -107,11 +163,14 @@ func (b *Budget) observe(key string, usage reportedUsage) error {
|
|||||||
}
|
}
|
||||||
b.state.UsedUSD = total
|
b.state.UsedUSD = total
|
||||||
old.UsedUSD, old.USDKnown, b.state.USDKnown = value, true, true
|
old.UsedUSD, old.USDKnown, b.state.USDKnown = value, true, true
|
||||||
|
b.state.CostReports++
|
||||||
} else if b.state.LimitUSD > 0 {
|
} else if b.state.LimitUSD > 0 {
|
||||||
return fmt.Errorf("%w: cost was not reported", ErrBudgetAccounting)
|
return fmt.Errorf("%w: cost was not reported", ErrBudgetAccounting)
|
||||||
}
|
}
|
||||||
t := usage.Tokens
|
t := usage.Tokens
|
||||||
if t != nil && t.Input != nil && t.Output != nil && t.Reasoning != nil && t.Cache != nil && t.Cache.Read != nil && t.Cache.Write != nil {
|
components := t != nil && t.Input != nil && t.Output != nil && t.Reasoning != nil && t.Cache != nil && t.Cache.Read != nil && t.Cache.Write != nil
|
||||||
|
switch {
|
||||||
|
case components:
|
||||||
var total int64
|
var total int64
|
||||||
for _, value := range []int64{*t.Input, *t.Output, *t.Reasoning, *t.Cache.Read, *t.Cache.Write} {
|
for _, value := range []int64{*t.Input, *t.Output, *t.Reasoning, *t.Cache.Read, *t.Cache.Write} {
|
||||||
if value < 0 || value > math.MaxInt64-total {
|
if value < 0 || value > math.MaxInt64-total {
|
||||||
@@ -125,7 +184,42 @@ func (b *Budget) observe(key string, usage reportedUsage) error {
|
|||||||
}
|
}
|
||||||
b.state.UsedTokens += total - old.UsedTokens
|
b.state.UsedTokens += total - old.UsedTokens
|
||||||
old.UsedTokens, old.TokensKnown, b.state.TokensKnown = total, true, true
|
old.UsedTokens, old.TokensKnown, b.state.TokensKnown = total, true, true
|
||||||
} else if b.state.LimitTokens > 0 {
|
// Component tokens use the same per-key max/delta accounting so a repeated
|
||||||
|
// poll cannot double count. The split is recorded only when the harness
|
||||||
|
// actually reported separate components, never synthesised from a total.
|
||||||
|
for _, component := range []struct {
|
||||||
|
reported int64
|
||||||
|
state *int64
|
||||||
|
record *int64
|
||||||
|
}{
|
||||||
|
{*t.Input, &b.state.TokensInput, &old.TokensInput},
|
||||||
|
{*t.Output, &b.state.TokensOutput, &old.TokensOutput},
|
||||||
|
{*t.Reasoning, &b.state.TokensReasoning, &old.TokensReasoning},
|
||||||
|
{*t.Cache.Read, &b.state.CacheRead, &old.CacheRead},
|
||||||
|
{*t.Cache.Write, &b.state.CacheWrite, &old.CacheWrite},
|
||||||
|
} {
|
||||||
|
value := max(component.reported, *component.record)
|
||||||
|
*component.state += value - *component.record
|
||||||
|
*component.record = value
|
||||||
|
}
|
||||||
|
old.SplitKnown, b.state.SplitKnown = true, true
|
||||||
|
b.state.TokenReports++
|
||||||
|
b.state.SplitReports++
|
||||||
|
case t != nil && t.Total != nil:
|
||||||
|
// A total-only report (Pi) counts toward the token total and the limit but
|
||||||
|
// leaves the in/out split unknown rather than presenting the total as
|
||||||
|
// input tokens.
|
||||||
|
if *t.Total < 0 {
|
||||||
|
return fmt.Errorf("%w: invalid token count", ErrBudgetAccounting)
|
||||||
|
}
|
||||||
|
total := max(*t.Total, old.UsedTokens)
|
||||||
|
if total-old.UsedTokens > math.MaxInt64-b.state.UsedTokens {
|
||||||
|
return fmt.Errorf("%w: token count overflow", ErrBudgetAccounting)
|
||||||
|
}
|
||||||
|
b.state.UsedTokens += total - old.UsedTokens
|
||||||
|
old.UsedTokens, old.TokensKnown, b.state.TokensKnown = total, true, true
|
||||||
|
b.state.TokenReports++
|
||||||
|
case b.state.LimitTokens > 0:
|
||||||
return fmt.Errorf("%w: complete token usage was not reported", ErrBudgetAccounting)
|
return fmt.Errorf("%w: complete token usage was not reported", ErrBudgetAccounting)
|
||||||
}
|
}
|
||||||
return b.checkLocked()
|
return b.checkLocked()
|
||||||
|
|||||||
@@ -180,6 +180,10 @@ func TestLiveAgentBudgetInterruptsAndDoesNotReplay(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestBudgetLedgerDeduplicatesPollsAndIncludesCacheAndReasoning covers the
|
||||||
|
// reported component split: when a harness reports input/output/reasoning/cache
|
||||||
|
// distinctly the split is retained, and a total-only report (Pi) keeps the total
|
||||||
|
// without presenting it as input tokens.
|
||||||
func TestBudgetLedgerDeduplicatesPollsAndIncludesCacheAndReasoning(t *testing.T) {
|
func TestBudgetLedgerDeduplicatesPollsAndIncludesCacheAndReasoning(t *testing.T) {
|
||||||
b := NewBudget(10, 100)
|
b := NewBudget(10, 100)
|
||||||
data := []byte(`{"data":{"id":"ses_ledger","cost":0.2,"tokens":{"input":3,"output":4,"reasoning":5,"cache":{"read":6,"write":7}}}}`)
|
data := []byte(`{"data":{"id":"ses_ledger","cost":0.2,"tokens":{"input":3,"output":4,"reasoning":5,"cache":{"read":6,"write":7}}}}`)
|
||||||
@@ -350,6 +354,95 @@ func TestUnbudgetedV2AgentMissingUsageStaysUnknownAndNonFatal(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestBudgetRecordsReportedSplitVersusTotalOnly proves the in/out split is
|
||||||
|
// exposed only when the harness reported it: OpenCode's component report carries
|
||||||
|
// the split, while Pi's total-only report exposes the total and leaves the split
|
||||||
|
// unknown rather than presenting the total as input tokens.
|
||||||
|
func TestBudgetRecordsReportedSplitVersusTotalOnly(t *testing.T) {
|
||||||
|
b := NewBudget(0, 0)
|
||||||
|
if err := b.readUsage("ses_split", []byte(`{"data":{"id":"ses_split","cost":0.5,"tokens":{"input":3,"output":4,"reasoning":5,"cache":{"read":6,"write":7}}}}`), false); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s := b.Snapshot()
|
||||||
|
if !s.TokensKnown || !s.USDKnown || !s.SplitKnown || s.UsedTokens != 25 || s.TokensInput != 3 || s.TokensOutput != 4 {
|
||||||
|
t.Fatalf("reported split not retained: %+v", s)
|
||||||
|
}
|
||||||
|
// A repeated poll of the same report must not double count the components.
|
||||||
|
if err := b.readUsage("ses_split", []byte(`{"data":{"id":"ses_split","cost":0.5,"tokens":{"input":3,"output":4,"reasoning":5,"cache":{"read":6,"write":7}}}}`), false); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if s := b.Snapshot(); s.UsedTokens != 25 || s.TokensInput != 3 || s.TokensOutput != 4 {
|
||||||
|
t.Fatalf("repeated poll double counted components: %+v", s)
|
||||||
|
}
|
||||||
|
p := NewBudget(0, 0)
|
||||||
|
pi := &PiClient{budget: p}
|
||||||
|
if err := pi.observeTotals(17, 0.25); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ps := p.Snapshot()
|
||||||
|
if !ps.TokensKnown || ps.UsedTokens != 17 || ps.SplitKnown || ps.TokensInput != 0 || ps.TokensOutput != 0 {
|
||||||
|
t.Fatalf("total-only report presented as a split: %+v", ps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUnknownCostStaysUnknown proves a report without a cost leaves the cost
|
||||||
|
// unknown rather than fabricating a zero, while still recording tokens.
|
||||||
|
func TestUnknownCostStaysUnknown(t *testing.T) {
|
||||||
|
b := NewBudget(0, 0)
|
||||||
|
if err := b.readUsage("ses_nocost", []byte(`{"data":{"id":"ses_nocost","tokens":{"input":5,"output":5,"reasoning":0,"cache":{"read":0,"write":0}}}}`), false); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s := b.Snapshot()
|
||||||
|
if !s.TokensKnown || s.USDKnown || s.UsedUSD != 0 {
|
||||||
|
t.Fatalf("missing cost marked known or fabricated: %+v", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestBudgetSnapshotSinceCountsOnlyTheIncrement proves a visit to a budget shared
|
||||||
|
// across stage revisits records only what that visit added, so a run total does
|
||||||
|
// not count an earlier visit's usage again.
|
||||||
|
func TestBudgetSnapshotSinceCountsOnlyTheIncrement(t *testing.T) {
|
||||||
|
b := NewBudget(0, 0)
|
||||||
|
before := b.Snapshot()
|
||||||
|
if err := b.readUsage("ses_v1", usageResponse("ses_v1", 0.4, 3), false); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if first := b.Snapshot().Since(before); first.UsedTokens != 3 || first.TokensInput != 3 || math.Abs(first.UsedUSD-0.4) > 1e-9 {
|
||||||
|
t.Fatalf("first visit increment = %+v", first)
|
||||||
|
}
|
||||||
|
mid := b.Snapshot()
|
||||||
|
if err := b.readUsage("ses_v2", usageResponse("ses_v2", 0.4, 3), false); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if second := b.Snapshot().Since(mid); second.UsedTokens != 3 || second.TokensInput != 3 || math.Abs(second.UsedUSD-0.4) > 1e-9 {
|
||||||
|
t.Fatalf("second visit increment = %+v", second)
|
||||||
|
}
|
||||||
|
if total := b.Snapshot(); total.UsedTokens != 6 || math.Abs(total.UsedUSD-0.8) > 1e-9 {
|
||||||
|
t.Fatalf("cumulative budget = %+v", total)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestBudgetSnapshotSinceClearsUnknownsWhenNoNewReport proves a visit that
|
||||||
|
// observed no new report does not present the budget's unchanged known flags as
|
||||||
|
// a known zero: a failed or timed-out revisit records no usage rather than a
|
||||||
|
// fabricated zero cost and token total.
|
||||||
|
func TestBudgetSnapshotSinceClearsUnknownsWhenNoNewReport(t *testing.T) {
|
||||||
|
b := NewBudget(0, 0)
|
||||||
|
if err := b.readUsage("ses_reported", usageResponse("ses_reported", 0.4, 3), false); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before := b.Snapshot()
|
||||||
|
if !before.TokensKnown || !before.USDKnown || !before.SplitKnown {
|
||||||
|
t.Fatalf("baseline report not recorded: %+v", before)
|
||||||
|
}
|
||||||
|
// The revisit observes nothing new: the cumulative flags are unchanged but
|
||||||
|
// must describe this visit as unknown, not as a known zero.
|
||||||
|
delta := b.Snapshot().Since(before)
|
||||||
|
if delta.TokensKnown || delta.USDKnown || delta.SplitKnown || delta.UsedTokens != 0 || delta.UsedUSD != 0 {
|
||||||
|
t.Fatalf("no-op revisit fabricated known zero usage: %+v", delta)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFormatRepairUsesTheSameBudget(t *testing.T) {
|
func TestFormatRepairUsesTheSameBudget(t *testing.T) {
|
||||||
def := modelDefinition()
|
def := modelDefinition()
|
||||||
def.Configs["budget_tokens"] = "5"
|
def.Configs["budget_tokens"] = "5"
|
||||||
|
|||||||
@@ -459,11 +459,15 @@ func (c *PiClient) receive(ctx context.Context) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *PiClient) observeTotals(tokens int64, cost float64) error {
|
func (c *PiClient) observeTotals(tokens int64, cost float64) error {
|
||||||
zero := int64(0)
|
// Pi's get_session_stats and message_end events report only a total token
|
||||||
|
// count, so account it as a total with no component split: the run record
|
||||||
|
// must expose the total and mark the in/out split unavailable rather than
|
||||||
|
// presenting the total as input tokens.
|
||||||
return c.budget.observe(c.SessionID, reportedUsage{USD: &cost, Tokens: &struct {
|
return c.budget.observe(c.SessionID, reportedUsage{USD: &cost, Tokens: &struct {
|
||||||
Input, Output, Reasoning *int64
|
Input, Output, Reasoning *int64
|
||||||
Cache *struct{ Read, Write *int64 }
|
Cache *struct{ Read, Write *int64 }
|
||||||
}{Input: &tokens, Output: &zero, Reasoning: &zero, Cache: &struct{ Read, Write *int64 }{&zero, &zero}}})
|
Total *int64
|
||||||
|
}{Total: &tokens}})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *PiClient) observeStats(data []byte) error {
|
func (c *PiClient) observeStats(data []byte) error {
|
||||||
|
|||||||
Reference in New Issue
Block a user