feat(web): display reported chat throughput tokens context and cost
This commit is contained in:
+17
@@ -347,6 +347,23 @@ A new subscriber receives any retained active compaction. Clear stale activity
|
||||
before reconnecting, then preserve fresh replayed activity while history loads.
|
||||
This reports native observations only; it does not request or trigger compaction.
|
||||
|
||||
Usage frames (`chat_event.kind="usage"`) may carry full native snapshots with
|
||||
`usage.scope="session"`, `total_tokens`, `context_tokens`, `context_window`,
|
||||
`cost_usd`, and `output_tokens_per_second`. Optional fields are unknown when
|
||||
absent, not zero; reported zero values are retained explicitly. A new snapshot
|
||||
replaces the previous one—never add repeated/replayed totals together.
|
||||
`rate_basis="generation"` uses a native generation interval;
|
||||
`rate_basis="turn"` denotes an end-to-end average that may include tools/waits.
|
||||
Throughput must not be inferred from output text length.
|
||||
|
||||
Total tokens count processed usage across model calls; they are not the current
|
||||
context size. Context usage/window and USD costs are shown only when available
|
||||
from the harness. In particular, a subscription without reported USD cost is not
|
||||
free, and the browser does not estimate a bill from a price table. Usage frames
|
||||
are session metadata: no assistant/tool message is created and no queued prompt
|
||||
is admitted or completed by a usage update. Subscriptions replay retained usage;
|
||||
native readback refreshes it where supported.
|
||||
|
||||
The brokered OpenCode chat exposes the operator's own sessions, separate from the
|
||||
sessions pipeline runs create. The protocol types are:
|
||||
|
||||
|
||||
@@ -8,6 +8,34 @@ test, or a code change without a capture, is `partial` at best.
|
||||
|
||||
Status legend: `todo` · `partial` (some of it works) · `done` (working + captured).
|
||||
|
||||
### Chat performance and usage (2026-10-21)
|
||||
|
||||
The chat header now shows reported output tokens/sec, total processed tokens,
|
||||
last prompt/context usage and context-window limit, and reported USD cost.
|
||||
Missing values are explicitly **Unavailable**; a genuine reported zero stays
|
||||
zero. Native snapshots replace previous values, including duplicate/reconnect
|
||||
updates and context reductions after compaction. Cumulative processed tokens are
|
||||
never used as context size, output text is never used to estimate tokens, and
|
||||
unreported subscription costs are not guessed from a pricing table.
|
||||
|
||||
Metrics-only updates patch the header rail without repainting the transcript or
|
||||
composer, changing focus/selection, creating messages, or advancing a queued
|
||||
prompt. Last reported values remain available when the turn settles.
|
||||
|
||||
`test/chat-usage-browser.mjs` captures known, missing, partial and zero-valued
|
||||
snapshots at 390/420/1100px plus a 390px short viewport. It checks formatting,
|
||||
no horizontal overflow, unchanged message count, no spurious Stop control,
|
||||
duplicate-snapshot replacement and preserved composer/transcript DOM and cursor.
|
||||
Captures: approved OpenCode temporary `chat-usage-captures/` directory. These
|
||||
fixtures initiate no provider turns. Native adapters have separate regression
|
||||
fixtures and retain/replay numeric snapshots without synthetic transcript items.
|
||||
|
||||
OpenCode V2 reports session counters/cost and model metadata; measured rates use
|
||||
non-reasoning output and native request-to-stream timing (including request
|
||||
latency, excluding tool execution). Codex reports token/context snapshots but
|
||||
provides no compatible generation-rate or USD billing data, so those values
|
||||
remain unavailable. Shell cache v12 refreshes the changed modules on reload.
|
||||
|
||||
### Context compaction activity (2026-10-20)
|
||||
|
||||
**Partial against the live-provider gate.** A native compaction frame shows
|
||||
|
||||
@@ -9,6 +9,9 @@ what is done; this file covers how to run, test and verify it.
|
||||
framework, no bundler, no build step, no npm install.
|
||||
- **Browser-native transport.** The page opens one `WebSocket` to `/connect`,
|
||||
matched by request id, with the same server-URL rules as the Android client.
|
||||
- **Reported chat metrics.** Output tokens/sec, total tokens, context usage/window,
|
||||
and reported USD cost stay visible above the transcript. Unreported metrics are
|
||||
unavailable, not zero or a subscription-price estimate.
|
||||
- **Verified in a real browser.** UI claims are backed by PNGs captured from
|
||||
headless Chrome against a live server (`test/ui-smoke.mjs`); a unit test alone
|
||||
never marks a parity row done.
|
||||
|
||||
@@ -516,6 +516,30 @@
|
||||
.chat-compaction-break { flex-basis: 100%; height: 0; }
|
||||
.chat-compaction-status { color: var(--accent); }
|
||||
|
||||
/* Native usage snapshots sit outside the transcript scroller. Streaming stats
|
||||
patch only this rail, never the focused composer or its selection. */
|
||||
.chat-usage-metrics {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
gap: 8px 16px;
|
||||
margin: 0 0 10px;
|
||||
padding: 6px 2px 10px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
.chat-usage-metric { min-width: 0; }
|
||||
.chat-usage-metric dt { color: var(--muted); font-size: 11px; }
|
||||
.chat-usage-metric dd {
|
||||
margin: 2px 0 0;
|
||||
font-size: 12px;
|
||||
line-height: 18px;
|
||||
font-variant-numeric: tabular-nums;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
.chat-usage-metric[data-available="false"] dd { color: var(--muted); }
|
||||
@media (max-width: 600px) {
|
||||
.chat-usage-metrics { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
}
|
||||
|
||||
.chat-jump { position: absolute; right: 14px; bottom: 14px; border-color: var(--accent); color: var(--accent); }
|
||||
|
||||
/* The tool-call chip and its panel (C6). The panel is a fixed overlay, so it
|
||||
|
||||
+12
-6
@@ -52,6 +52,7 @@
|
||||
// kind is called or coloured.
|
||||
import { ChangeLineKind } from "./changes.js";
|
||||
import { humanBytes } from "./artifacts.js";
|
||||
import { normalizeChatUsage } from "./chatUsage.js";
|
||||
|
||||
/** ChatMessage.Role values (internal/protocol/chat.go). */
|
||||
export const ROLE_USER = "user";
|
||||
@@ -1400,6 +1401,7 @@ export function newChatThread(sessionId = "") {
|
||||
nativeTurnId: "",
|
||||
runningSince: 0,
|
||||
compaction: null,
|
||||
usage: null,
|
||||
toolStartedAt: {},
|
||||
lastDeltaAt: 0,
|
||||
queued: false,
|
||||
@@ -1609,7 +1611,7 @@ export function applyChatForm(thread, form) {
|
||||
|
||||
/**
|
||||
* Applies one pushed `chat_event` frame to a conversation and returns the next
|
||||
* thread. Turn frames clear `queued` (compaction does not admit a queued prompt);
|
||||
* thread. Turn frames clear `queued` (usage/compaction do not admit a queued prompt);
|
||||
* `user` echoes another
|
||||
* client's prompt once (never this client's own), `text`/`reasoning` append to the
|
||||
* part their kind started, `tool` merges field by field, `step` drives the running
|
||||
@@ -1618,6 +1620,12 @@ export function applyChatForm(thread, form) {
|
||||
* targets is created on first sight, so a live turn never gains a second bubble.
|
||||
*/
|
||||
export function applyChatEvent(thread, frame, now = Date.now()) {
|
||||
if (frame?.kind === EVENT_USAGE) {
|
||||
const current = thread ?? newChatThread(str(frame?.session_id));
|
||||
if (current.sessionId && frame.session_id !== current.sessionId) return current;
|
||||
const usage = normalizeChatUsage(frame.usage);
|
||||
return usage ? { ...current, usage } : current;
|
||||
}
|
||||
if (frame?.kind === EVENT_COMPACTION) return applyChatCompaction(thread, frame, now);
|
||||
if (frame?.kind === EVENT_RESYNC) return thread ?? newChatThread(str(frame?.session_id));
|
||||
if (frame?.kind === EVENT_STEP && frame.step?.status === STEP_ENDED && frame.step?.turn_id && frame.step.turn_id !== thread?.nativeTurnId) return thread ?? newChatThread(str(frame?.session_id));
|
||||
@@ -1644,8 +1652,6 @@ export function applyChatEvent(thread, frame, now = Date.now()) {
|
||||
return { ...current, shells: mergeShellFrame(current.shells, frame?.shell) };
|
||||
case EVENT_FORM:
|
||||
return applyChatForm(current, frame?.form);
|
||||
// `usage` is recognised but not folded; it still cleared `queued` above.
|
||||
case EVENT_USAGE:
|
||||
default:
|
||||
return current;
|
||||
}
|
||||
@@ -1978,11 +1984,11 @@ export function composerSubmit(thread, selected = 0) {
|
||||
/**
|
||||
* The queued flag after one pushed frame for the session: any frame means the
|
||||
* turn the prompt was waiting behind has moved on, so the queued notice clears
|
||||
* (mirrors ChatStore.event). Null/compaction frames leave it untouched: context
|
||||
* maintenance alone does not admit the queued prompt.
|
||||
* (mirrors ChatStore.event). Null/usage/compaction frames leave it untouched:
|
||||
* usage reporting or context maintenance alone does not admit the queued prompt.
|
||||
*/
|
||||
export function queuedAfterFrame(queued, frame) {
|
||||
if (frame == null || frame.kind === EVENT_COMPACTION) return queued === true;
|
||||
if (frame == null || frame.kind === EVENT_COMPACTION || frame.kind === EVENT_USAGE) return queued === true;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
// Native snapshots, not tokenizers, pricing estimates, or accumulated deltas.
|
||||
const count = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? (value === 0 ? 0 : value) : null;
|
||||
const amount = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? (value === 0 ? 0 : value) : null;
|
||||
|
||||
export function normalizeChatUsage(usage) {
|
||||
if (!usage || typeof usage !== "object" || Array.isArray(usage)) return null;
|
||||
const window = count(usage.context_window);
|
||||
const result = {
|
||||
scope: ["session", "turn"].includes(usage.scope) ? usage.scope : "",
|
||||
total_tokens: count(usage.total_tokens),
|
||||
context_tokens: count(usage.context_tokens),
|
||||
context_window: window > 0 ? window : null,
|
||||
cost_usd: amount(usage.cost_usd),
|
||||
output_tokens_per_second: amount(usage.output_tokens_per_second),
|
||||
rate_basis: ["generation", "turn"].includes(usage.rate_basis) ? usage.rate_basis : "",
|
||||
};
|
||||
// Legacy input/output counters do not prove a total that includes cache and
|
||||
// reasoning; legacy zero-valued costs cannot distinguish unknown from free.
|
||||
return result.scope || Object.entries(result).some(([key, value]) => key !== "scope" && key !== "rate_basis" && value !== null) ? result : null;
|
||||
}
|
||||
|
||||
export function chatUsageMetrics(value, locale) {
|
||||
const usage = normalizeChatUsage(value);
|
||||
const integer = new Intl.NumberFormat(locale, { maximumFractionDigits: 0 });
|
||||
const decimal = new Intl.NumberFormat(locale, { maximumFractionDigits: 1 });
|
||||
const dollars = new Intl.NumberFormat(locale, { style: "currency", currency: "USD", minimumFractionDigits: 4, maximumFractionDigits: 6 });
|
||||
const metric = (key, label, number, format, title) => ({ key, label, available: number !== null && number !== undefined,
|
||||
value: number === null || number === undefined ? "Unavailable" : format(number), title });
|
||||
const rateTitle = usage?.rate_basis === "turn"
|
||||
? "Average output tokens per second over the measured turn, including tools and waiting."
|
||||
: usage?.rate_basis === "generation"
|
||||
? "Non-reasoning output tokens per second over the latest native request-to-stream-completion interval, including request latency but excluding tool execution."
|
||||
: "Native output-token rate when reported; never estimated from text length.";
|
||||
const used = usage?.context_tokens ?? null;
|
||||
const limit = usage?.context_window ?? null;
|
||||
const context = used !== null && limit !== null ? `${integer.format(used)} / ${integer.format(limit)}`
|
||||
: limit !== null ? `${integer.format(limit)} limit`
|
||||
: used !== null ? `${integer.format(used)} used` : "Unavailable";
|
||||
return [
|
||||
metric("rate", usage?.rate_basis === "turn" ? "Avg output rate" : "Output rate", usage?.output_tokens_per_second,
|
||||
(number) => `${number > 0 && number < 0.1 ? "<" + decimal.format(0.1) : decimal.format(number)} tok/s`, rateTitle),
|
||||
metric("tokens", usage?.scope === "turn" ? "Turn tokens" : "Total tokens", usage?.total_tokens,
|
||||
(number) => integer.format(number), usage?.scope === "turn" ? "Reported token total for the measured turn. Snapshots are not added together."
|
||||
: "Reported token total across the conversation's model calls, not the current context size. Snapshots are not added together."),
|
||||
{ key: "context", label: "Context", available: used !== null || limit !== null, value: context,
|
||||
title: "Latest reported prompt/context usage / model context-window limit. Harness accounting may differ; cumulative processed tokens are not context usage." },
|
||||
metric("cost", "Reported cost", usage?.cost_usd,
|
||||
(number) => number > 0 && number < 0.000001 ? "<" + dollars.format(0.000001) : dollars.format(number),
|
||||
"USD cost reported by the harness, not a subscription invoice. We do not estimate unreported costs or infer subscription billing."),
|
||||
];
|
||||
}
|
||||
+34
-1
@@ -8,6 +8,7 @@ import { isLocalArtifactLink } from "../artifactNavigation.js";
|
||||
import { formFieldVisible, formFieldValue, formAnswerError, visibleFormAnswers, safeFormURL } from "../chatForms.js";
|
||||
import { readChatStars, writeChatStars } from "../chatStars.js";
|
||||
import { chatModelLabel, chatModelSelectionNote, supportsChatModels } from "../chatModels.js";
|
||||
import { chatUsageMetrics } from "../chatUsage.js";
|
||||
import { describeError } from "../format.js";
|
||||
import { SHELL_OUTPUT_LIMIT, nativeChatMessageId, newChatOperationId } from "../repository.js";
|
||||
import { codeBlockLabel, parseInline, parseMarkdown } from "../markdown.js";
|
||||
@@ -790,6 +791,13 @@ function onChatEvent(frame) {
|
||||
scheduleHistorySync(sessionId, "", {force:true});
|
||||
return;
|
||||
}
|
||||
if (frame.kind === "usage") {
|
||||
const current = state.chatThreads.get(sessionId);
|
||||
const next = applyChatEvent(current, frame);
|
||||
state.chatThreads.set(sessionId, next);
|
||||
if (state.view === "chat" && sessionId === state.chatActiveId) refreshChatUsageMetrics(sessionId, next);
|
||||
return;
|
||||
}
|
||||
markBackgroundChatUnseen(sessionId);
|
||||
const thread = state.chatThreads.get(sessionId);
|
||||
const messageId = String(frame?.message_id ?? "");
|
||||
@@ -1384,7 +1392,7 @@ function onChatFrame(message) {
|
||||
if (!frame || !frame.session_id) return;
|
||||
// onChatEvent already folded this session activity with native identity
|
||||
// checks and refreshed the composer. Do not reapply an ignored stale start.
|
||||
if (frame.kind === "compaction") return;
|
||||
if (frame.kind === "compaction" || frame.kind === "usage") return;
|
||||
rt.touchView?.("chat");
|
||||
const sessionId = frame.session_id;
|
||||
const thread = state.chatThreads.get(sessionId);
|
||||
@@ -2462,6 +2470,30 @@ function renderChatActionsMenu(session, duplicateTitles = new Set()) {
|
||||
return wrap;
|
||||
}
|
||||
|
||||
/** Reported usage, with explicit unknowns and no inferred token/cost values. */
|
||||
function renderChatUsageMetrics(thread, sessionId) {
|
||||
const metrics = h("dl", {
|
||||
class: "chat-usage-metrics",
|
||||
id: "chatUsageMetrics",
|
||||
"data-session": sessionId,
|
||||
"aria-label": "Chat usage and performance",
|
||||
});
|
||||
for (const metric of chatUsageMetrics(thread?.usage)) {
|
||||
metrics.append(h("div", {
|
||||
class: "chat-usage-metric",
|
||||
"data-metric": metric.key,
|
||||
"data-available": String(metric.available),
|
||||
title: metric.title,
|
||||
}, h("dt", { text: metric.label }), h("dd", { text: metric.value })));
|
||||
}
|
||||
return metrics;
|
||||
}
|
||||
|
||||
function refreshChatUsageMetrics(sessionId, thread) {
|
||||
const metrics = document.getElementById("chatUsageMetrics");
|
||||
if (metrics?.dataset.session === sessionId) metrics.replaceWith(renderChatUsageMetrics(thread, sessionId));
|
||||
}
|
||||
|
||||
/** The active conversation: identity, context, transcript and its controls. */
|
||||
function renderChatConversation() {
|
||||
const pane = h("div", { class: "detail chat-detail" });
|
||||
@@ -2530,6 +2562,7 @@ function renderChatConversation() {
|
||||
context.append(modelButton);
|
||||
pane.append(context);
|
||||
}
|
||||
pane.append(renderChatUsageMetrics(thread, state.chatActiveId));
|
||||
|
||||
// The background-tasks bar (C11) sits above the transcript, outside the
|
||||
// scroller so the C14 scroll positions are untouched. It is present only while
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// A shell cache for an installed Pipelines app. The API (the WebSocket proxy and
|
||||
// artifact files) is never cached; static files prefer the network and fall back
|
||||
// to the last good copy so an installed app still opens when the server is away.
|
||||
const CACHE = "pipelines-shell-v11";
|
||||
const CACHE = "pipelines-shell-v12";
|
||||
const SHELL = ["/", "/index.html", "/manifest.webmanifest", "/favicon.svg"];
|
||||
|
||||
self.addEventListener("install", (event) => {
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
// Actual chat view/event handler with native-shaped usage fixtures. No backend
|
||||
// connection, provider calls, production chats, or price estimates.
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer } from "node:http";
|
||||
import { readFile, mkdir } from "node:fs/promises";
|
||||
import { resolve, extname, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { launchChrome } from "./browser.mjs";
|
||||
|
||||
const root=resolve(dirname(fileURLToPath(import.meta.url)),"..");
|
||||
const out=resolve(process.argv[2]??"captures-chat-usage");
|
||||
await mkdir(out,{recursive:true});
|
||||
const html=`<!doctype html><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
${["design/tokens/tokens","base","shell","lists","run-detail","chat","dialogs"].map(n=>`<link rel="stylesheet" href="/${n}.css">`).join("")}
|
||||
<div class="app-root"><main class="workspace"><div class="pane-main"><div class="content" id="content"></div></div></main></div>
|
||||
<script type="module">
|
||||
import { createChatView } from '/src/views/chat.js';
|
||||
import { newChatThread } from '/src/chat.js';
|
||||
const id='scratch';
|
||||
const state={view:'chat',chatActiveId:id,chatSessions:[{id,title:'Chat usage',backend:'opencode'}],chatOverrides:{},chatDrafts:{},chatThreads:new Map([[id,{...newChatThread(id),loaded:true,messages:[{id:'user',role:'user',parts:[{type:'text',text:'Inspect the results.'}]},{id:'assistant',role:'assistant',parts:[{type:'text',text:'Usage is reported by the harness, not inferred from this answer.'}]}]}]])};
|
||||
const rt={state,repo:{},renderContent:()=>paint(),closeDetail:()=>{}};
|
||||
const view=createChatView(rt);
|
||||
window.paint=()=>document.querySelector('#content').replaceChildren(view.renderChatConversation());
|
||||
window.usage=(snapshot)=>view.onChatEvent({session_id:id,kind:'usage',usage:snapshot});
|
||||
window.messageCount=()=>state.chatThreads.get(id).messages.length;
|
||||
paint();
|
||||
</script>`;
|
||||
const server=createServer(async(req,res)=>{try{
|
||||
const path=new URL(req.url,"http://fixture").pathname;
|
||||
if(path==="/"){res.setHeader("Content-Type","text/html");res.end(html);return;}
|
||||
const file=resolve(root,`.${path}`);if(!file.startsWith(root+"/"))throw Error("outside fixture");
|
||||
res.setHeader("Content-Type",extname(file)===".css"?"text/css":"text/javascript");res.end(await readFile(file));
|
||||
}catch{res.writeHead(404);res.end();}});
|
||||
await new Promise(r=>server.listen(0,"127.0.0.1",r));
|
||||
let browser;
|
||||
const known={scope:"session",total_tokens:132408,context_tokens:42000,context_window:200000,cost_usd:.1423,output_tokens_per_second:24.6,rate_basis:"generation"};
|
||||
try {
|
||||
browser=await launchChrome();
|
||||
const page=await browser.openPage(`http://127.0.0.1:${server.address().port}/`);
|
||||
await page.waitForSelector('#chatUsageMetrics');
|
||||
const values=()=>page.evaluate("Array.from(document.querySelectorAll('#chatUsageMetrics dd'),node=>node.textContent)");
|
||||
for(const width of [390,420,1100]) {
|
||||
await page.resize(width,844);
|
||||
await page.evaluate("usage({scope:'session'})");
|
||||
assert.deepEqual(await values(),Array(4).fill('Unavailable'));
|
||||
await page.screenshot(resolve(out,`usage-unavailable-${width}.png`));
|
||||
await page.fill('#chatInput','Keep this draft and cursor.');
|
||||
await page.evaluate("window.savedInput=document.querySelector('#chatInput');savedInput.focus();savedInput.setSelectionRange(5,10);window.savedTranscript=document.querySelector('#chatTranscript')");
|
||||
await page.evaluate(`usage(${JSON.stringify(known)})`);
|
||||
assert.deepEqual(await values(),['24.6 tok/s','132,408','42,000 / 200,000','$0.1423']);
|
||||
const intact=await page.evaluate("savedInput===document.querySelector('#chatInput')&&savedInput===document.activeElement&&savedInput.value==='Keep this draft and cursor.'&&savedInput.selectionStart===5&&savedInput.selectionEnd===10&&savedTranscript===document.querySelector('#chatTranscript')");
|
||||
assert.equal(intact,true,'usage updates must not repaint the composer/transcript');
|
||||
assert.equal(await page.evaluate('messageCount()'),2);
|
||||
assert.equal(await page.evaluate("document.querySelectorAll('#chatStop').length"),0,'usage must not start a turn');
|
||||
assert.equal(await page.evaluate('document.documentElement.scrollWidth>innerWidth'),false);
|
||||
await page.screenshot(resolve(out,`usage-reported-${width}.png`));
|
||||
await page.evaluate(`usage(${JSON.stringify(known)})`);
|
||||
assert.deepEqual(await values(),['24.6 tok/s','132,408','42,000 / 200,000','$0.1423']);
|
||||
await page.evaluate("usage({scope:'session',total_tokens:140000,context_tokens:3000,context_window:1048576})");
|
||||
assert.deepEqual(await values(),['Unavailable','140,000','3,000 / 1,048,576','Unavailable']);
|
||||
await page.screenshot(resolve(out,`usage-partial-native-${width}.png`));
|
||||
await page.evaluate("usage({scope:'session',total_tokens:0,context_tokens:0,context_window:200000,cost_usd:0,output_tokens_per_second:0})");
|
||||
assert.deepEqual(await values(),['0 tok/s','0','0 / 200,000','$0.0000']);
|
||||
await page.screenshot(resolve(out,`usage-reported-zero-${width}.png`));
|
||||
}
|
||||
await page.resize(390,500);
|
||||
await page.evaluate(`usage(${JSON.stringify(known)})`);
|
||||
assert.equal(await page.evaluate("document.querySelector('#chatComposer').getBoundingClientRect().bottom<=innerHeight"),true);
|
||||
assert.equal(await page.evaluate('document.documentElement.scrollWidth>innerWidth'),false);
|
||||
await page.screenshot(resolve(out,'usage-short-viewport-390.png'));
|
||||
console.log('PASS: live metric rail, exact reported/unknown/zero states, no double counting or fake messages, mobile fit, and preserved composer focus/selection and transcript DOM.');
|
||||
} finally {await browser?.close();await new Promise(r=>server.close(r));}
|
||||
@@ -1286,11 +1286,10 @@ test("step started runs the turn and step ended settles it", () => {
|
||||
assert.equal(thread.lastDeltaAt, 0);
|
||||
});
|
||||
|
||||
test("any frame clears the queued flag", () => {
|
||||
test("turn-content frames clear the queued flag", () => {
|
||||
for (const frame of [
|
||||
evt("text", { message_id: "a1", part: { id: "p" }, delta: "x" }),
|
||||
evt("step", { step: { status: "started" } }),
|
||||
evt("usage", { usage: { input_tokens: 1 } }),
|
||||
evt("shell", { shell: { id: "sh1", status: "running" } }),
|
||||
]) {
|
||||
const queued = { ...newChatThread("s1"), queued: true };
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { chatUsageMetrics, normalizeChatUsage } from "../src/chatUsage.js";
|
||||
import { newChatThread, applyChatEvent, settledThread, resumedThread, historyResync, queuedAfterFrame } from "../src/chat.js";
|
||||
|
||||
const snapshot = {scope:"session",total_tokens:132408,context_tokens:42000,context_window:200000,cost_usd:0.1423,output_tokens_per_second:24.6,rate_basis:"generation"};
|
||||
const event = (usage) => ({session_id:"chat",kind:"usage",usage});
|
||||
|
||||
test("usage metrics format native counters without conflating context and processed totals", () => {
|
||||
const rows=chatUsageMetrics(snapshot,"en-US");
|
||||
assert.deepEqual(rows.map(row=>row.value),["24.6 tok/s","132,408","42,000 / 200,000","$0.1423"]);
|
||||
assert.ok(rows.every(row=>row.available));
|
||||
assert.match(rows[0].title,/native request-to-stream-completion/);
|
||||
});
|
||||
|
||||
test("unreported values stay unavailable, including legacy zero or unsupported subscription costs", () => {
|
||||
assert.ok(chatUsageMetrics(null).every(row=>row.value==="Unavailable"));
|
||||
assert.ok(chatUsageMetrics({input_tokens:100,output_tokens:50,cost:0}).every(row=>!row.available));
|
||||
const codex=chatUsageMetrics({scope:"session",total_tokens:1000,context_window:128000},"en-US");
|
||||
assert.equal(codex[3].value,"Unavailable");
|
||||
assert.equal(codex[2].value,"128,000 limit");
|
||||
assert.equal(codex[0].value,"Unavailable");
|
||||
});
|
||||
|
||||
test("reported zeros are not missing and tiny positive values are not displayed as zero", () => {
|
||||
const zero=chatUsageMetrics({scope:"session",total_tokens:0,context_tokens:0,context_window:1000,cost_usd:0,output_tokens_per_second:0},"en-US");
|
||||
assert.deepEqual(zero.map(row=>row.value),["0 tok/s","0","0 / 1,000","$0.0000"]);
|
||||
const small=chatUsageMetrics({cost_usd:0.00000001,output_tokens_per_second:0.01},"en-US");
|
||||
assert.equal(small[0].value,"<0.1 tok/s");
|
||||
assert.equal(small[3].value,"<$0.000001");
|
||||
});
|
||||
|
||||
test("invalid, fractional or unsafe counters are never turned into fabricated metrics", () => {
|
||||
const invalid=normalizeChatUsage({scope:"session",total_tokens:1.5,context_tokens:-1,context_window:0,cost_usd:Infinity,output_tokens_per_second:"100"});
|
||||
assert.ok(chatUsageMetrics(invalid).every(row=>!row.available));
|
||||
assert.equal(normalizeChatUsage({total_tokens:Number.MAX_SAFE_INTEGER+1}),null);
|
||||
assert.equal(normalizeChatUsage([]),null);
|
||||
});
|
||||
|
||||
test("usage snapshots replace counters, preserve queue/turn state and do not create messages", () => {
|
||||
const original={...newChatThread("chat"),queued:true};
|
||||
const once=applyChatEvent(original,event(snapshot));
|
||||
const twice=applyChatEvent(once,event(snapshot));
|
||||
assert.equal(twice.usage.total_tokens,132408);
|
||||
assert.equal(twice.queued,true);
|
||||
assert.equal(twice.running,false);
|
||||
assert.deepEqual(twice.messages,[]);
|
||||
assert.equal(queuedAfterFrame(true,event(snapshot)),true);
|
||||
const next=applyChatEvent(twice,event({...snapshot,total_tokens:133000,context_tokens:3000}));
|
||||
assert.equal(next.usage.total_tokens,133000);
|
||||
assert.equal(next.usage.context_tokens,3000);
|
||||
assert.equal(applyChatEvent(next,{...event(snapshot),session_id:"other"}),next);
|
||||
});
|
||||
|
||||
test("last reported metrics survive idle, history refresh and reconnect without being summed", () => {
|
||||
const thread=applyChatEvent(newChatThread("chat"),event(snapshot));
|
||||
for(const result of [settledThread(thread),resumedThread(thread),historyResync(thread,{messages:[]})]) {
|
||||
assert.equal(result.usage.total_tokens,132408);
|
||||
assert.equal(result.usage.cost_usd,0.1423);
|
||||
}
|
||||
const unknown=applyChatEvent(thread,event({scope:"session"}));
|
||||
assert.ok(chatUsageMetrics(unknown.usage).every(row=>!row.available));
|
||||
});
|
||||
|
||||
test("turn-wall-time averages are labeled rather than presented as pure model generation speed", () => {
|
||||
const rate=chatUsageMetrics({...snapshot,rate_basis:"turn"})[0];
|
||||
assert.equal(rate.label,"Avg output rate");
|
||||
assert.match(rate.title,/including tools and waiting/);
|
||||
});
|
||||
@@ -9,9 +9,9 @@ test("shell updates retain unrelated caches and the current app cache", async ()
|
||||
let activation;
|
||||
vm.runInNewContext(await readFile(new URL("../sw.js", import.meta.url), "utf8"), {
|
||||
self: { addEventListener: (type, handler) => handlers.set(type, handler), clients: { claim: async () => {} } },
|
||||
caches: { keys: async () => ["pipelines-shell-v10", "pipelines-shell-v11", "unrelated-user-cache"], delete: async (key) => deleted.push(key) },
|
||||
caches: { keys: async () => ["pipelines-shell-v11", "pipelines-shell-v12", "unrelated-user-cache"], delete: async (key) => deleted.push(key) },
|
||||
});
|
||||
handlers.get("activate")({ waitUntil: (promise) => { activation = promise; } });
|
||||
await activation;
|
||||
assert.deepEqual(deleted, ["pipelines-shell-v10"]);
|
||||
assert.deepEqual(deleted, ["pipelines-shell-v11"]);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user