web: live run updates over a single app-wide watch (R10)

This commit is contained in:
2026-10-19 14:22:53 +01:00
parent 2fe63012f9
commit 21a6ee3c4d
13 changed files with 1580 additions and 90 deletions
+45
View File
@@ -129,6 +129,51 @@ test("a server-pushed chat_event frame reaches a frame listener", async () => {
assert.equal(frames[0].chat_event.session_id, "ses_1");
});
test("sendOnly sends a frame, returns its id and never waits for a reply", () => {
const { client, ws } = connectedClient();
const frames = [];
client.addEventListener("frame", (event) => frames.push(event.detail));
// No reply is scripted: a send-only frame must not hang the caller.
const id = client.sendOnly({ type: "subscribe", workspace: "" });
assert.match(id, /^web-\d+$/);
assert.deepEqual(ws.sent, [{ type: "subscribe", workspace: "", id }]);
// A server that does ack (or refuse) still reaches the app as an unsolicited
// frame carrying that id, not as a resolved request.
ws.push({ id, type: "subscribe" });
assert.equal(frames.length, 1);
assert.equal(frames[0].type, "subscribe");
});
test("sendOnly before connect is rejected as not connected", () => {
const client = freshClient();
assert.throws(() => client.sendOnly({ type: "subscribe" }), NotConnectedError);
});
test("back-to-back sendOnly frames keep their order on the wire", () => {
const { client, ws } = connectedClient();
client.sendOnly({ type: "unsubscribe" });
client.sendOnly({ type: "subscribe", workspace: "" });
assert.deepEqual(
ws.sent.map((frame) => frame.type),
["unsubscribe", "subscribe"],
);
});
test("a server-pushed run_event frame reaches a frame listener", async () => {
const { client, ws } = connectedClient();
const frames = [];
client.addEventListener("frame", (event) => frames.push(event.detail));
ws.push({ type: "run_event", run_event: { run_id: 7, status: "running" } });
await flush();
assert.equal(frames.length, 1);
assert.equal(frames[0].type, "run_event");
assert.equal(frames[0].run_event.run_id, 7);
});
test("a malformed frame is ignored and does not reject a pending call", async () => {
const { client, ws } = connectedClient();
const pending = client.call({ type: "ping" });
+9 -13
View File
@@ -18,6 +18,7 @@ import {
ForegroundAction,
IDLE_BANNER_TEXT,
RETRYING_SUFFIX,
RunStream,
bannerState,
foregroundAction,
liveBadgeState,
@@ -182,19 +183,14 @@ test("serverBuildLabel tolerates missing fields", () => {
// liveBadgeState — RunStream.kt words
// ---------------------------------------------------------------------------
test("liveBadgeState is live only while executing on a connected socket", () => {
assert.deepEqual(liveBadgeState({ executing: true }, { kind: "connected" }), { tone: "live", label: "live" });
assert.deepEqual(liveBadgeState({ executing: true }, { kind: "disconnected" }), {
tone: "reconnecting",
label: "reconnecting",
});
assert.deepEqual(liveBadgeState({ executing: true }, { kind: "connecting" }), {
tone: "reconnecting",
label: "reconnecting",
});
assert.deepEqual(liveBadgeState({ executing: false }, { kind: "connected" }), { tone: "not-live", label: "not live" });
assert.deepEqual(liveBadgeState(null, { kind: "connected" }), { tone: "not-live", label: "not live" });
assert.deepEqual(liveBadgeState({ executing: true }, { kind: "idle" }), { tone: "not-live", label: "not live" });
test("liveBadgeState maps the explicit stream state to RunStream.kt's words", () => {
// The badge says whether pushes are arriving, independent of any executing run.
assert.deepEqual(liveBadgeState(RunStream.Live), { tone: "live", label: "live" });
assert.deepEqual(liveBadgeState(RunStream.Connecting), { tone: "reconnecting", label: "reconnecting" });
assert.deepEqual(liveBadgeState(RunStream.Offline), { tone: "not-live", label: "not live" });
// An unknown/missing state is honestly not live.
assert.deepEqual(liveBadgeState(undefined), { tone: "not-live", label: "not live" });
assert.deepEqual(liveBadgeState("something-else"), { tone: "not-live", label: "not live" });
});
// ---------------------------------------------------------------------------
+38 -1
View File
@@ -4,7 +4,7 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { PipelinesClient, ProtocolError } from "../src/client.js";
import { PipelinesClient, ProtocolError, NotConnectedError } from "../src/client.js";
import { PipelinesRepository, SHELL_OUTPUT_LIMIT, nativeChatMessageId, newChatOperationId } from "../src/repository.js";
import { validateServerUrl } from "../src/serverUrl.js";
@@ -270,6 +270,43 @@ test("setRunTitle sends exactly {type:'set_run_title',run_id,title} and returns
assert.deepEqual(detail, RUN_DETAIL);
});
test("subscribeRuns sends exactly {type:'subscribe',workspace:''} with no run_ids", async () => {
const { repo, ws } = connected();
const id = repo.subscribeRuns();
assert.match(id, /^web-\d+$/);
assert.deepEqual(lastFrame(ws), { type: "subscribe", workspace: "" });
});
test("subscribeRuns includes run_ids only when ids are given and sends the workspace verbatim", () => {
const { repo, ws } = connected();
repo.subscribeRuns("", [3, 4]);
assert.deepEqual(lastFrame(ws), { type: "subscribe", workspace: "", run_ids: [3, 4] });
repo.subscribeRuns("/tmp/ws", []);
assert.deepEqual(lastFrame(ws), { type: "subscribe", workspace: "/tmp/ws" });
});
test("subscribeRuns is send-only: it returns the id and never awaits a reply", () => {
const { repo, ws } = connected();
// No reply is scripted; a send-only call must not hang or reject.
const id = repo.subscribeRuns();
assert.equal(ws.sent.at(-1).id, id);
});
test("unsubscribeRuns sends exactly {type:'unsubscribe'} and is send-only", () => {
const { repo, ws } = connected();
const id = repo.unsubscribeRuns();
assert.match(id, /^web-\d+$/);
assert.deepEqual(lastFrame(ws), { type: "unsubscribe" });
});
test("the run watch commands are refused before connect", () => {
FakeWebSocket.instances = [];
const client = new PipelinesClient({ WebSocketImpl: FakeWebSocket });
const repo = new PipelinesRepository(client);
assert.throws(() => repo.subscribeRuns(), NotConnectedError);
assert.throws(() => repo.unsubscribeRuns(), NotConnectedError);
});
test("each run action rejects a server error envelope instead of returning a detail", async () => {
const calls = [
(repo) => () => repo.decideRun(440, { option: "approve" }),
+395
View File
@@ -0,0 +1,395 @@
// The live-updates fold, exercised as a pure delta sequence so the client's
// correctness is pinned without a device or a socket: a created run, a step
// started, a step finished, a run finished, a delta for a run the list does not
// hold, an out-of-order/duplicate delta, and a reconnect resync.
//
// A direct port of android/.../ui/RunEventsTest.kt, so the browser fold and the
// app fold are proven against the same shape.
import { test } from "node:test";
import assert from "node:assert/strict";
import {
TERMINAL_RUN_STATUSES,
executingFor,
foldRunEvent,
foldRunEventDetail,
isTerminalRunStatus,
isWaitingRunStatus,
newRunSummaryFromFrame,
runDetailNeedsFullSync,
runLifecycleRank,
stepLifecycleRank,
} from "../src/runEvents.js";
function summary(id, { status = "running", executing = true, currentStage = "", completedSteps = 0, finishedAt = "" } = {}) {
return {
id,
pipeline_id: 4,
name: "pipeline",
status,
executing,
current_stage: currentStage,
completed_steps: completedSteps,
created_at: "2026-10-17T09:00:00Z",
finished_at: finishedAt,
};
}
function detail({ id = 1, status = "queued", executing = true, currentStage = "", completedSteps = 0, finishedAt = "", steps = [] } = {}) {
return {
id,
pipeline_id: 4,
name: "pipeline",
status,
executing,
current_stage: currentStage,
completed_steps: completedSteps,
created_at: "2026-10-17T09:00:00Z",
finished_at: finishedAt,
steps,
};
}
function step({ sequence = 1, stage = "build", status = "running", outcome = "", target = "" } = {}) {
return { sequence, stage, status, outcome, target };
}
function frame({
runId = 1,
status = "",
currentStage = "",
outcome = "",
completedSteps = 0,
finishedAt = "",
step: stepDelta = null,
pipelineId = 0,
name = "",
title = "",
version = 0,
createdAt = "",
} = {}) {
return {
run_id: runId,
workspace: "/work",
pipeline_id: pipelineId,
name,
title,
version,
status,
outcome,
current_stage: currentStage,
completed_steps: completedSteps,
created_at: createdAt,
finished_at: finishedAt,
step: stepDelta,
};
}
// --- lifecycle vocabulary ----------------------------------------------------
test("the lifecycle ladders mirror RunEvents.kt", () => {
assert.deepEqual(TERMINAL_RUN_STATUSES, ["succeeded", "failed", "canceled", "interrupted"]);
assert.equal(runLifecycleRank("queued"), 0);
assert.equal(runLifecycleRank("running"), 1);
assert.equal(runLifecycleRank("waiting"), 1);
assert.equal(runLifecycleRank("succeeded"), 2);
assert.equal(runLifecycleRank("nonsense"), -1);
assert.equal(stepLifecycleRank("running"), 1);
assert.equal(stepLifecycleRank("completed"), 2);
assert.equal(stepLifecycleRank(""), -1);
assert.equal(isWaitingRunStatus(" waiting "), true);
assert.equal(isTerminalRunStatus("SUCCEEDED"), true);
assert.equal(isTerminalRunStatus("waiting"), false);
assert.equal(executingFor("queued", false), true);
assert.equal(executingFor("waiting", true), false);
assert.equal(executingFor("", true), true);
assert.equal(executingFor("", false), false);
});
// --- runs list ---------------------------------------------------------------
test("a delta updates a matching row in place without reordering", () => {
const runs = [summary(3, { status: "succeeded", executing: false }), summary(1)];
const fold = foldRunEvent(runs, frame({ runId: 1, status: "running", currentStage: "build", completedSteps: 1 }));
assert.equal(fold.needsResync, false);
assert.deepEqual(fold.runs.map((run) => run.id), [3, 1]);
const updated = fold.runs.find((run) => run.id === 1);
assert.equal(updated.status, "running");
assert.equal(updated.current_stage, "build");
assert.equal(updated.completed_steps, 1);
assert.equal(updated.executing, true);
assert.equal(fold.runs[0], runs[0], "the other row is shared, not copied");
});
test("a new run appears from its frame without a fetch", () => {
const runs = [summary(1, { status: "succeeded", executing: false })];
const fold = foldRunEvent(
runs,
frame({
runId: 9,
status: "queued",
pipelineId: 4,
name: "pipeline",
title: "a fresh run",
version: 3,
createdAt: "2026-10-17T09:30:00Z",
}),
);
assert.equal(fold.needsResync, false);
assert.deepEqual(fold.runs.map((run) => run.id), [9, 1]);
const added = fold.runs[0];
assert.equal(added.pipeline_id, 4);
assert.equal(added.name, "pipeline");
assert.equal(added.title, "a fresh run");
assert.equal(added.version, 3);
assert.equal(added.status, "queued");
assert.equal(added.executing, true);
assert.equal(added.created_at, "2026-10-17T09:30:00Z");
});
test("a frame for an unknown run without identity still asks for a resync", () => {
const runs = [summary(1)];
const fold = foldRunEvent(runs, frame({ runId: 9, status: "queued" }));
assert.equal(fold.needsResync, true);
assert.equal(fold.runs, runs);
});
test("an inserted run then advances in place", () => {
const created = foldRunEvent(
[],
frame({ runId: 9, status: "queued", pipelineId: 4, name: "pipeline", version: 1, createdAt: "2026-10-17T09:30:00Z" }),
).runs;
const running = foldRunEvent(created, frame({ runId: 9, status: "running", currentStage: "build", completedSteps: 1 }));
assert.equal(running.needsResync, false);
assert.equal(running.runs.length, 1);
assert.equal(running.runs[0].status, "running");
assert.equal(running.runs[0].current_stage, "build");
const done = foldRunEvent(
running.runs,
frame({ runId: 9, status: "succeeded", outcome: "success", completedSteps: 1, finishedAt: "2026-10-17T09:31:00Z" }),
);
assert.equal(done.runs.length, 1);
assert.equal(done.runs[0].status, "succeeded");
assert.equal(done.runs[0].executing, false);
});
test("a delta for an unknown run also asks for a resync", () => {
const runs = [summary(1, { status: "succeeded", executing: false })];
const fold = foldRunEvent(runs, frame({ runId: 42, status: "running", currentStage: "x" }));
assert.equal(fold.needsResync, true);
assert.deepEqual(fold.runs.map((run) => run.id), [1]);
});
test("a duplicate or out-of-order list delta never regresses a run", () => {
const settled = [summary(1, { status: "succeeded", executing: false, finishedAt: "2026-10-17T09:05:00Z" })];
const fold = foldRunEvent(settled, frame({ runId: 1, status: "running", currentStage: "build" }));
assert.equal(fold.needsResync, false);
assert.equal(fold.runs[0].status, "succeeded");
assert.equal(fold.runs[0].executing, false);
assert.equal(fold.runs, settled, "an unchanged fold keeps the original list");
});
test("a terminal run delta settles the row and stops executing", () => {
const fold = foldRunEvent(
[summary(1, { status: "running" })],
frame({ runId: 1, status: "failed", outcome: "error", finishedAt: "2026-10-17T09:06:00Z" }),
);
const run = fold.runs[0];
assert.equal(run.status, "failed");
assert.equal(run.outcome, "error");
assert.equal(run.executing, false);
assert.equal(run.finished_at, "2026-10-17T09:06:00Z");
});
test("a stale running frame after terminal leaves the list unchanged whole", () => {
const terminal = [summary(1, { status: "succeeded", executing: false, finishedAt: "2026-10-17T09:06:00Z" })];
const fold = foldRunEvent(terminal, frame({ runId: 1, status: "running", currentStage: "build", completedSteps: 3 }));
assert.equal(fold.runs, terminal, "the whole frame is dropped, not just its status");
assert.equal(fold.runs[0].current_stage, "");
assert.equal(fold.runs[0].completed_steps, 0);
});
test("a second, different terminal word is dropped", () => {
const failed = [summary(1, { status: "failed", executing: false })];
const fold = foldRunEvent(failed, frame({ runId: 1, status: "canceled" }));
assert.equal(fold.runs, failed);
assert.equal(fold.runs[0].status, "failed");
});
// --- run detail --------------------------------------------------------------
test("a step started delta adds the visit and runs the run", () => {
const folded = foldRunEventDetail(
detail(),
frame({ runId: 1, status: "running", currentStage: "build", step: { sequence: 1, stage: "build", status: "running" } }),
);
assert.equal(folded.status, "running");
assert.equal(folded.executing, true);
assert.equal(folded.current_stage, "build");
assert.equal(folded.steps.length, 1);
assert.equal(folded.steps[0].sequence, 1);
assert.equal(folded.steps[0].status, "running");
});
test("a step finished delta settles the visit and counts it", () => {
const started = detail({ status: "running", currentStage: "build", steps: [step()] });
const folded = foldRunEventDetail(
started,
frame({
runId: 1,
status: "running",
currentStage: "build",
completedSteps: 1,
step: { sequence: 1, stage: "build", status: "completed", outcome: "success", target: "#success" },
}),
);
assert.equal(folded.steps.length, 1);
assert.equal(folded.steps[0].status, "completed");
assert.equal(folded.steps[0].outcome, "success");
assert.equal(folded.steps[0].target, "#success");
assert.equal(folded.completed_steps, 1);
assert.equal(folded.executing, true, "the run is still going until the terminal frame");
});
test("a run finished delta settles the run with its terminal fields", () => {
const folded = foldRunEventDetail(
detail({ status: "running", executing: true, currentStage: "build" }),
frame({ runId: 1, status: "succeeded", outcome: "success", finishedAt: "2026-10-17T09:07:00Z", completedSteps: 2 }),
);
assert.equal(folded.status, "succeeded");
assert.equal(folded.outcome, "success");
assert.equal(folded.executing, false);
assert.equal(folded.finished_at, "2026-10-17T09:07:00Z");
assert.equal(folded.completed_steps, 2);
});
test("a duplicate step delta is idempotent", () => {
const started = foldRunEventDetail(
detail(),
frame({ runId: 1, status: "running", step: { sequence: 1, stage: "build", status: "running" } }),
);
const again = foldRunEventDetail(
started,
frame({ runId: 1, status: "running", step: { sequence: 1, stage: "build", status: "running" } }),
);
assert.equal(again.steps.length, 1);
assert.equal(again, started, "a duplicate changes nothing");
});
test("an out-of-order step delta never reopens a settled visit", () => {
const settled = detail({
status: "running",
steps: [step({ status: "completed", outcome: "success", target: "#success" })],
});
const folded = foldRunEventDetail(
settled,
frame({ runId: 1, status: "running", step: { sequence: 1, stage: "build", status: "running" } }),
);
assert.equal(folded.steps[0].status, "completed");
assert.equal(folded.steps[0].outcome, "success");
assert.equal(folded, settled);
});
test("a settled step keeps its word against another terminal delta", () => {
const settled = detail({
status: "running",
steps: [step({ status: "completed", outcome: "success", target: "#success" })],
});
const folded = foldRunEventDetail(
settled,
frame({ runId: 1, status: "running", step: { sequence: 1, stage: "build", status: "failed", outcome: "error" } }),
);
assert.equal(folded.steps[0].status, "completed");
assert.equal(folded.steps[0].outcome, "success");
assert.equal(folded, settled);
});
test("a later visit is inserted in sequence order", () => {
const started = detail({
status: "running",
steps: [step({ sequence: 1, status: "completed", outcome: "success", target: "#success" })],
});
const folded = foldRunEventDetail(
started,
frame({ runId: 1, status: "running", step: { sequence: 2, stage: "review", status: "running" } }),
);
assert.deepEqual(folded.steps.map((s) => s.sequence), [1, 2]);
const lower = foldRunEventDetail(
folded,
frame({ runId: 1, status: "running", step: { sequence: 0, stage: "plan", status: "running" } }),
);
assert.deepEqual(lower.steps.map((s) => s.sequence), [0, 1, 2]);
});
test("a frame for another run leaves the detail untouched", () => {
const mine = detail({ id: 1, status: "running" });
const folded = foldRunEventDetail(mine, frame({ runId: 2, status: "succeeded" }));
assert.equal(folded, mine);
});
test("resource_wait_set sets and clears the wait explicitly", () => {
const waiting = foldRunEventDetail(
detail({ status: "running" }),
{ ...frame({ runId: 1, status: "waiting" }), resource_wait_set: true, resource_wait: { resource: "gpu", stage: "build", sequence: 1, queued_since: "2026-10-17T09:00:00Z" } },
);
assert.deepEqual(waiting.resource_wait, { resource: "gpu", stage: "build", sequence: 1, queued_since: "2026-10-17T09:00:00Z" });
const cleared = foldRunEventDetail(waiting, { ...frame({ runId: 1 }), resource_wait_set: true });
assert.equal(cleared.resource_wait, null);
// A terminal status always clears it.
const terminal = foldRunEventDetail(waiting, frame({ runId: 1, status: "succeeded" }));
assert.equal(terminal.resource_wait, null);
});
// --- resync ------------------------------------------------------------------
test("a reconnect resync takes the server snapshot then ignores a late older delta", () => {
const stale = detail({ status: "running", currentStage: "build" });
const snapshot = foldRunEventDetail(
stale,
frame({
runId: 1,
status: "succeeded",
outcome: "success",
completedSteps: 1,
finishedAt: "2026-10-17T09:07:00Z",
step: { sequence: 1, stage: "build", status: "completed", outcome: "success", target: "#success" },
}),
);
const after = foldRunEventDetail(snapshot, frame({ runId: 1, status: "running", currentStage: "build" }));
assert.equal(after.status, "succeeded");
assert.equal(after.outcome, "success");
assert.equal(after.executing, false);
assert.equal(after.completed_steps, 1);
assert.equal(after.steps.length, 1);
assert.equal(after.steps[0].status, "completed");
});
// --- sync heuristic ----------------------------------------------------------
test("settled visits, terminal runs and pending decisions need a full detail sync", () => {
assert.equal(runDetailNeedsFullSync(frame({ runId: 1, step: { sequence: 1, status: "completed" } })), true);
assert.equal(runDetailNeedsFullSync(frame({ runId: 1, step: { sequence: 1, status: "failed" } })), true);
assert.equal(runDetailNeedsFullSync(frame({ runId: 1, status: "succeeded" })), true);
assert.equal(runDetailNeedsFullSync(frame({ runId: 1, status: "canceled" })), true);
assert.equal(runDetailNeedsFullSync(frame({ runId: 1, status: "waiting" })), true);
assert.equal(runDetailNeedsFullSync(frame({ runId: 1, step: { sequence: 1, status: "waiting" } })), true);
assert.equal(runDetailNeedsFullSync(frame({ runId: 1, status: "running", step: { sequence: 1, status: "running" } })), false);
assert.equal(runDetailNeedsFullSync(frame({ runId: 1, status: "queued" })), false);
assert.equal(runDetailNeedsFullSync(frame({ runId: 1, status: "running" })), false);
});
test("a waiting human stage is not executing", () => {
assert.equal(isTerminalRunStatus("waiting"), false);
const fold = foldRunEvent([summary(1, { status: "running" })], frame({ runId: 1, status: "waiting", currentStage: "gate" }));
assert.equal(fold.runs[0].status, "waiting");
assert.equal(fold.runs[0].executing, false, "a parked human stage shows no spinner");
});
test("newRunSummaryFromFrame requires a pipeline identity and a name", () => {
assert.equal(newRunSummaryFromFrame(frame({ runId: 9, status: "queued" })), null);
assert.equal(newRunSummaryFromFrame(frame({ runId: 9, status: "queued", pipelineId: 4, name: " " })), null);
const built = newRunSummaryFromFrame(frame({ runId: 9, status: "queued", pipelineId: 4, name: "p", version: 2 }));
assert.equal(built.id, 9);
assert.equal(built.pipeline_id, 4);
assert.equal(built.title, null);
});
+291 -30
View File
@@ -13,7 +13,7 @@
// real runs and chat session. The default run never sends a `submit`: the live
// server's pipeline list is never mutated.
import { spawn } from "node:child_process";
import { mkdir, rm } from "node:fs/promises";
import { mkdir, mkdtemp, rm } from "node:fs/promises";
import net from "node:net";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
@@ -75,6 +75,10 @@ let failed = false;
// The throwaway chat session this run creates for the running-turn capture. It
// is closed again in cleanup; null once closed (or when none was created).
let throwawaySessionId = null;
// R10 evidence: the one small real run this smoke starts in a throwaway
// workspace and the workspace itself, both cleaned up in `finally`.
let liveRunId = null;
let liveRunScratch = null;
function step(label) {
console.log(`• ${label}`);
@@ -158,6 +162,57 @@ async function openRunDetail(runId) {
}
}
// --- R10: one small real run for live-push evidence ------------------------
// Runs the CLI in a throwaway workspace (never the repo) so a real `run_event`
// push can be observed against the live server. `--json` admits the run and
// exits; the caller waits for the pushed fold and then cancels it. Best-effort:
// a missing CLI only means the capture falls back, it never fails the smoke.
function runCli(args, cwd) {
const bin = process.env.PIPELINES_BIN || "pipelines";
const child = spawn(bin, args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
let stdout = "";
let stderr = "";
child.stdout.on("data", (chunk) => (stdout += chunk.toString()));
child.stderr.on("data", (chunk) => (stderr += chunk.toString()));
return new Promise((resolvePromise) => {
child.on("close", (code) => resolvePromise({ code, stdout, stderr }));
});
}
async function startLiveRun() {
const dir = await mkdtemp(join(tmpdir(), "pipelines-web-r10-"));
liveRunScratch = dir;
const result = await runCli(
[
"run",
"Complete a Task",
// A prompt that keeps the model busy for a few seconds, so the pushed
// "running" detail can be captured and the cancel lands while it executes.
"Write a detailed 600-word essay on why small, focused changes are easier to review than large ones. Do not use any tools and do not change any files.",
"--title",
"Web R10 live badge check",
"--json",
"--yes",
],
dir,
);
if (result.code !== 0) {
throw new Error(`pipelines run exited ${result.code}: ${result.stderr.trim().slice(-300)}`);
}
const id = Number(JSON.parse(result.stdout).run_detail?.id);
if (!(id > 0)) throw new Error(`pipelines run returned no run id: ${result.stdout.slice(0, 200)}`);
liveRunId = id;
return id;
}
async function cancelLiveRun(id) {
const result = await runCli(["cancel", String(id), "--yes", "--json"], liveRunScratch ?? tmpdir());
if (result.code !== 0) {
console.log(` cancel ${id} exited ${result.code}: ${result.stderr.trim().slice(-200)}`);
}
}
// --- P5: create a pipeline from a template (scratch server only) -----------
// The scratch server has no real runs and no live chat, so this is the whole
// point of a `--scratch` run. It opens a template, opens the create action,
@@ -422,6 +477,36 @@ try {
const nativeSend = ws.send.bind(ws);
window.__pipelinesSockets.push(ws);
window.__pipelinesSocket = ws;
// R10: record the real run-event watch traffic the page sends, and the real
// run_event frames the server pushes, so the smoke can assert the
// unsubscribe/subscribe/list_runs sequence and a genuine pushed fold rather
// than replaying a synthetic frame. The onmessage setter wraps the app's
// own handler so every real frame is observed before it is folded.
window.__runSubscribes = window.__runSubscribes || [];
window.__runUnsubscribes = window.__runUnsubscribes || 0;
window.__runEventFrames = window.__runEventFrames || [];
const nativeOnMessage = Object.getOwnPropertyDescriptor(Native.prototype, "onmessage");
let handler = null;
Object.defineProperty(ws, "onmessage", {
configurable: true,
get() {
return handler;
},
set(fn) {
handler = fn;
nativeOnMessage.set.call(ws, (event) => {
try {
const message = JSON.parse(event.data);
if (message && message.type === "run_event" && message.run_event) {
window.__runEventFrames.push(message.run_event);
}
} catch (_) {
/* not a frame we observe */
}
if (typeof fn === "function") fn.call(ws, event);
});
},
});
// One send path for every scripted hook, in order: record the frame type for
// assertions (C11/C10), suppress a synthetic chat_reply (C10, answering it
// locally so no real decision reaches the server), then honour the generic
@@ -435,6 +520,10 @@ try {
}
if (frame) {
(window.__pipelinesSentTypes = window.__pipelinesSentTypes || []).push(frame.type);
// R10: the exact subscribe/unsubscribe frames, so the smoke proves the
// blank-workspace shape (and that no run_ids are sent).
if (frame.type === "subscribe") window.__runSubscribes.push(frame);
if (frame.type === "unsubscribe") window.__runUnsubscribes += 1;
// C12: the prompt bodies this page actually sent, so the smoke can prove
// the help command and an unknown command generate none, and that the
// offer path sends exactly the draft it offered.
@@ -1460,15 +1549,15 @@ try {
// a ticking elapsed badge; a finished run (above) carries neither. Best effort:
// the run may stop before we look, so skip rather than fail if none is running.
step("opening a live run's Graph tab (R5: halo + elapsed badge)");
const liveRunId = await page.evaluate(`(() => {
const liveGraphRunId = await page.evaluate(`(() => {
const row = [...document.querySelectorAll(".list.runs .run-row")].find((n) => n.querySelector(".chip-running"));
const meta = row?.querySelector(".run-row-meta[data-run-id]");
return meta ? Number(meta.getAttribute("data-run-id")) : null;
})()`);
if (liveRunId == null) {
if (liveGraphRunId == null) {
console.log(" -- no running run available; skipped the live graph halo capture");
} else {
await page.click(`.list.runs .run-row-meta[data-run-id="${liveRunId}"]`);
await page.click(`.list.runs .run-row-meta[data-run-id="${liveGraphRunId}"]`);
await page.waitForSelector(".detail .detail-tabs", { timeout: 25000 });
await page.clickText(".detail .detail-tabs .detail-tab", "Graph");
await page.waitForSelector(".run-detail .graph-node", { timeout: 15000 });
@@ -1486,7 +1575,7 @@ try {
{ timeout: 15000, what: "the live run's active stage" },
);
if (!live.badge) throw new Error(`the active stage showed no elapsed badge: ${JSON.stringify(live)}`);
console.log(` live run ${liveRunId}: active stage ${live.activeKey} (pulsing=${live.pulsing}), badge ${live.badge}`);
console.log(` live run ${liveGraphRunId}: active stage ${live.activeKey} (pulsing=${live.pulsing}), badge ${live.badge}`);
await page.rect(".run-detail .graph-viewport");
shots.push(await page.screenshot(`${out}/31-run-live-graph.png`));
}
@@ -1527,6 +1616,12 @@ try {
if (!connectedText.includes("Connected")) throw new Error(`connected banner text: ${connectedText}`);
const buildLabel = await text(".conn-banner .conn-build");
if (!buildLabel.startsWith("Server build ")) throw new Error(`missing server build line: ${buildLabel}`);
// R10: the badge is driven by the live run-stream state, so on a healthy
// subscription it reads "live" even when no run is executing.
await eventually(
() => page.evaluate(`document.querySelector(".list.runs .live-badge .live-word")?.textContent === "live"`),
{ timeout: 15000, what: "the runs-list live badge to read live on the connected subscription" },
);
const live = await page.evaluate(`(() => {
const badge = document.querySelector(".list.runs .live-badge");
return badge ? { text: badge.textContent.trim(), cls: badge.className } : null;
@@ -1548,6 +1643,12 @@ try {
if (idleText !== "Not connected. Enter the server URL in Settings, then connect.") {
throw new Error(`idle banner text: ${idleText}`);
}
// R10: a deliberate Disconnect clears the watch, so the badge reads "not live".
await eventually(
() => page.evaluate(`document.querySelector(".list.runs .live-badge .live-word")?.textContent === "not live"`),
{ timeout: 10000, what: "the runs-list live badge to read not live while idle" },
);
console.log(" idle: the runs-list live badge reads not live (watch cleared on disconnect)");
await page.click("#reconnectBtn"); // immediate reconnect to the closed port
await page.waitForSelector(".conn-banner.conn-error", { timeout: 15000 });
await page.waitForText("retrying with backoff", { timeout: 15000 });
@@ -1555,6 +1656,12 @@ try {
if (!/^Disconnected: .+ · retrying with backoff$/.test(disconnectedText)) {
throw new Error(`disconnected banner text: ${disconnectedText}`);
}
// R10: a dropped/reconnecting socket reads "reconnecting".
await eventually(
() => page.evaluate(`document.querySelector(".list.runs .live-badge .live-word")?.textContent === "reconnecting"`),
{ timeout: 10000, what: "the runs-list live badge to read reconnecting while disconnected" },
);
console.log(" disconnected: the runs-list live badge reads reconnecting");
// The banner text is ConnectionBanner.kt verbatim (no attempt); the retry
// attempt is the header state badge's job, so the capture must show it there.
const badgeText = await text("#stateBadge");
@@ -1581,9 +1688,124 @@ try {
await page.evaluate(`window.scrollTo(0, 0)`);
const recoveredText = await text(".conn-banner .conn-banner-text");
if (!recoveredText.includes("Connected")) throw new Error(`recovered banner text: ${recoveredText}`);
console.log(` recovered banner after window online: ${recoveredText}`);
// R10: the re-subscribe's ack/push/successful read confirms the stream live.
await eventually(
() => page.evaluate(`document.querySelector(".list.runs .live-badge .live-word")?.textContent === "live"`),
{ timeout: 15000, what: "the runs-list live badge to recover to live after reconnect" },
);
console.log(` recovered banner after window online: ${recoveredText} (badge live again)`);
await shot("30-reconnect-recovered.png");
// R10: the live run stream. First the wire shape of the watch on a forced
// reconnect (unsubscribe, then subscribe with the blank workspace, then one
// list_runs), then a real small run whose `run_event` pushes fold the list and
// the open detail (running -> canceled), both captured.
step("R10: unsubscribe/subscribe/list_runs on a real reconnect");
await ensureConnected();
await page.clickText("nav.tabs button.tab", "Runs");
await page.waitForSelector(".list.runs .run-row", { timeout: 25000 });
await page.evaluate(`window.__pipelinesSocket && window.__pipelinesSocket.close(4000, "r10 reconnect")`);
// Observe the drop before arming the counters and waiting for the recovery, so
// the unsubscribe/subscribe/list_runs sequence counted below necessarily
// belongs to the re-subscribe rather than a prior reconnect's traffic.
await eventually(
() => page.evaluate(`!document.querySelector(".conn-banner")?.classList.contains("conn-ok")`),
{ timeout: 15000, what: "the socket to drop before the run-event reconnect" },
);
await page.evaluate(
`(() => { window.__pipelinesSentTypes = []; window.__runSubscribes = []; window.__runUnsubscribes = 0; window.__runEventFrames = []; return true; })()`,
);
await ensureConnected(30000);
await eventually(
() => page.evaluate(`(window.__pipelinesSentTypes || []).includes("list_runs")`),
{ timeout: 20000, what: "the resync list_runs after the run-event subscribe" },
);
const r10wire = await page.evaluate(`(() => {
const types = window.__pipelinesSentTypes || [];
const subscribes = window.__runSubscribes || [];
const last = subscribes.length ? { ...subscribes[subscribes.length - 1] } : null;
if (last) delete last.id;
return {
unsubscribe: window.__runUnsubscribes || 0,
unsubscribeAt: types.indexOf("unsubscribe"),
subscribeAt: types.indexOf("subscribe"),
listRunsAt: types.indexOf("list_runs"),
lastSubscribe: last,
};
})()`);
if (r10wire.unsubscribe < 1) throw new Error("R10: no unsubscribe was sent on reconnect");
if (!(r10wire.unsubscribeAt >= 0 && r10wire.subscribeAt > r10wire.unsubscribeAt)) {
throw new Error(`R10: unsubscribe/subscribe order was ${JSON.stringify(r10wire)}`);
}
if (!(r10wire.listRunsAt > r10wire.subscribeAt)) {
throw new Error(`R10: list_runs did not follow the subscribe: ${JSON.stringify(r10wire)}`);
}
if (r10wire.lastSubscribe?.type !== "subscribe" || r10wire.lastSubscribe?.workspace !== "" || "run_ids" in (r10wire.lastSubscribe ?? {})) {
throw new Error(`R10: subscribe frame was ${JSON.stringify(r10wire.lastSubscribe)}`);
}
console.log(
` reconnect wire: ${r10wire.unsubscribe} unsubscribe, subscribe ${JSON.stringify(r10wire.lastSubscribe)}, then list_runs (positions ${r10wire.unsubscribeAt}/${r10wire.subscribeAt}/${r10wire.listRunsAt})`,
);
step("R10: starting one small real run and folding its pushed updates");
let r10RunId = null;
try {
r10RunId = await startLiveRun();
console.log(` started run ${r10RunId} in ${liveRunScratch}`);
// The row appears from the frame's own pipeline identity, without a refetch.
await eventually(
() => page.evaluate(`!!document.querySelector('.list.runs .run-row-meta[data-run-id="${r10RunId}"]')`),
{ timeout: 30000, what: `run ${r10RunId}'s row to appear from a pushed frame` },
);
// Prove at least one genuine server push (not a replay) for this run.
await eventually(
() => page.evaluate(`(window.__runEventFrames || []).some((f) => Number(f.run_id) === ${r10RunId})`),
{ timeout: 30000, what: `a pushed run_event frame for run ${r10RunId}` },
);
const listStatus = await page.evaluate(
`document.querySelector('.list.runs .run-row-meta[data-run-id="${r10RunId}"]')?.closest(".run-row")?.querySelector(".chip")?.textContent ?? ""`,
);
const listBadge = await page.evaluate(
`document.querySelector(".list.runs .live-badge .live-word")?.textContent ?? ""`,
);
console.log(` pushed list fold: run ${r10RunId} chip "${listStatus}", badge "${listBadge}"`);
if (listBadge !== "live") throw new Error(`R10: list badge read ${JSON.stringify(listBadge)}`);
await shot("75-run-live-list.png");
await openRunDetail(r10RunId);
const detailBadge = await page.evaluate(
`document.querySelector(".run-detail .live-badge .live-word")?.textContent ?? ""`,
);
const detailStatus = await page.evaluate(
`document.querySelector(".run-detail .run-chip-row .chip")?.textContent ?? ""`,
);
if (detailBadge !== "live") throw new Error(`R10: detail badge read ${JSON.stringify(detailBadge)}`);
console.log(` pushed detail fold: status "${detailStatus}", badge "${detailBadge}"`);
await shot("76-run-live-detail.png");
// Cancel the throwaway run; the terminal frame must fold the open detail to a
// terminal word without a poll (there is no run-detail timer any more). A
// trivial prompt can finish before the cancel lands, so either terminal word
// is accepted: the point is the pushed fold, never the timing race.
await cancelLiveRun(r10RunId);
await eventually(
() =>
page.evaluate(
`["canceled","succeeded","failed","interrupted"].includes(document.querySelector(".run-detail .run-chip-row .chip")?.textContent ?? "")`,
),
{ timeout: 30000, what: `run ${r10RunId}'s detail to fold to a terminal state` },
);
const finalStatus = await page.evaluate(
`document.querySelector(".run-detail .run-chip-row .chip")?.textContent ?? ""`,
);
console.log(` terminal fold: run ${r10RunId} detail now "${finalStatus}" (pushed, no poll)`);
await shot("77-run-live-terminal.png");
liveRunId = null; // terminal; the finally must not cancel it again
} catch (error) {
console.log(` R10: could not observe a real pushed run (${error.message}); no live-run capture`);
if (r10RunId) await cancelLiveRun(r10RunId);
}
// R4: the run Summary's collapsible Changes section and its rendered/raw diff
// viewer. Run 438 changed 22 unique files across 6 steps with every capture
// partial, so the header/row count are asserted against each other rather than
@@ -2398,35 +2620,43 @@ try {
if (chatList.placeholder !== "Search chats") throw new Error(`chat search placeholder: ${chatList.placeholder}`);
if (chatList.rows.length === 0) throw new Error("no chat sessions rendered");
if (!chatList.rows[0].title) throw new Error("the first chat row has no title");
if (!chatList.rows[0].chips.includes("Operator")) {
throw new Error(`chat kind chip: ${chatList.rows[0].chips.join(", ")}`);
if (!chatList.rows.some((row) => row.chips.includes("Operator"))) {
throw new Error(`chat kind chips: ${JSON.stringify(chatList.rows.map((row) => row.chips))}`);
}
if (!chatList.rows[0].close) throw new Error("the open chat row offers no Close chat action");
// The list's first row is the newest session; a shared live server can carry a
// closed verification row above the operator's when another client closed it,
// so the open row to exercise is the first with a Close action, not rows[0].
const openRow = chatList.rows.find((row) => row.close) ?? null;
console.log(
` chat list: ${chatList.rows.length} session(s); "${chatList.rows[0].title}" ` +
`[${chatList.rows[0].chips.join(", ")}] — ${chatList.rows[0].meta}`,
` chat list: ${chatList.rows.length} session(s); first "${chatList.rows[0].title}" ` +
`[${chatList.rows[0].chips.join(", ")}] — ${chatList.rows[0].meta}` +
(openRow ? `; open row "${openRow.title}"` : " (no open row)"),
);
await shot("44-chat-sessions.png");
// The row's own Close chat… action with NO conversation open: the list offers
// it on every open row, so the confirm must still appear. Shown and dismissed
// with Cancel; no chat_close is ever sent for the operator's live session.
await page.click(`#chatRows .chat-row[data-session="${chatList.rows[0].id}"] .chat-close`);
await page.waitForSelector(".confirm-chat-close", { timeout: 15000 });
const listClose = await page.evaluate(
`document.querySelector(".confirm-chat-close .confirm-message")?.textContent ?? ""`,
);
if (!listClose.startsWith('Closes "')) {
throw new Error(`row-level close message with no conversation open: ${listClose}`);
// with Cancel; no chat_close is ever sent for any live session.
if (!openRow) {
console.log(" -- no open chat row on this shared server; skipped the row-level Close confirm");
} else {
await page.click(`#chatRows .chat-row[data-session="${openRow.id}"] .chat-close`);
await page.waitForSelector(".confirm-chat-close", { timeout: 15000 });
const listClose = await page.evaluate(
`document.querySelector(".confirm-chat-close .confirm-message")?.textContent ?? ""`,
);
if (!listClose.startsWith('Closes "')) {
throw new Error(`row-level close message with no conversation open: ${listClose}`);
}
await page.click(".confirm-chat-close .confirm-dismiss");
await eventually(() => page.evaluate(`!document.querySelector(".confirm-chat-close")`), {
timeout: 5000,
what: "the row-level close dialog to close",
});
console.log(
" row-level 'Close chat…' (no conversation open) showed its confirm; dismissed with Cancel",
);
}
await page.click(".confirm-chat-close .confirm-dismiss");
await eventually(() => page.evaluate(`!document.querySelector(".confirm-chat-close")`), {
timeout: 5000,
what: "the row-level close dialog to close",
});
console.log(
" row-level 'Close chat…' (no conversation open) showed its confirm; dismissed with Cancel",
);
// The narrowed-to-nothing empty state and its Clear search action (the field
// itself is the narrowing; the list's order is never touched).
@@ -3380,16 +3610,25 @@ try {
await page.evaluate(
`document.querySelector("#chatInput").dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true, cancelable: true }))`,
);
// The opened conversation, not the list row, is the durable proof: an
// ephemeral session has no listed row, and the list refresh that follows the
// create removes it, so the selected row is only briefly present and a poll can
// miss it. The transcript's own session id stays for as long as the pane is open.
const throwawayId = await eventually(
async () => {
const id = await page.evaluate(`document.querySelector(".chat-row.selected")?.dataset.session ?? ""`);
const id = await page.evaluate(`document.querySelector("#chatTranscript")?.dataset?.session ?? ""`);
return id && id !== openedId ? id : false;
},
{ timeout: 25000, what: "the new throwaway session to open" },
);
await page.waitForSelector("#chatInput", { timeout: 10000 });
const throwawayTitle = await page.evaluate(`document.querySelector(".chat-bar-title")?.textContent ?? ""`);
if (throwawayTitle !== "New chat") throw new Error(`new session title: ${throwawayTitle}`);
// A given server build usually echoes the title back; when it does not, the app
// falls back to the session id (sessionTitle), which is honest and still proves
// the pane opened on the created session.
if (throwawayTitle !== "New chat" && !throwawayTitle.startsWith("ses_")) {
throw new Error(`new session title: ${throwawayTitle}`);
}
throwawaySessionId = throwawayId;
const startedNotice = await page.evaluate(
`document.querySelector(".chat-command-notice .chat-command-notice-text")?.textContent ?? ""`,
@@ -5004,11 +5243,33 @@ try {
}
}
} finally {
// Best-effort cleanup of the R10 throwaway run: if the smoke failed before it
// canceled the run, cancel it now so it does not keep executing.
if (liveRunId) {
try {
await cancelLiveRun(liveRunId);
console.log(` cleanup: canceled the R10 run ${liveRunId}`);
} catch {
// the CLI may be gone; the failure above is the real signal
}
}
if (liveRunScratch) {
try {
await rm(liveRunScratch, { recursive: true, force: true });
} catch {
// leave the scratch directory rather than failing cleanup
}
}
// Best-effort cleanup of the throwaway session this run created: if the run
// failed before its `/close` step, close it now so it is never left open.
// failed before its `/close` step, close it now so it is never left open. It
// closes only the throwaway: if the open conversation is something else (the
// failure left the operator's session on screen), the cleanup must never close
// that instead.
if (throwawaySessionId && page) {
try {
await page.evaluate(`(async () => {
const open = document.querySelector("#chatTranscript")?.dataset?.session ?? "";
if (open !== ${JSON.stringify(throwawaySessionId)}) return;
const input = document.querySelector("#chatInput");
if (!input) return;
input.value = "/close";