web: exercise the human decision sheet live (R11)

Add an opt-in --decide smoke mode that reuses or submits a human-only pipeline, starts two runs in throwaway workspaces, and presses the real card, sheet and Confirm for the feedback and no-feedback shapes. Record R11 as done with the live captures.
This commit is contained in:
2026-10-19 16:16:05 +01:00
parent 0d3093ac27
commit 2deaaea05a
2 changed files with 450 additions and 12 deletions
+445 -7
View File
@@ -14,8 +14,14 @@
// against a scratch server and skips the live suite, which needs the operator's
// real runs and chat session. The default run never sends a `submit`: the live
// server's pipeline list is never mutated.
//
// `--decide` drives ONLY R11's live human-decision flow: it reuses an existing
// entry-stage human pipeline or submits one uniquely named create-only
// definition, starts two throwaway runs in fresh temp workspaces, and presses
// the real card/sheet/Confirm decision for both the feedback and no-feedback
// paths. It is deliberately opt-in because it is the only mode that submits.
import { spawn } from "node:child_process";
import { mkdir, mkdtemp, rm } from "node:fs/promises";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import net from "node:net";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
@@ -74,6 +80,8 @@ const out = resolve(
const scratch = process.argv.includes("--scratch");
// Focused notification evidence only; skips the unrelated live-suite steps.
const notifyOnly = process.argv.includes("--notify");
// Focused R11 evidence only; the one explicit mode allowed to submit a pipeline.
const decideOnly = process.argv.includes("--decide");
// The page must connect through its own proxy (the server rejects a cross-origin
// upgrade), but serve.mjs only proxies /connect — not /artifacts. Point the URL
@@ -81,6 +89,7 @@ const notifyOnly = process.argv.includes("--notify");
// viewer resolve to real files; that is exactly the URL a correctly configured
// page would carry.
const artifactServerUrl = `ws://${pipelines}/connect`;
const pipelineUrl = artifactServerUrl;
const shots = [];
let serve = null;
@@ -94,6 +103,14 @@ let throwawaySessionId = null;
// workspace and the workspace itself, both cleaned up in `finally`.
let liveRunId = null;
let liveRunScratch = null;
// R11 evidence: the human-stage pipeline used (reused or uniquely submitted),
// every throwaway run started for the decision path, and their temp workspaces.
// Cleanup cancels only these runs while they are still active and removes only
// these directories; the submitted pipeline definition is intentionally retained.
let decidePipelineName = null;
let decidePipelineReused = false;
const decideRunIds = [];
const decideScratchDirs = [];
// C2's one real session. It is closed in its own step; kept here only so a
// failure before that step can still close it rather than leave it in the list.
let createdSessionId = null;
@@ -231,6 +248,387 @@ async function cancelLiveRun(id) {
}
}
// --- R11: live human-decision evidence -------------------------------------
// A human stage parks a run without starting a model, so it is safe to run on
// the live server in throwaway workspaces. This path is reached only with the
// explicit `--decide` flag: the default suite still never submits a pipeline.
const DECIDE_PROMPT = "R11 live decision: review the run and choose approve or hold.";
const DECIDE_OPTIONS = ["approve", "hold"];
function cliJson(result, what) {
if (result.code !== 0) {
throw new Error(`${what} exited ${result.code}: ${result.stderr.trim().slice(-400)}`);
}
try {
return JSON.parse(result.stdout);
} catch {
throw new Error(`${what} returned no JSON: ${result.stdout.slice(0, 300)}`);
}
}
async function inspectRun(id) {
return cliJson(
await runCli(["inspect", String(id), "--json", "--url", pipelineUrl], tmpdir()),
`pipelines inspect ${id}`,
);
}
async function waitForRun(id, predicate, what, timeout = 60000) {
const deadline = Date.now() + timeout;
let detail = null;
while (Date.now() < deadline) {
detail = await inspectRun(id);
if (predicate(detail)) return detail;
await new Promise((resolvePromise) => setTimeout(resolvePromise, 250));
}
throw new Error(`run ${id} did not ${what}; last status ${detail?.status ?? "unknown"}`);
}
/**
* The one pipeline R11 may use without submitting: an existing definition whose
* entry stage is already `type=human`, so starting it runs no model stage. A
* human stage later in a model pipeline is deliberately not reused.
*/
async function findEntryHumanPipeline() {
const list = cliJson(
await runCli(["list", "--json", "--url", pipelineUrl], tmpdir()),
"pipelines list",
);
for (const item of list.pipelines ?? []) {
const shown = await runCli(["show", String(item.id), "--json", "--url", pipelineUrl], tmpdir());
if (shown.code !== 0) continue;
let detail = null;
try {
detail = JSON.parse(shown.stdout);
} catch {
continue;
}
if (detail?.graph?.stages?.[0]?.type === "human") {
return { id: Number(item.id), name: String(detail.name ?? item.name) };
}
}
return null;
}
/** The tiny create-only definition: one human stage, both options succeed. */
function humanOnlySource() {
return [
"# stage gate",
"",
"```output",
"Decision evidence.",
"```",
"",
"```prompt",
DECIDE_PROMPT,
"```",
"",
"```edges",
"approve -> #success",
"hold -> #success",
"error -> #failure",
"```",
"",
"```configs",
"type=human",
"```",
"",
].join("\n");
}
/**
* Use an existing entry-human pipeline, or submit exactly one uniquely named
* create-only definition. Then start two runs in separate fresh workspaces (the
* server permits one run per workspace) and wait for both to park in `waiting`.
* Every run id and workspace is tracked for the `finally` cleanup.
*/
async function prepareHumanDecisionRuns() {
const existing = await findEntryHumanPipeline();
if (existing) {
decidePipelineName = existing.name;
decidePipelineReused = true;
console.log(` reusing entry-human pipeline ${existing.name} (id ${existing.id}); no submit`);
} else {
const definitionDir = await mkdtemp(join(tmpdir(), "pipelines-web-r11-def-"));
decideScratchDirs.push(definitionDir);
const definition = join(definitionDir, "human.md");
await writeFile(definition, humanOnlySource(), "utf8");
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
const requested = `Web R11 decision ${stamp}`;
const submitted = cliJson(
await runCli(
[
"submit",
definition,
"--name",
requested,
"--create-only",
"--yes",
"--json",
"--url",
pipelineUrl,
],
definitionDir,
),
"pipelines submit",
);
decidePipelineName = String(submitted.name ?? requested).trim() || requested;
console.log(` submitted create-only pipeline ${JSON.stringify(decidePipelineName)}`);
}
const prepared = [];
for (const [index, suffix] of [
[1, "feedback"],
[2, "no-feedback"],
]) {
const workspace = await mkdtemp(join(tmpdir(), "pipelines-web-r11-"));
decideScratchDirs.push(workspace);
const title = `Web R11 decision ${index} (${suffix})`;
const started = cliJson(
await runCli(
[
"run",
decidePipelineName,
"R11 web decision smoke",
"--workspace",
workspace,
"--title",
title,
"--json",
"--yes",
"--url",
pipelineUrl,
],
workspace,
),
`pipelines run ${decidePipelineName}`,
);
const id = Number(started.run_detail?.id);
if (!(id > 0)) throw new Error(`pipelines run returned no run id: ${started ? JSON.stringify(started).slice(0, 200) : "no output"}`);
decideRunIds.push(id);
prepared.push({ id, title, suffix, workspace });
}
for (const run of prepared) {
const detail = await waitForRun(
run.id,
(d) => d.status === "waiting" && d.current_stage === "gate",
"reach waiting on the gate stage",
);
const waitingStep = (detail.steps ?? []).find((step_) => step_.status === "waiting");
const options = waitingStep?.output?.options ?? [];
if (JSON.stringify(options) !== JSON.stringify(DECIDE_OPTIONS)) {
throw new Error(`run ${run.id} waiting options: ${JSON.stringify(options)}`);
}
if (waitingStep?.output?.prompt !== DECIDE_PROMPT) {
throw new Error(`run ${run.id} waiting instructions: ${JSON.stringify(waitingStep?.output?.prompt)}`);
}
console.log(` run ${run.id} waiting on gate in ${run.workspace} (options ${options.join(", ")})`);
}
return prepared;
}
/**
* One complete live decision: card, sheet, deliberate selection, one Confirm,
* the resumed run and the recorded verdict. Feedback is optional; the same flow
* proves both wire shapes.
*/
async function exerciseDecisionRun(run, { option, feedback, labels }) {
const { id, title } = run;
const expectedMeta = `gate · run ${id} · ${title}`;
console.log(`• R11: ${labels.label} (run ${id})`);
await ensureConnected();
await eventually(
() => page.evaluate(`!!document.querySelector('.list.runs .run-row-meta[data-run-id="${id}"]')`),
{ timeout: 30000, what: `run ${id}'s waiting row to appear` },
);
await openRunDetail(id);
await page.waitForSelector(".run-detail .decision-card", { timeout: 20000 });
const card = await page.evaluate(`(() => ({
title: document.querySelector(".decision-card .decision-card-title")?.textContent ?? "",
meta: document.querySelector(".decision-card .decision-card-meta")?.textContent ?? "",
action: document.querySelector(".decision-card .decision-open")?.textContent ?? "",
}))()`);
if (card.title !== "Waiting for your decision") throw new Error(`card title: ${card.title}`);
if (card.meta !== expectedMeta) throw new Error(`card meta: ${card.meta}`);
if (card.action !== "Decide") throw new Error(`card action: ${card.action}`);
console.log(` card: ${card.title} — ${card.meta}`);
shots.push(await page.screenshot(`${out}/${labels.card}`));
await page.click(".decision-card .decision-open");
await page.waitForSelector(".decision-sheet .decision-option", { timeout: 20000 });
const sheet = await page.evaluate(`(() => ({
title: document.querySelector(".decision-sheet-title")?.textContent ?? "",
meta: document.querySelector(".decision-sheet-meta")?.textContent ?? "",
instructions: document.querySelector(".decision-instructions")?.textContent ?? "",
options: [...document.querySelectorAll(".decision-sheet .decision-option")].map((n) => ({
key: n.querySelector(".decision-option-key")?.textContent ?? "",
target: n.querySelector(".decision-option-target")?.textContent ?? "",
})),
feedbackLabel: document.querySelector(".decision-field-label")?.textContent ?? "",
confirm: document.querySelector(".decision-confirm")?.textContent ?? "",
confirmDisabled: document.querySelector(".decision-confirm")?.disabled ?? null,
}))()`);
if (sheet.title !== "Decide gate") throw new Error(`sheet title: ${sheet.title}`);
if (sheet.meta !== `run ${id} · ${title}`) throw new Error(`sheet meta: ${sheet.meta}`);
if (sheet.instructions !== DECIDE_PROMPT) {
throw new Error(`sheet instructions: ${JSON.stringify(sheet.instructions)}`);
}
const optionKeys = sheet.options.map((row) => row.key);
if (JSON.stringify(optionKeys) !== JSON.stringify(DECIDE_OPTIONS)) {
throw new Error(`sheet options: ${JSON.stringify(optionKeys)}`);
}
if (sheet.options.some((row) => row.target !== "\u2192 SUCCESS")) {
throw new Error(`sheet targets: ${JSON.stringify(sheet.options)}`);
}
if (sheet.feedbackLabel !== "Feedback (optional)") {
throw new Error(`sheet feedback label: ${sheet.feedbackLabel}`);
}
if (sheet.confirm !== "Confirm decision") throw new Error(`sheet confirm: ${sheet.confirm}`);
if (sheet.confirmDisabled !== true) throw new Error("Confirm decision must start disabled");
console.log(
` sheet: ${sheet.title} — ${sheet.meta}; ${sheet.options
.map((row) => `${row.key} ${row.target}`)
.join(", ")}; ${sheet.feedbackLabel}; Confirm disabled`,
);
shots.push(await page.screenshot(`${out}/${labels.sheet}`));
await page.click(`.decision-sheet .decision-option[data-option="${option}"]`);
await eventually(() => page.evaluate(`document.querySelector(".decision-confirm")?.disabled === false`), {
timeout: 5000,
what: "Confirm to enable after selecting an option",
});
const selected = await page.evaluate(`(() => ({
checked: document.querySelector('.decision-sheet .decision-option[data-option="${option}"]')?.getAttribute("aria-checked") ?? "",
decideBodies: (window.__decideBodies ?? []).filter((body) => Number(body.run_id) === ${id}),
}))()`);
if (selected.checked !== "true") throw new Error(`option ${option} did not render selected`);
if (selected.decideBodies.length !== 0) {
throw new Error(`selecting an option sent decide_run: ${JSON.stringify(selected.decideBodies)}`);
}
const stillWaiting = await inspectRun(id);
if (stillWaiting.status !== "waiting") {
throw new Error(`run ${id} left waiting on selection: ${stillWaiting.status}`);
}
console.log(" selecting sent no decide_run; the run is still waiting");
if (feedback) await page.fill(".decision-sheet .decision-feedback", feedback);
shots.push(await page.screenshot(`${out}/${labels.selected}`));
await page.click(".decision-sheet .decision-confirm");
await eventually(
() =>
page.evaluate(
`!document.querySelector(".decision-sheet") && !document.querySelector(".decision-card")`,
),
{ timeout: 20000, what: "the sheet and card to close after Confirm" },
);
const final = await waitForRun(
id,
(d) => d.status === "succeeded",
"succeed after the decision",
);
const bodies = await page.evaluate(
`(window.__decideBodies ?? []).filter((body) => Number(body.run_id) === ${id})`,
);
if (bodies.length !== 1) throw new Error(`expected exactly one decide_run, got ${JSON.stringify(bodies)}`);
const body = bodies[0];
if (body.type !== "decide_run" || Number(body.run_id) !== id || body.stage !== "gate" || body.option !== option) {
throw new Error(`decide_run body: ${JSON.stringify(body)}`);
}
if (feedback) {
if (body.feedback !== feedback) throw new Error(`decide_run feedback: ${JSON.stringify(body.feedback)}`);
} else if (Object.hasOwn(body, "feedback")) {
throw new Error(`no-feedback decide_run carried feedback: ${JSON.stringify(body)}`);
}
const gate = (final.steps ?? [])[0] ?? {};
if (gate.outcome !== option || gate.output?.option !== option) {
throw new Error(`recorded decision: ${JSON.stringify(gate.output)} outcome ${gate.outcome}`);
}
const recordedFeedback = String(gate.output?.feedback ?? "").trim();
if (recordedFeedback !== feedback) {
throw new Error(`recorded feedback: ${JSON.stringify(gate.output?.feedback)}`);
}
const expectedVerdict = `${option} \u2192 success`;
if (gate.verdict?.option !== expectedVerdict) {
throw new Error(`recorded verdict option: ${JSON.stringify(gate.verdict?.option)}`);
}
if (String(gate.verdict?.reason ?? "") !== feedback) {
throw new Error(`recorded verdict reason: ${JSON.stringify(gate.verdict?.reason)}`);
}
console.log(
` one decide_run ${JSON.stringify(body)}; run succeeded with ${expectedVerdict}` +
(feedback ? ` and the exact reason` : " and no reason"),
);
await eventually(
() =>
page.evaluate(
`["succeeded","Succeeded"].includes(document.querySelector(".run-detail .run-chip-row .chip")?.textContent ?? "")`,
),
{ timeout: 20000, what: `run ${id}'s detail to show succeeded` },
);
shots.push(await page.screenshot(`${out}/${labels.confirmed}`));
await page.clickText(".detail .detail-tabs .detail-tab", "Timeline");
await page.waitForSelector(`.detail .timeline-row[data-stage-key="gate"]`, { timeout: 20000 });
await page.click(`.detail .timeline-row[data-stage-key="gate"]`);
await page.waitForSelector(".stage-pane .verdict", { timeout: 20000 });
const verdict = await page.evaluate(`(() => ({
title: document.querySelector(".stage-pane .verdict-title")?.textContent ?? "",
option: document.querySelector(".stage-pane .verdict .outcome-chip")?.textContent ?? "",
reason: document.querySelector(".stage-pane .verdict-reason")?.textContent ?? "",
}))()`);
if (verdict.title !== "Verdict") throw new Error(`verdict heading: ${verdict.title}`);
if (verdict.option !== expectedVerdict) throw new Error(`verdict chip: ${verdict.option}`);
if (verdict.reason !== feedback) throw new Error(`verdict reason: ${JSON.stringify(verdict.reason)}`);
console.log(` verdict: ${verdict.option}${feedback ? ` — ${verdict.reason}` : " — no reason"}`);
shots.push(await page.screenshot(`${out}/${labels.verdict}`));
await page.evaluate(
`document.dispatchEvent(new KeyboardEvent("keydown", { key: "Escape", bubbles: true }))`,
);
await eventually(() => page.evaluate(`!document.querySelector(".stage-pane")`), {
timeout: 5000,
what: "the stage pane to close",
});
}
/** The focused R11 suite: two real waiting runs, both decided in the browser. */
async function runDecisionChecks() {
const prepared = await prepareHumanDecisionRuns();
await ensureConnected();
await page.clickText("nav.tabs button.tab", "Runs");
await page.waitForSelector(".list.runs .run-row", { timeout: 25000 });
await exerciseDecisionRun(prepared[0], {
option: "approve",
feedback: "Approved in the live browser decision sheet.",
labels: {
label: "decision with feedback",
card: `41-run-${prepared[0].id}-decision-card.png`,
sheet: `42-run-${prepared[0].id}-decision-sheet.png`,
selected: `43-run-${prepared[0].id}-decision-selected.png`,
confirmed: `44-run-${prepared[0].id}-decision-confirmed.png`,
verdict: `45-run-${prepared[0].id}-decision-verdict.png`,
},
});
await exerciseDecisionRun(prepared[1], {
option: "hold",
feedback: "",
labels: {
label: "decision without feedback",
card: `46-run-${prepared[1].id}-decision-card.png`,
sheet: `47-run-${prepared[1].id}-decision-sheet.png`,
selected: `48-run-${prepared[1].id}-decision-selected.png`,
confirmed: `49-run-${prepared[1].id}-decision-confirmed.png`,
verdict: `50-run-${prepared[1].id}-decision-verdict.png`,
},
});
console.log(
` pipeline ${JSON.stringify(decidePipelineName)} (${decidePipelineReused ? "reused" : "submitted create-only"}), runs ${decideRunIds.join(", ")} both succeeded`,
);
}
// Open one chat row's Chat actions menu (the row's management items live behind
// it since C13: Rename… / Close chat… or Reopen / Hide on this device).
async function openChatRowMenu(sessionId) {
@@ -719,6 +1117,10 @@ try {
// ack, so the card's refusal path (reopen with the reason) is exercised.
window.__failChatReplyIds = [];
window.__chatReplyCounts = { suppressed: 0, forwarded: 0, failed: 0 };
// R11: the exact decide_run bodies the page actually sends, retained (not
// suppressed) so the smoke can prove a selection sends nothing and Confirm
// sends exactly one body with the chosen option and optional feedback.
window.__decideBodies = [];
function Shim(url, protocols) {
const ws = protocols === undefined ? new Native(url) : new Native(url, protocols);
const nativeSend = ws.send.bind(ws);
@@ -779,6 +1181,8 @@ try {
frame.chat_prompt?.text ?? "",
);
}
// R11: retain the real decision frames without altering them.
if (frame.type === "decide_run") window.__decideBodies.push(frame);
// C2: the create bodies this page actually sent, so the smoke can prove
// the picker's chosen value reached chat_create.
if (frame.type === "chat_create") {
@@ -852,7 +1256,11 @@ try {
for (const shot of shots) console.log(` ${shot}`);
}
if (notifyOnly) {
if (decideOnly) {
await runDecisionChecks();
console.log("\nUI SMOKE PASS");
for (const shot of shots) console.log(` ${shot}`);
} else if (notifyOnly) {
await runNotificationChecks();
console.log("\nUI SMOKE PASS");
for (const shot of shots) console.log(` ${shot}`);
@@ -2814,10 +3222,11 @@ try {
}
}
// R11: the decision card/sheet, only when the live server actually has a run
// waiting on a person with recorded options. There is normally none (a human
// stage blocks a run until a person answers), so this stays a best-effort
// capture rather than a failure; the model is unit-tested in actions.test.mjs.
// R11: the decision card/sheet, only when the live server already has a run
// waiting on a person with recorded options. This default suite never submits
// and never confirms: it captures the waiting card, the sheet and the selected
// state and then dismisses with Cancel. The full live Confirm (both feedback
// paths) is exercised by the opt-in `--decide` mode above.
step("R11: the human-decision card/sheet (only when a waiting run is present)");
const waitingRunId = await page.evaluate(`(() => {
const row = [...document.querySelectorAll(".list.runs .run-row")].find((n) => n.querySelector(".chip-waiting"));
@@ -2825,7 +3234,7 @@ try {
return meta ? Number(meta.getAttribute("data-run-id")) : null;
})()`);
if (waitingRunId == null) {
console.log(" -- no waiting run available; skipped the decision card/sheet capture (R11 stays partial)");
console.log(" -- no waiting run available; skipped the decision card/sheet capture (full live path covered by --decide)");
} else {
await openRunDetail(waitingRunId);
await page.waitForSelector(".run-detail .decision-card", { timeout: 15000 });
@@ -6063,6 +6472,35 @@ try {
// leave the scratch directory rather than failing cleanup
}
}
// Best-effort cleanup of the R11 decision runs: cancel only these tracked runs
// while they are still active (never another run), then remove only the temp
// workspaces/definition directories this smoke created. The uniquely submitted
// human-only pipeline definition is intentionally retained and named in PARITY.
for (const id of decideRunIds) {
try {
const detail = await inspectRun(id);
if (["queued", "running", "waiting"].includes(detail.status)) {
const result = await runCli(
["cancel", String(id), "--yes", "--json", "--url", pipelineUrl],
tmpdir(),
);
console.log(
result.code === 0
? ` cleanup: canceled the R11 decision run ${id} (${detail.status})`
: ` cleanup: cancel ${id} exited ${result.code}`,
);
}
} catch {
// the server/CLI may be gone; the failure above is the real signal
}
}
for (const dir of decideScratchDirs) {
try {
await rm(dir, { recursive: true, force: true });
} catch {
// leave the scratch directory rather than failing cleanup
}
}
// Best-effort cleanup of the throwaway session this run created: if the run
// failed before its `/close` step, close it now so it is never left open. It
// closes only the throwaway: if the open conversation is something else (the