Files
pipelines/app/tool/capture/page.mjs
T
alan d6f636a325 Chat: Ctrl+1-9 puts the keyboard in the opened chat's composer
A chat opened from the keyboard asks its composer for focus; the composer
takes it after the frame it is built in (new detail pane or phone page), or
at once when that chat is already open. Taps and other opens cancel the
request, so they keep focus where it was. The capture kit's press step
accepts modifier chords such as Control+2.
2026-11-03 12:42:28 +00:00

209 lines
10 KiB
JavaScript

// Flutter-web page actions over one DevTools page session: open at a size,
// enable semantics, find targets by semantics label / identifier / point,
// click, type, screenshot and film.
import { setTimeout as delay } from 'node:timers/promises';
/** Parses `label:Text`, `key:id`, `xy:10,20`, `10,20` or bare `Text` (a label). */
export function parseTarget(target) {
if (typeof target === 'object') return target;
const s = String(target);
if (/^\d+(\.\d+)?,\d+(\.\d+)?$/.test(s)) return { xy: s.split(',').map(Number) };
const m = s.match(/^(label|key|xy|eval):(.*)$/s);
if (!m) return { label: s };
if (m[1] === 'eval') return { eval: m[2] };
return m[1] === 'xy' ? { xy: m[2].split(',').map(Number) } : { [m[1]]: m[2] };
}
export const describe = (t) => t.key ? `key ${t.key}` : t.xy ? `point ${t.xy}` : t.eval ? `eval ${t.eval}` : `label ${JSON.stringify(t.label)}`;
// In-page lookup. Labels: aria-label or a node's own text; an exact match beats
// a prefix match, then the smallest visible node wins (the most specific one).
const FIND = `(t) => {
const vis = (el) => { const r = el.getBoundingClientRect();
return r.width > 0 && r.height > 0 && r.bottom > 0 && r.right > 0
&& r.top < innerHeight && r.left < innerWidth ? r : null; };
const textOf = (el) => (el.getAttribute('aria-label') || el.innerText || el.value || '').trim();
let nodes = [];
if (t.key) nodes = [...document.querySelectorAll('[flt-semantics-identifier="' + CSS.escape(t.key) + '"], #' + CSS.escape(t.key))];
else {
const all = [...document.querySelectorAll('flt-semantics, [aria-label], input, textarea')];
nodes = all.filter((el) => textOf(el) === t.label);
if (!nodes.length) nodes = all.filter((el) => textOf(el).startsWith(t.label));
}
const hits = nodes.map((el) => ({ el, r: vis(el) })).filter((h) => h.r)
.sort((a, b) => a.r.width * a.r.height - b.r.width * b.r.height);
if (!hits.length) return null;
const r = hits[0].r;
return { x: r.left + r.width / 2, y: r.top + r.height / 2, text: textOf(hits[0].el).slice(0, 80) };
}`;
const LABELS = `(() => [...document.querySelectorAll('flt-semantics, [aria-label], input, textarea')]
.map((el) => { const r = el.getBoundingClientRect();
return { label: (el.getAttribute('aria-label') || (el.children.length ? '' : el.innerText) || '').trim(),
key: el.getAttribute('flt-semantics-identifier') || undefined,
role: el.getAttribute('role') || el.tagName.toLowerCase(),
rect: [r.left, r.top, r.width, r.height].map(Math.round) }; })
.filter((n) => (n.label || n.key) && n.rect[2] > 0 && n.rect[3] > 0))()`;
export function flutterPage(browser, log) {
const { send } = browser;
let viewport = { width: 880, height: 1000 };
const errors = [];
browser.on('Runtime.exceptionThrown', (p) => errors.push(p.exceptionDetails?.exception?.description ?? p.exceptionDetails?.text));
browser.on('Runtime.consoleAPICalled', (p) => {
if (p.type === 'error') errors.push(p.args.map((a) => a.value ?? a.description).join(' '));
});
const evaluate = async (expression) => {
const r = await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true });
if (r.exceptionDetails) throw new Error(`page: ${r.exceptionDetails.exception?.description ?? r.exceptionDetails.text}`);
return r.result.value;
};
const until = async (fn, what, timeout) => {
const end = Date.now() + timeout;
for (;;) {
const v = await fn();
if (v) return v;
if (Date.now() > end) throw new Error(`Timed out after ${timeout} ms waiting for ${what}`);
await delay(100);
}
};
const resize = async (width, height) => {
viewport = { width, height };
if (width < 500) log(`warning: headless Chrome misrenders Flutter below ~500 px (asked ${width}); use 560 for phone`);
await send('Emulation.setDeviceMetricsOverride', { width, height, deviceScaleFactor: 1, mobile: false });
};
const locate = async (target, timeout = 5000) => {
const t = parseTarget(target);
if (t.xy) return { x: t.xy[0], y: t.xy[1], text: '' };
if (t.eval) {
// An expression yielding [x, y] (or null while the target is absent).
const point = await until(() => evaluate(t.eval), describe(t), timeout);
return { x: point[0], y: point[1], text: '' };
}
try {
return await until(() => evaluate(`(${FIND})(${JSON.stringify(t)})`), describe(t), timeout);
} catch (error) {
const known = (await evaluate(LABELS)).map((n) => n.key ? `${n.label} [key ${n.key}]` : n.label);
throw new Error(`${error.message}. Visible labels: ${JSON.stringify(known)}`);
}
};
const tap = async ({ x, y }) => {
for (const type of ['mouseMoved', 'mousePressed', 'mouseReleased']) {
await send('Input.dispatchMouseEvent', { type, x, y, button: 'left', clickCount: 1 });
}
};
const png = async () => Buffer.from((await send('Page.captureScreenshot', { format: 'png' })).data, 'base64');
return {
errors,
resize,
labels: () => evaluate(LABELS),
shot: png,
async open(url, { width, height, theme }) {
await send('Page.enable');
await send('Runtime.enable');
await send('Page.bringToFront');
await resize(width, height);
await send('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-color-scheme', value: theme ?? '' }] });
await send('Page.navigate', { url });
await until(() => evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))"), 'Flutter to start', 30000);
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
await until(() => evaluate("document.querySelectorAll('flt-semantics').length > 0"), 'the semantics tree', 15000);
await delay(800);
},
async click(target, timeout) {
const point = await locate(target, timeout);
await tap(point);
return point;
},
waitFor: (target, timeout) => locate(target, timeout),
async type(text) {
const focused = await evaluate(`(() => {
const a = document.activeElement;
if (a && (a.tagName === 'INPUT' || a.tagName === 'TEXTAREA')) return true;
const el = [...document.querySelectorAll('input, textarea')].find((e) => e.checkVisibility?.() ?? true);
if (!el) return false; el.focus(); return true; })()`);
if (!focused) throw new Error('No text field to type into; click the field first');
await send('Input.insertText', { text });
},
/** Presses [chord]: a key, optionally after `Control+`, `Shift+`, `Alt+` or `Meta+` (e.g. `Control+2`). */
async press(chord) {
const parts = String(chord).split('+');
const key = parts.pop();
const bits = { Alt: 1, Control: 2, Meta: 4, Shift: 8 };
const modifiers = parts.reduce((m, p) => m | (bits[p] ?? 0), 0);
const codes = { Enter: 13, Escape: 27, Tab: 9, Backspace: 8, ArrowDown: 40, ArrowUp: 38 };
const digit = /^\d$/.test(key);
const code = digit ? `Digit${key}` : /^[a-z]$/i.test(key) ? `Key${key.toUpperCase()}` : key;
const vk = codes[key] ?? (key.length === 1 ? key.toUpperCase().charCodeAt(0) : 0);
const text = key === 'Enter' ? '\r' : key.length === 1 && !modifiers ? key : undefined;
const base = { key, code, windowsVirtualKeyCode: vk, modifiers, text };
for (const p of parts) {
await send('Input.dispatchKeyEvent', { type: 'keyDown', key: p, code: `${p}Left`, modifiers });
}
await send('Input.dispatchKeyEvent', { type: 'keyDown', ...base });
await send('Input.dispatchKeyEvent', { type: 'keyUp', ...base, text: undefined });
for (const p of parts.reverse()) {
await send('Input.dispatchKeyEvent', { type: 'keyUp', key: p, code: `${p}Left`, modifiers: 0 });
}
},
/** Wheel ticks at [target] (default the centre): [repeat] events of dx/dy each, [gap] ms apart. */
async scroll(dy, target, { dx = 0, repeat = 1, gap = 16 } = {}) {
const at = target ? await locate(target) : { x: viewport.width / 2, y: viewport.height / 2 };
await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: at.x, y: at.y });
for (let i = 0; i < repeat; i++) {
if (i) await delay(gap);
await send('Input.dispatchMouseEvent', { type: 'mouseWheel', x: at.x, y: at.y, deltaX: dx, deltaY: dy });
}
return at;
},
/** Moves the mouse onto [target] without pressing (hover states, cursors). */
async hover(target, timeout) {
const at = await locate(target, timeout);
await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: at.x, y: at.y });
return at;
},
/**
* Presses the mouse on [target], moves it by dx/dy in [steps] moves [gap] ms
* apart, and releases it unless [hold] (to capture mid-drag; the next
* mouse step or the end of the run releases it).
*/
async drag(target, { dx = 0, dy = 0, steps = 12, gap = 16, hold = false } = {}) {
const at = await locate(target);
const mouse = (type, x, y, extra = {}) => send('Input.dispatchMouseEvent', { type, x, y, button: 'left', ...extra });
await mouse('mouseMoved', at.x, at.y, { button: 'none' });
await mouse('mousePressed', at.x, at.y, { buttons: 1, clickCount: 1 });
for (let i = 1; i <= steps; i++) {
await delay(gap);
await mouse('mouseMoved', at.x + (dx * i) / steps, at.y + (dy * i) / steps, { buttons: 1 });
}
if (!hold) await mouse('mouseReleased', at.x + dx, at.y + dy, { buttons: 0, clickCount: 1 });
return { from: at, to: { x: at.x + dx, y: at.y + dy } };
},
evaluate,
/** Screencasts [ms] after clicking [target]; frames are `{ data, ms }` relative to the click. */
async film(target, ms) {
const point = await locate(target);
const frames = [];
const ack = (p) => {
frames.push({ data: p.data, t: p.metadata.timestamp });
send('Page.screencastFrameAck', { sessionId: p.sessionId }).catch(() => {});
};
const off = browser.on('Page.screencastFrame', ack);
await send('Page.startScreencast', { format: 'png', everyNthFrame: 1, maxWidth: viewport.width, maxHeight: viewport.height });
await delay(300);
const clickAt = Date.now() / 1000;
await tap(point);
await delay(ms);
await send('Page.stopScreencast');
await delay(100);
off();
const kept = frames.filter((f) => f.t >= clickAt - 0.05);
// A still page sends no frames; the last pre-click frame is the start state.
const before = frames.filter((f) => f.t < clickAt - 0.05).at(-1);
return [...(before ? [before] : []), ...kept].map((f) => ({ data: f.data, ms: Math.round((f.t - clickAt) * 1000) }));
},
};
}