228 lines
12 KiB
JavaScript
228 lines
12 KiB
JavaScript
// Flutter-web page actions over one DevTools page session: open at a size,
|
|
// enable semantics, find targets by semantics label / identifier / point,
|
|
// click, type, screenshot and film.
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { enableTouch, pinch, swipe } from './touch.mjs';
|
|
|
|
/** Parses `label:Text`, `key:id`, `xy:10,20`, `10,20` or bare `Text` (a label). */
|
|
export function parseTarget(target) {
|
|
if (typeof target === 'object') return target;
|
|
const s = String(target);
|
|
if (/^\d+(\.\d+)?,\d+(\.\d+)?$/.test(s)) return { xy: s.split(',').map(Number) };
|
|
const m = s.match(/^(label|key|xy|eval):(.*)$/s);
|
|
if (!m) return { label: s };
|
|
if (m[1] === 'eval') return { eval: m[2] };
|
|
return m[1] === 'xy' ? { xy: m[2].split(',').map(Number) } : { [m[1]]: m[2] };
|
|
}
|
|
|
|
export const describe = (t) => t.key ? `key ${t.key}` : t.xy ? `point ${t.xy}` : t.eval ? `eval ${t.eval}` : `label ${JSON.stringify(t.label)}`;
|
|
|
|
// In-page lookup. Labels: aria-label or a node's own text; an exact match beats
|
|
// a prefix match, then the smallest visible node wins (the most specific one).
|
|
const FIND = `(t) => {
|
|
const vis = (el) => { const r = el.getBoundingClientRect();
|
|
return r.width > 0 && r.height > 0 && r.bottom > 0 && r.right > 0
|
|
&& r.top < innerHeight && r.left < innerWidth ? r : null; };
|
|
const textOf = (el) => (el.getAttribute('aria-label') || el.innerText || el.value || '').trim();
|
|
let nodes = [];
|
|
if (t.key) nodes = [...document.querySelectorAll('[flt-semantics-identifier="' + CSS.escape(t.key) + '"], #' + CSS.escape(t.key))];
|
|
else {
|
|
const all = [...document.querySelectorAll('flt-semantics, [aria-label], input, textarea')];
|
|
nodes = all.filter((el) => textOf(el) === t.label);
|
|
if (!nodes.length) nodes = all.filter((el) => textOf(el).startsWith(t.label));
|
|
}
|
|
const hits = nodes.map((el) => ({ el, r: vis(el) })).filter((h) => h.r)
|
|
.sort((a, b) => a.r.width * a.r.height - b.r.width * b.r.height);
|
|
if (!hits.length) return null;
|
|
const r = hits[0].r;
|
|
return { x: r.left + r.width / 2, y: r.top + r.height / 2, text: textOf(hits[0].el).slice(0, 80) };
|
|
}`;
|
|
|
|
const LABELS = `(() => [...document.querySelectorAll('flt-semantics, [aria-label], input, textarea')]
|
|
.map((el) => { const r = el.getBoundingClientRect();
|
|
return { label: (el.getAttribute('aria-label') || (el.children.length ? '' : el.innerText) || '').trim(),
|
|
key: el.getAttribute('flt-semantics-identifier') || undefined,
|
|
role: el.getAttribute('role') || el.tagName.toLowerCase(),
|
|
rect: [r.left, r.top, r.width, r.height].map(Math.round) }; })
|
|
.filter((n) => (n.label || n.key) && n.rect[2] > 0 && n.rect[3] > 0))()`;
|
|
|
|
export function flutterPage(browser, log, { touch = false } = {}) {
|
|
const { send } = browser;
|
|
let viewport = { width: 880, height: 1000 };
|
|
const errors = [];
|
|
browser.on('Runtime.exceptionThrown', (p) => errors.push(p.exceptionDetails?.exception?.description ?? p.exceptionDetails?.text));
|
|
browser.on('Runtime.consoleAPICalled', (p) => {
|
|
if (p.type === 'error') errors.push(p.args.map((a) => a.value ?? a.description).join(' '));
|
|
});
|
|
|
|
const evaluate = async (expression) => {
|
|
const r = await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true });
|
|
if (r.exceptionDetails) throw new Error(`page: ${r.exceptionDetails.exception?.description ?? r.exceptionDetails.text}`);
|
|
return r.result.value;
|
|
};
|
|
const until = async (fn, what, timeout) => {
|
|
const end = Date.now() + timeout;
|
|
for (;;) {
|
|
const v = await fn();
|
|
if (v) return v;
|
|
if (Date.now() > end) throw new Error(`Timed out after ${timeout} ms waiting for ${what}`);
|
|
await delay(100);
|
|
}
|
|
};
|
|
const resize = async (width, height) => {
|
|
viewport = { width, height };
|
|
if (width < 500) log(`warning: headless Chrome misrenders Flutter below ~500 px (asked ${width}); use 560 for phone`);
|
|
await send('Emulation.setDeviceMetricsOverride', { width, height, deviceScaleFactor: 1, mobile: false });
|
|
};
|
|
const locate = async (target, timeout = 5000) => {
|
|
const t = parseTarget(target);
|
|
if (t.xy) return { x: t.xy[0], y: t.xy[1], text: '' };
|
|
if (t.eval) {
|
|
// An expression yielding [x, y] (or null while the target is absent).
|
|
const point = await until(() => evaluate(t.eval), describe(t), timeout);
|
|
return { x: point[0], y: point[1], text: '' };
|
|
}
|
|
try {
|
|
return await until(() => evaluate(`(${FIND})(${JSON.stringify(t)})`), describe(t), timeout);
|
|
} catch (error) {
|
|
const known = (await evaluate(LABELS)).map((n) => n.key ? `${n.label} [key ${n.key}]` : n.label);
|
|
throw new Error(`${error.message}. Visible labels: ${JSON.stringify(known)}`);
|
|
}
|
|
};
|
|
const tap = async ({ x, y }) => {
|
|
for (const type of ['mouseMoved', 'mousePressed', 'mouseReleased']) {
|
|
await send('Input.dispatchMouseEvent', { type, x, y, button: 'left', clickCount: 1 });
|
|
}
|
|
};
|
|
const png = async () => Buffer.from((await send('Page.captureScreenshot', { format: 'png' })).data, 'base64');
|
|
|
|
return {
|
|
errors,
|
|
resize,
|
|
labels: () => evaluate(LABELS),
|
|
shot: png,
|
|
async open(url, { width, height, theme }) {
|
|
await send('Page.enable');
|
|
await send('Runtime.enable');
|
|
await send('Page.bringToFront');
|
|
await resize(width, height);
|
|
await send('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-color-scheme', value: theme ?? '' }] });
|
|
await send('Page.navigate', { url });
|
|
await until(() => evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))"), 'Flutter to start', 30000);
|
|
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
|
|
await until(() => evaluate("document.querySelectorAll('flt-semantics').length > 0"), 'the semantics tree', 15000);
|
|
// Touch starts once semantics is on: the placeholder's click does not
|
|
// turn semantics on under touch emulation.
|
|
if (touch) await enableTouch(send, viewport);
|
|
await delay(800);
|
|
},
|
|
async click(target, timeout) {
|
|
const point = await locate(target, timeout);
|
|
await tap(point);
|
|
return point;
|
|
},
|
|
waitFor: (target, timeout) => locate(target, timeout),
|
|
/** Answers the next file chooser the page opens with [paths] (absolute). */
|
|
async armFiles(paths) {
|
|
await send('Page.setInterceptFileChooserDialog', { enabled: true });
|
|
const off = browser.on('Page.fileChooserOpened', (p) => {
|
|
off();
|
|
send('DOM.setFileInputFiles', { files: paths, backendNodeId: p.backendNodeId })
|
|
.then(() => log(` chose ${paths.join(', ')}`))
|
|
.catch((e) => errors.push(`files: ${e.message}`))
|
|
.finally(() => send('Page.setInterceptFileChooserDialog', { enabled: false }).catch(() => {}));
|
|
});
|
|
},
|
|
async type(text) {
|
|
const focused = await evaluate(`(() => {
|
|
const a = document.activeElement;
|
|
if (a && (a.tagName === 'INPUT' || a.tagName === 'TEXTAREA')) return true;
|
|
const el = [...document.querySelectorAll('input, textarea')].find((e) => e.checkVisibility?.() ?? true);
|
|
if (!el) return false; el.focus(); return true; })()`);
|
|
if (!focused) throw new Error('No text field to type into; click the field first');
|
|
await send('Input.insertText', { text });
|
|
},
|
|
/** Presses [chord]: a key, optionally after `Control+`, `Shift+`, `Alt+` or `Meta+` (e.g. `Control+2`). */
|
|
async press(chord) {
|
|
const parts = String(chord).split('+');
|
|
const key = parts.pop();
|
|
const bits = { Alt: 1, Control: 2, Meta: 4, Shift: 8 };
|
|
const modifiers = parts.reduce((m, p) => m | (bits[p] ?? 0), 0);
|
|
const codes = { Enter: 13, Escape: 27, Tab: 9, Backspace: 8, ArrowDown: 40, ArrowUp: 38, ArrowLeft: 37, ArrowRight: 39, "/": 191 };
|
|
const digit = /^\d$/.test(key);
|
|
const code = key === '/' ? 'Slash' : digit ? `Digit${key}` : /^[a-z]$/i.test(key) ? `Key${key.toUpperCase()}` : key;
|
|
const vk = codes[key] ?? (key.length === 1 ? key.toUpperCase().charCodeAt(0) : 0);
|
|
const text = key === 'Enter' ? '\r' : key.length === 1 && !modifiers ? key : undefined;
|
|
const base = { key, code, windowsVirtualKeyCode: vk, modifiers, text };
|
|
for (const p of parts) {
|
|
await send('Input.dispatchKeyEvent', { type: 'keyDown', key: p, code: `${p}Left`, modifiers });
|
|
}
|
|
await send('Input.dispatchKeyEvent', { type: 'keyDown', ...base });
|
|
await send('Input.dispatchKeyEvent', { type: 'keyUp', ...base, text: undefined });
|
|
for (const p of parts.reverse()) {
|
|
await send('Input.dispatchKeyEvent', { type: 'keyUp', key: p, code: `${p}Left`, modifiers: 0 });
|
|
}
|
|
},
|
|
/** Wheel ticks at [target] (default the centre): [repeat] events of dx/dy each, [gap] ms apart. */
|
|
async scroll(dy, target, { dx = 0, repeat = 1, gap = 16 } = {}) {
|
|
const at = target ? await locate(target) : { x: viewport.width / 2, y: viewport.height / 2 };
|
|
await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: at.x, y: at.y });
|
|
for (let i = 0; i < repeat; i++) {
|
|
if (i) await delay(gap);
|
|
await send('Input.dispatchMouseEvent', { type: 'mouseWheel', x: at.x, y: at.y, deltaX: dx, deltaY: dy });
|
|
}
|
|
return at;
|
|
},
|
|
/** Moves the mouse onto [target] without pressing (hover states, cursors). */
|
|
async hover(target, timeout) {
|
|
const at = await locate(target, timeout);
|
|
await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: at.x, y: at.y });
|
|
return at;
|
|
},
|
|
/**
|
|
* Presses the mouse on [target], moves it by dx/dy in [steps] moves [gap] ms
|
|
* apart, and releases it unless [hold] (to capture mid-drag; the next
|
|
* mouse step or the end of the run releases it).
|
|
*/
|
|
async drag(target, { dx = 0, dy = 0, steps = 12, gap = 16, hold = false } = {}) {
|
|
const at = await locate(target);
|
|
const mouse = (type, x, y, extra = {}) => send('Input.dispatchMouseEvent', { type, x, y, button: 'left', ...extra });
|
|
await mouse('mouseMoved', at.x, at.y, { button: 'none' });
|
|
await mouse('mousePressed', at.x, at.y, { buttons: 1, clickCount: 1 });
|
|
for (let i = 1; i <= steps; i++) {
|
|
await delay(gap);
|
|
await mouse('mouseMoved', at.x + (dx * i) / steps, at.y + (dy * i) / steps, { buttons: 1 });
|
|
}
|
|
if (!hold) await mouse('mouseReleased', at.x + dx, at.y + dy, { buttons: 0, clickCount: 1 });
|
|
return { from: at, to: { x: at.x + dx, y: at.y + dy } };
|
|
},
|
|
/** One-finger touch swipe from [target] by dx/dy (needs `--touch true`). */
|
|
swipe: async (target, moves) => swipe(send, await locate(target), moves),
|
|
/** Two-finger touch pinch at [target] by [scale] (needs `--touch true`). */
|
|
pinch: async (target, scale) => pinch(send, await locate(target), scale),
|
|
evaluate,
|
|
/** Screencasts [ms] after clicking [target]; frames are `{ data, ms }` relative to the click. */
|
|
async film(target, ms) {
|
|
const point = await locate(target);
|
|
const frames = [];
|
|
const ack = (p) => {
|
|
frames.push({ data: p.data, t: p.metadata.timestamp });
|
|
send('Page.screencastFrameAck', { sessionId: p.sessionId }).catch(() => {});
|
|
};
|
|
const off = browser.on('Page.screencastFrame', ack);
|
|
await send('Page.startScreencast', { format: 'png', everyNthFrame: 1, maxWidth: viewport.width, maxHeight: viewport.height });
|
|
await delay(300);
|
|
const clickAt = Date.now() / 1000;
|
|
await tap(point);
|
|
await delay(ms);
|
|
await send('Page.stopScreencast');
|
|
await delay(100);
|
|
off();
|
|
const kept = frames.filter((f) => f.t >= clickAt - 0.05);
|
|
// A still page sends no frames; the last pre-click frame is the start state.
|
|
const before = frames.filter((f) => f.t < clickAt - 0.05).at(-1);
|
|
return [...(before ? [before] : []), ...kept].map((f) => ({ data: f.data, ms: Math.round((f.t - clickAt) * 1000) }));
|
|
},
|
|
};
|
|
}
|