// Flutter-web page actions over one DevTools page session: open at a size, // enable semantics, find targets by semantics label / identifier / point, // click, type, screenshot and film. import { setTimeout as delay } from 'node:timers/promises'; /** Parses `label:Text`, `key:id`, `xy:10,20`, `10,20` or bare `Text` (a label). */ export function parseTarget(target) { if (typeof target === 'object') return target; const s = String(target); if (/^\d+(\.\d+)?,\d+(\.\d+)?$/.test(s)) return { xy: s.split(',').map(Number) }; const m = s.match(/^(label|key|xy):(.*)$/s); if (!m) return { label: s }; return m[1] === 'xy' ? { xy: m[2].split(',').map(Number) } : { [m[1]]: m[2] }; } export const describe = (t) => t.key ? `key ${t.key}` : t.xy ? `point ${t.xy}` : `label ${JSON.stringify(t.label)}`; // In-page lookup. Labels: aria-label or a node's own text; an exact match beats // a prefix match, then the smallest visible node wins (the most specific one). const FIND = `(t) => { const vis = (el) => { const r = el.getBoundingClientRect(); return r.width > 0 && r.height > 0 && r.bottom > 0 && r.right > 0 && r.top < innerHeight && r.left < innerWidth ? r : null; }; const textOf = (el) => (el.getAttribute('aria-label') || el.innerText || el.value || '').trim(); let nodes = []; if (t.key) nodes = [...document.querySelectorAll('[flt-semantics-identifier="' + CSS.escape(t.key) + '"], #' + CSS.escape(t.key))]; else { const all = [...document.querySelectorAll('flt-semantics, [aria-label], input, textarea')]; nodes = all.filter((el) => textOf(el) === t.label); if (!nodes.length) nodes = all.filter((el) => textOf(el).startsWith(t.label)); } const hits = nodes.map((el) => ({ el, r: vis(el) })).filter((h) => h.r) .sort((a, b) => a.r.width * a.r.height - b.r.width * b.r.height); if (!hits.length) return null; const r = hits[0].r; return { x: r.left + r.width / 2, y: r.top + r.height / 2, text: textOf(hits[0].el).slice(0, 80) }; }`; const LABELS = `(() => [...document.querySelectorAll('flt-semantics, [aria-label], input, textarea')] .map((el) => { const r = el.getBoundingClientRect(); return { label: (el.getAttribute('aria-label') || (el.children.length ? '' : el.innerText) || '').trim(), key: el.getAttribute('flt-semantics-identifier') || undefined, role: el.getAttribute('role') || el.tagName.toLowerCase(), rect: [r.left, r.top, r.width, r.height].map(Math.round) }; }) .filter((n) => (n.label || n.key) && n.rect[2] > 0 && n.rect[3] > 0))()`; export function flutterPage(browser, log) { const { send } = browser; let viewport = { width: 880, height: 1000 }; const errors = []; browser.on('Runtime.exceptionThrown', (p) => errors.push(p.exceptionDetails?.exception?.description ?? p.exceptionDetails?.text)); browser.on('Runtime.consoleAPICalled', (p) => { if (p.type === 'error') errors.push(p.args.map((a) => a.value ?? a.description).join(' ')); }); const evaluate = async (expression) => { const r = await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }); if (r.exceptionDetails) throw new Error(`page: ${r.exceptionDetails.exception?.description ?? r.exceptionDetails.text}`); return r.result.value; }; const until = async (fn, what, timeout) => { const end = Date.now() + timeout; for (;;) { const v = await fn(); if (v) return v; if (Date.now() > end) throw new Error(`Timed out after ${timeout} ms waiting for ${what}`); await delay(100); } }; const resize = async (width, height) => { viewport = { width, height }; if (width < 500) log(`warning: headless Chrome misrenders Flutter below ~500 px (asked ${width}); use 560 for phone`); await send('Emulation.setDeviceMetricsOverride', { width, height, deviceScaleFactor: 1, mobile: false }); }; const locate = async (target, timeout = 5000) => { const t = parseTarget(target); if (t.xy) return { x: t.xy[0], y: t.xy[1], text: '' }; try { return await until(() => evaluate(`(${FIND})(${JSON.stringify(t)})`), describe(t), timeout); } catch (error) { const known = (await evaluate(LABELS)).map((n) => n.key ? `${n.label} [key ${n.key}]` : n.label); throw new Error(`${error.message}. Visible labels: ${JSON.stringify(known)}`); } }; const tap = async ({ x, y }) => { for (const type of ['mouseMoved', 'mousePressed', 'mouseReleased']) { await send('Input.dispatchMouseEvent', { type, x, y, button: 'left', clickCount: 1 }); } }; const png = async () => Buffer.from((await send('Page.captureScreenshot', { format: 'png' })).data, 'base64'); return { errors, resize, labels: () => evaluate(LABELS), shot: png, async open(url, { width, height, theme }) { await send('Page.enable'); await send('Runtime.enable'); await send('Page.bringToFront'); await resize(width, height); await send('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-color-scheme', value: theme ?? '' }] }); await send('Page.navigate', { url }); await until(() => evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))"), 'Flutter to start', 30000); await evaluate("document.querySelector('flt-semantics-placeholder')?.click()"); await until(() => evaluate("document.querySelectorAll('flt-semantics').length > 0"), 'the semantics tree', 15000); await delay(800); }, async click(target, timeout) { const point = await locate(target, timeout); await tap(point); return point; }, waitFor: (target, timeout) => locate(target, timeout), async type(text) { const focused = await evaluate(`(() => { const a = document.activeElement; if (a && (a.tagName === 'INPUT' || a.tagName === 'TEXTAREA')) return true; const el = [...document.querySelectorAll('input, textarea')].find((e) => e.checkVisibility?.() ?? true); if (!el) return false; el.focus(); return true; })()`); if (!focused) throw new Error('No text field to type into; click the field first'); await send('Input.insertText', { text }); }, async press(key) { const codes = { Enter: 13, Escape: 27, Tab: 9, Backspace: 8, ArrowDown: 40, ArrowUp: 38 }; const base = { key, code: key, windowsVirtualKeyCode: codes[key] ?? 0, text: key === 'Enter' ? '\r' : undefined }; await send('Input.dispatchKeyEvent', { type: 'keyDown', ...base }); await send('Input.dispatchKeyEvent', { type: 'keyUp', ...base, text: undefined }); }, async scroll(dy, target) { const at = target ? await locate(target) : { x: viewport.width / 2, y: viewport.height / 2 }; await send('Input.dispatchMouseEvent', { type: 'mouseWheel', x: at.x, y: at.y, deltaX: 0, deltaY: dy }); }, /** Screencasts [ms] after clicking [target]; frames are `{ data, ms }` relative to the click. */ async film(target, ms) { const point = await locate(target); const frames = []; const ack = (p) => { frames.push({ data: p.data, t: p.metadata.timestamp }); send('Page.screencastFrameAck', { sessionId: p.sessionId }).catch(() => {}); }; const off = browser.on('Page.screencastFrame', ack); await send('Page.startScreencast', { format: 'png', everyNthFrame: 1, maxWidth: viewport.width, maxHeight: viewport.height }); await delay(300); const clickAt = Date.now() / 1000; await tap(point); await delay(ms); await send('Page.stopScreencast'); await delay(100); off(); const kept = frames.filter((f) => f.t >= clickAt - 0.05); // A still page sends no frames; the last pre-click frame is the start state. const before = frames.filter((f) => f.t < clickAt - 0.05).at(-1); return [...(before ? [before] : []), ...kept].map((f) => ({ data: f.data, ms: Math.round((f.t - clickAt) * 1000) })); }, }; }