memwatch: quiet bounded test summary off a terminal; bounded analyze wrapper
run.sh now records flutter's JSON results beside the log and prints a summary from tool/memwatch/summary.dart: one verdict line (pass/fail/skip counts, peak MB, duration, exit status), only failing tests with their error and app stack frames (framework frames dropped, per-test and total caps, tail of printed output), the tests still running when killed, and the log and JSON paths. Quiet is the default when stdout is not a terminal; MEMWATCH_OUTPUT=full also prints the whole log. The watchdog loop and exit status are unchanged. tool/analyze.sh bounds flutter analyze to a verdict line and the first 40 issues. Tests: test/tool/memwatch/summary_test.dart and the end-to-end tool/memwatch/quiet_test.sh over fixture suites.
This commit is contained in:
+1
-1
@@ -18,7 +18,7 @@
|
||||
- Tests mirror directories under `test/`; tests use local fixtures/fake gateways, never mutate production. Put item evidence in `notes/<item>.md`, not shared queue.
|
||||
|
||||
## Tools and verification
|
||||
Flutter: `/Users/alan/development/flutter/bin/flutter`; Dart beside it. Run pub get per worktree; never upgrade shared SDK. Run tests only through the memory watchdog, from `app/`: `tool/memwatch/full.sh test` (or a focused `tool/memwatch/full.sh test/<area>` while editing). Never run a bare `flutter test`, because a runaway test has crashed the Mac. `tool/memwatch/install.sh` restores the legacy `/tmp/memwatch` shims. Use `flutter analyze` and `flutter build web` for integrated acceptance. No Go/legacy client checks needed for app-only changes.
|
||||
Flutter: `/Users/alan/development/flutter/bin/flutter`; Dart beside it. Run pub get per worktree; never upgrade shared SDK. Run tests only through the memory watchdog, from `app/`: `tool/memwatch/full.sh test` (or a focused `tool/memwatch/full.sh test/<area>` while editing). Never run a bare `flutter test`, because a runaway test has crashed the Mac. Off a terminal it is quiet: one verdict line (counts, peak MB, duration, exit), only failing tests with bounded errors, and the full log path; read that log with grep or a line range rather than whole, or set `MEMWATCH_OUTPUT=full`. `tool/memwatch/quiet_test.sh` checks this output. `tool/memwatch/install.sh` restores the legacy `/tmp/memwatch` shims. Use `tool/analyze.sh` (bounded `flutter analyze`) and `flutter build web` for integrated acceptance. No Go/legacy client checks needed for app-only changes.
|
||||
Rendered changes require real browser interaction/captures; widget tests alone are not browser evidence. Use isolated loopback servers and OS-assigned ports. Browser automation may use Chrome DevTools, with independent scripts under `app/tool/`; do not modify legacy browser drivers. Never disable authentication on exposed services.
|
||||
UI capture and review: use `tool/capture/` (see its README). It is one generic Chrome DevTools kit over the full-app fixture `tool/search_selection_probe_entry.dart`. It opens any route at a given size, platform and theme, clicks by semantics label, identifier or point, types text, takes screenshots and films transitions. If a feature's legacy `tool/*_probe.mjs` driver is stale (missing labels, outdated status text), capture with this kit; a stale driver is not a reason to report blocked. Inspect every PNG you cite.
|
||||
Keep commits referenced; do not discard unrelated work or delete branches. Deploy only when requested by the user; preserve existing public URLs unless a cutover is part of that request. Land candidates per selected pipeline; manager verifies actual ancestry and records source, checks, landing and deployment separately.
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
|
||||
import '../../../tool/memwatch/summary.dart';
|
||||
|
||||
const _root = '/work/app';
|
||||
|
||||
String _event(Map<String, Object?> event) => jsonEncode(event);
|
||||
|
||||
List<String> _run({List<Map<String, Object?>> tests = const [], List<String> extra = const []}) => [
|
||||
_event({'type': 'suite', 'suite': {'id': 0, 'path': '$_root/test/a_test.dart'}}),
|
||||
for (final t in tests) ...[
|
||||
_event({'type': 'testStart', 'test': {'id': t['id'], 'name': t['name'], 'suiteID': 0}}),
|
||||
for (final p in (t['prints'] as List<String>? ?? const []))
|
||||
_event({'type': 'print', 'testID': t['id'], 'message': p}),
|
||||
if (t['error'] != null)
|
||||
_event({'type': 'error', 'testID': t['id'], 'error': t['error'], 'stackTrace': t['stack'] ?? ''}),
|
||||
if (t['done'] != false)
|
||||
_event({
|
||||
'type': 'testDone',
|
||||
'testID': t['id'],
|
||||
'result': t['error'] == null ? 'success' : 'failure',
|
||||
'skipped': t['skipped'] ?? false,
|
||||
'hidden': t['hidden'] ?? false,
|
||||
}),
|
||||
],
|
||||
...extra,
|
||||
];
|
||||
|
||||
String _summary(List<String> events, {int status = 1, List<String> logTail = const []}) => summarize(
|
||||
events,
|
||||
status: status,
|
||||
peakMb: 812,
|
||||
seconds: 9,
|
||||
target: 'test/a',
|
||||
logPath: '/tmp/memwatch/out.x',
|
||||
jsonPath: '/tmp/memwatch/out.x.json',
|
||||
logTail: logTail,
|
||||
appPackage: 'pipelines_app',
|
||||
appRoot: _root,
|
||||
);
|
||||
|
||||
void main() {
|
||||
test('a passing run is the verdict line and the log paths only', () {
|
||||
final summary = _summary(
|
||||
_run(tests: [
|
||||
{'id': 1, 'name': 'loading $_root/test/a_test.dart', 'hidden': true},
|
||||
{'id': 2, 'name': 'ok', 'prints': ['noise']},
|
||||
{'id': 3, 'name': 'skip me', 'skipped': true},
|
||||
]),
|
||||
status: 0,
|
||||
);
|
||||
expect(summary.split('\n'), [
|
||||
'memwatch: PASS: 1 passed, 0 failed, 1 skipped; peak 812 MB; 9s; exit 0 (test/a)',
|
||||
'full output: /tmp/memwatch/out.x',
|
||||
'json results: /tmp/memwatch/out.x.json',
|
||||
]);
|
||||
});
|
||||
|
||||
test('a failure shows its error and app frames, not framework frames', () {
|
||||
final summary = _summary(_run(tests: [
|
||||
{
|
||||
'id': 2,
|
||||
'name': 'group adds',
|
||||
'error': "Expected: <2>\n Actual: <3>",
|
||||
'stack': 'package:matcher expect\n'
|
||||
'package:flutter_test/src/widget_tester.dart 473:18 expect\n'
|
||||
'test/a_test.dart 11:5 main.<fn>\n',
|
||||
},
|
||||
]));
|
||||
expect(summary, contains('memwatch: FAIL: 0 passed, 1 failed, 0 skipped'));
|
||||
expect(summary, contains('FAILED test/a_test.dart: group adds'));
|
||||
expect(summary, contains('Actual: <3>'));
|
||||
expect(summary, contains('test/a_test.dart 11:5'));
|
||||
expect(summary, isNot(contains('package:matcher')));
|
||||
expect(summary, isNot(contains('widget_tester.dart')));
|
||||
});
|
||||
|
||||
test('a flooding failure is bounded and keeps the exception before prints', () {
|
||||
final flood = [for (var i = 0; i < 5000; i++) 'flood $i ${'y' * 500}'];
|
||||
final exception = [
|
||||
'══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞══',
|
||||
'The following TestFailure was thrown running a test:',
|
||||
'Expected: exactly one matching candidate',
|
||||
for (var i = 0; i < 200; i++) '#$i frame (package:flutter/src/widgets/framework.dart:$i:1)',
|
||||
'#201 main.<anonymous closure> (file://$_root/test/a_test.dart:20:5)',
|
||||
].join('\n');
|
||||
final summary = _summary(_run(tests: [
|
||||
for (var id = 2; id < 52; id++)
|
||||
{'id': id, 'name': 'fails $id', 'prints': [...flood, exception], 'error': 'Test failed. See exception logs above.'},
|
||||
]));
|
||||
expect(summary.length, lessThan(maxDetailChars + 4000));
|
||||
expect(summary, contains('Expected: exactly one matching candidate'));
|
||||
expect(summary, contains('(test/a_test.dart:20:5)'));
|
||||
expect(summary, isNot(contains('framework.dart')));
|
||||
expect(summary, contains('printed output (last 8 of 5000 lines):'));
|
||||
expect(summary, contains('... and 20 more failures (see full log)'));
|
||||
expect(summary.indexOf('EXCEPTION CAUGHT'), lessThan(summary.indexOf('printed output')));
|
||||
});
|
||||
|
||||
test('a killed run names the tests still running', () {
|
||||
final summary = _summary(
|
||||
_run(tests: [
|
||||
{'id': 2, 'name': 'finished'},
|
||||
{'id': 3, 'name': 'runaway', 'done': false},
|
||||
]),
|
||||
status: 137,
|
||||
);
|
||||
expect(summary, startsWith('memwatch: KILLED: 1 passed, 0 failed'));
|
||||
expect(summary, contains('RUNNING when killed: test/a_test.dart: runaway'));
|
||||
});
|
||||
|
||||
test('a run without results shows the bounded log tail', () {
|
||||
final summary = _summary(const [], logTail: ['Error: no test files', 'x' * 1000]);
|
||||
expect(summary, contains('no test results were recorded; last log lines:'));
|
||||
expect(summary, contains(' Error: no test files'));
|
||||
expect(summary.length, lessThan(800));
|
||||
});
|
||||
}
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
# Bounded `flutter analyze` for agents (from app/): tool/analyze.sh [paths...]
|
||||
# Prints one verdict line with error/warning/info counts and the duration, at
|
||||
# most ANALYZE_MAX_ISSUES (default 40) issue lines, and the full log path.
|
||||
# Exits with flutter analyze's own status.
|
||||
FLUTTER=${FLUTTER:-/Users/alan/development/flutter/bin/flutter}
|
||||
max=${ANALYZE_MAX_ISSUES:-40}
|
||||
[ -f pubspec.yaml ] || cd "$(dirname "$0")/.." || exit 1
|
||||
tmp=${TMPDIR:-/tmp}
|
||||
log=$(mktemp "${tmp%/}/flutter-analyze.XXXXXX") || exit 1
|
||||
start=$SECONDS
|
||||
"$FLUTTER" analyze "$@" > "$log" 2>&1; status=$?
|
||||
issues=$(grep -cE '^ *(error|warning|info) • ' "$log")
|
||||
count() { grep -cE "^ *$1 • " "$log"; }
|
||||
if [ "$status" = 0 ] && [ "$issues" = 0 ]; then
|
||||
echo "analyze: clean; $((SECONDS-start))s; exit 0"
|
||||
else
|
||||
echo "analyze: $(count error) errors, $(count warning) warnings, $(count info) infos; $((SECONDS-start))s; exit $status"
|
||||
if [ "$issues" = 0 ]; then
|
||||
tail -20 "$log" | cut -c1-300
|
||||
else
|
||||
grep -E '^ *(error|warning|info) • ' "$log" | head -n "$max" | cut -c1-300
|
||||
[ "$issues" -gt "$max" ] && echo "... and $((issues-max)) more issues"
|
||||
fi
|
||||
fi
|
||||
echo "full output: $log"
|
||||
exit $status
|
||||
@@ -0,0 +1,22 @@
|
||||
// Fixture for tool/memwatch/quiet_test.sh: one passing test, one plain
|
||||
// expectation failure and one widget-test failure that prints far more than
|
||||
// the quiet summary may show. It lives outside test/ so the app suite skips it.
|
||||
import 'package:flutter/widgets.dart';
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
|
||||
void main() {
|
||||
test('passing neighbour', () => expect(1, 1));
|
||||
|
||||
test('plain expectation fails', () {
|
||||
expect('memwatch-actual', 'memwatch-expected');
|
||||
});
|
||||
|
||||
testWidgets('widget failure with long output', (tester) async {
|
||||
for (var i = 0; i < 2000; i++) {
|
||||
// ignore: avoid_print
|
||||
print('memwatch-flood line $i ${'x' * 60}');
|
||||
}
|
||||
await tester.pumpWidget(const Text('memwatch', textDirection: TextDirection.ltr));
|
||||
expect(find.text('memwatch-missing-widget'), findsOneWidget);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// Fixture for tool/memwatch/quiet_test.sh: a passing suite whose output must
|
||||
// not appear in quiet mode. It lives outside test/ so the app suite skips it.
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
|
||||
void main() {
|
||||
for (var i = 0; i < 3; i++) {
|
||||
test('passing fixture $i', () {
|
||||
// Printed output of a passing test is not part of the quiet summary.
|
||||
// ignore: avoid_print
|
||||
print('memwatch-noise-from-passing-test $i');
|
||||
expect(i, lessThan(3));
|
||||
});
|
||||
}
|
||||
test('skipped fixture', () {}, skip: 'fixture skip');
|
||||
}
|
||||
@@ -2,4 +2,6 @@
|
||||
# THE way to run app tests (from app/): tool/memwatch/full.sh test
|
||||
# Also for focused runs: tool/memwatch/full.sh test/features/runs
|
||||
# Concurrency 4, 12 GB cap for this run only (see run.sh beside this file).
|
||||
# Off a terminal the output is quiet: one verdict line, the failing tests only
|
||||
# (bounded), and the full log path. MEMWATCH_OUTPUT=full also prints the log.
|
||||
CONC=4 CAP_KB=12000000 exec "$(dirname "$0")/run.sh" "$@"
|
||||
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
# End-to-end check of the memwatch output modes against tiny fixture suites,
|
||||
# always under the watchdog (full.sh / run.sh). Exit 0 when every check passes.
|
||||
#
|
||||
# Usage (from app/): tool/memwatch/quiet_test.sh
|
||||
set -u
|
||||
cd "$(dirname "$0")/../.." || exit 1
|
||||
tmp=${TMPDIR:-/tmp}
|
||||
dir=$(mktemp -d "${tmp%/}/memwatch-quiet-test.XXXXXX") || exit 1
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
export MEMWATCH_DIR="$dir/logs"
|
||||
pass=tool/memwatch/fixtures/passing_fixture_test.dart
|
||||
fail=tool/memwatch/fixtures/failing_fixture_test.dart
|
||||
failures=0
|
||||
check() { # check <description> <command...>
|
||||
local what=$1; shift
|
||||
if "$@"; then echo "ok $what"; else echo "FAIL $what"; failures=$((failures+1)); fi
|
||||
}
|
||||
has() { grep -qF -- "$2" "$1"; }
|
||||
lacks() { ! grep -qF -- "$2" "$1"; }
|
||||
|
||||
# Stdout is a file here, so quiet is the default.
|
||||
tool/memwatch/full.sh "$pass" > "$dir/pass.out"; status=$?
|
||||
check "passing suite exits 0" [ $status = 0 ]
|
||||
check "passing verdict line" has "$dir/pass.out" "memwatch: PASS: 3 passed, 0 failed, 1 skipped; peak "
|
||||
check "passing output omits test prints" lacks "$dir/pass.out" "memwatch-noise"
|
||||
check "passing output is three lines" [ "$(wc -l < "$dir/pass.out")" -le 3 ]
|
||||
|
||||
tool/memwatch/full.sh "$fail" > "$dir/fail.out"; status=$?
|
||||
check "failing suite exits 1" [ $status = 1 ]
|
||||
check "failing verdict line" has "$dir/fail.out" "memwatch: FAIL: 1 passed, 2 failed, 0 skipped; peak "
|
||||
check "names the expectation failure" has "$dir/fail.out" "FAILED $fail: plain expectation fails"
|
||||
check "shows the expectation values" has "$dir/fail.out" "Actual: 'memwatch-actual'"
|
||||
check "names the widget failure" has "$dir/fail.out" "FAILED $fail: widget failure with long output"
|
||||
check "shows the widget exception" has "$dir/fail.out" "memwatch-missing-widget"
|
||||
check "shows the failing app frame" has "$dir/fail.out" "$fail:20:5"
|
||||
check "bounds the flood of prints" has "$dir/fail.out" "printed output (last 8 of 2000 lines):"
|
||||
check "quiet failure output under 6 KB" [ "$(wc -c < "$dir/fail.out")" -lt 6000 ]
|
||||
log=$(sed -n 's/^full output: //p' "$dir/fail.out")
|
||||
check "full log keeps every line" has "$log" "memwatch-flood line 0 "
|
||||
|
||||
MEMWATCH_OUTPUT=full tool/memwatch/full.sh "$fail" > "$dir/full.out"; status=$?
|
||||
check "full mode keeps exit 1" [ $status = 1 ]
|
||||
check "full mode prints the log" has "$dir/full.out" "memwatch-flood line 0 "
|
||||
check "full mode ends with the summary" has "$dir/full.out" "memwatch: FAIL: 1 passed, 2 failed"
|
||||
|
||||
# A 1 KB cap forces the watchdog to kill the run: the exit status stays 137.
|
||||
CAP_KB=1 tool/memwatch/run.sh "$pass" > "$dir/killed.out"; status=$?
|
||||
check "killed run exits 137" [ $status = 137 ]
|
||||
check "watchdog reports the kill" has "$dir/killed.out" "KILLED at "
|
||||
check "summary reports the kill" has "$dir/killed.out" "memwatch: KILLED: "
|
||||
|
||||
tool/memwatch/full.sh tool/memwatch/fixtures/missing_test.dart > "$dir/missing.out"; status=$?
|
||||
check "missing test file exits non-zero" [ $status != 0 ]
|
||||
check "missing test file still summarised" has "$dir/missing.out" "memwatch: FAIL: "
|
||||
|
||||
[ $failures = 0 ] && echo "quiet_test: all checks passed" || echo "quiet_test: $failures checks failed"
|
||||
[ $failures = 0 ]
|
||||
@@ -6,22 +6,31 @@
|
||||
# CONC flutter test --concurrency (default 1)
|
||||
# CAP_KB total RSS cap for this run's own test processes (default 3 GB)
|
||||
# MEMWATCH_DIR per-invocation output directory (default /tmp/memwatch)
|
||||
# FLUTTER flutter executable
|
||||
# FLUTTER flutter executable; DART the dart beside it (runs summary.dart)
|
||||
# MEMWATCH_OUTPUT quiet or full. Default: quiet when stdout is not a terminal
|
||||
# (agents, pipelines, pipes), full on a terminal. full prints the whole
|
||||
# flutter log before the summary; quiet prints only the summary.
|
||||
#
|
||||
# Runs flutter test in the background and polls the RSS of ITS OWN descendant
|
||||
# processes only; above CAP_KB it kills them (never another run's flutter_tester).
|
||||
# Prints the peak and the suite's last line, and exits with flutter's status
|
||||
# (137 when killed). A runaway test once grew to ~30 GB per process and crashed
|
||||
# the Mac twice, so never run `flutter test` outside this watchdog.
|
||||
# Then summary.dart prints a bounded report from flutter's JSON results: one
|
||||
# verdict line (pass/fail/skip counts, peak MB, duration, exit status), only
|
||||
# the failing tests with their error and app stack frames, and the paths of
|
||||
# the full log and JSON. Exits with flutter's status (137 when killed). A
|
||||
# runaway test once grew to ~30 GB per process and crashed the Mac twice, so
|
||||
# never run `flutter test` outside this watchdog.
|
||||
CONC=${CONC:-1}; CAP_KB=${CAP_KB:-3000000}
|
||||
MEMWATCH_DIR=${MEMWATCH_DIR:-/tmp/memwatch}
|
||||
FLUTTER=${FLUTTER:-/Users/alan/development/flutter/bin/flutter}
|
||||
DART=${DART:-$(dirname "$FLUTTER")/dart}
|
||||
here=$(cd "$(dirname "$0")" && pwd) || exit 1
|
||||
# Called from elsewhere (for example the repo root): run in this script's app/.
|
||||
[ -f pubspec.yaml ] || cd "$(dirname "$0")/../.." || exit 1
|
||||
[ -f pubspec.yaml ] || cd "$here/../.." || exit 1
|
||||
mkdir -p "$MEMWATCH_DIR" || exit 1
|
||||
out=$(mktemp "$MEMWATCH_DIR/out.XXXXXX")
|
||||
start=$SECONDS
|
||||
# $1 is deliberately unquoted: it carries the test path plus extra arguments.
|
||||
"$FLUTTER" test --concurrency="$CONC" $1 > "$out" 2>&1 &
|
||||
"$FLUTTER" test --concurrency="$CONC" --file-reporter "json:$out.json" $1 > "$out" 2>&1 &
|
||||
pid=$!
|
||||
descendants() { local p; for p in $(pgrep -P "$1"); do echo "$p"; descendants "$p"; done; }
|
||||
peak=0; killed=0
|
||||
@@ -38,6 +47,12 @@ while kill -0 $pid 2>/dev/null; do
|
||||
done
|
||||
wait $pid 2>/dev/null; status=$?
|
||||
[ $killed = 1 ] && status=137
|
||||
echo "peak $((peak/1024)) MB: $1 :: $(tail -1 "$out")"
|
||||
echo "full output: $out"
|
||||
mode=${MEMWATCH_OUTPUT:-}
|
||||
if [ -z "$mode" ]; then if [ -t 1 ]; then mode=full; else mode=quiet; fi; fi
|
||||
[ "$mode" = full ] && cat "$out"
|
||||
# The summary never changes the exit status; if it cannot run, keep the old lines.
|
||||
if ! "$DART" "$here/summary.dart" "$status" "$((peak/1024))" "$((SECONDS-start))" "$1" "$out" "$out.json" 2>/dev/null; then
|
||||
echo "peak $((peak/1024)) MB: $1 :: $(tail -1 "$out")"
|
||||
echo "full output: $out"
|
||||
fi
|
||||
exit $status
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
// Bounded summary of one memwatch run, read from flutter test's JSON file
|
||||
// reporter. run.sh calls it after the watchdog has finished:
|
||||
//
|
||||
// dart tool/memwatch/summary.dart <exit> <peak MB> <seconds> <target> <log> <json>
|
||||
//
|
||||
// It prints one verdict line (counts, peak memory, duration, exit status),
|
||||
// then only the failing tests with their bounded error output, then the log
|
||||
// paths. Passing tests print nothing. The process exit status is always 0:
|
||||
// run.sh owns the real exit status.
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
/// Failures shown with their output; later failures are listed by name only.
|
||||
const maxDetailedFailures = 8;
|
||||
|
||||
/// Failure names listed at all; the rest are counted.
|
||||
const maxListedFailures = 30;
|
||||
|
||||
/// Error lines and characters kept per failing test.
|
||||
const maxLinesPerFailure = 40;
|
||||
const maxCharsPerFailure = 4000;
|
||||
|
||||
/// Trailing lines of a failing test's ordinary print() output.
|
||||
const maxPrintedLines = 8;
|
||||
|
||||
/// Characters of failure detail across the whole summary.
|
||||
const maxDetailChars = 12000;
|
||||
|
||||
/// Characters kept per output line.
|
||||
const maxLineChars = 300;
|
||||
|
||||
/// App stack frames kept per error block; framework frames are dropped.
|
||||
const maxFramesPerBlock = 6;
|
||||
|
||||
/// What one test reported: Flutter exception dumps, test errors and other prints.
|
||||
class _Output {
|
||||
final exceptions = <String>[];
|
||||
final errors = <String>[];
|
||||
final prints = <String>[];
|
||||
}
|
||||
|
||||
String summarize(
|
||||
Iterable<String> events, {
|
||||
required int status,
|
||||
required int peakMb,
|
||||
required int seconds,
|
||||
required String target,
|
||||
required String logPath,
|
||||
required String jsonPath,
|
||||
List<String> logTail = const [],
|
||||
String appPackage = '',
|
||||
String appRoot = '',
|
||||
}) {
|
||||
final names = <int, String>{};
|
||||
final suiteOf = <int, int>{};
|
||||
final suitePaths = <int, String>{};
|
||||
final outputs = <int, _Output>{};
|
||||
final finished = <int>{};
|
||||
final failed = <int>[];
|
||||
var passed = 0, skipped = 0, sawEvent = false;
|
||||
for (final line in events) {
|
||||
final Object? event;
|
||||
try {
|
||||
event = jsonDecode(line);
|
||||
} on FormatException {
|
||||
continue;
|
||||
}
|
||||
if (event is! Map) continue;
|
||||
sawEvent = true;
|
||||
final id = event['testID'];
|
||||
switch (event['type']) {
|
||||
case 'suite':
|
||||
final suite = event['suite'] as Map;
|
||||
suitePaths[suite['id'] as int] = _relative('${suite['path'] ?? ''}', appRoot);
|
||||
case 'testStart':
|
||||
final test = event['test'] as Map;
|
||||
names[test['id'] as int] = '${test['name'] ?? ''}';
|
||||
suiteOf[test['id'] as int] = test['suiteID'] as int? ?? -1;
|
||||
case 'print' when id is int:
|
||||
final message = '${event['message'] ?? ''}';
|
||||
final output = outputs.putIfAbsent(id, _Output.new);
|
||||
(message.contains('EXCEPTION CAUGHT BY') ? output.exceptions : output.prints).add(message);
|
||||
case 'error' when id is int:
|
||||
outputs.putIfAbsent(id, _Output.new).errors.add('${event['error'] ?? ''}\n${event['stackTrace'] ?? ''}');
|
||||
// An error reported after a test finished (for example in tearDown) still fails it.
|
||||
if (!failed.contains(id)) failed.add(id);
|
||||
case 'testDone' when id is int:
|
||||
finished.add(id);
|
||||
if (event['result'] != 'success') {
|
||||
if (!failed.contains(id)) failed.add(id);
|
||||
} else if (event['skipped'] == true) {
|
||||
skipped++;
|
||||
} else if (event['hidden'] != true) {
|
||||
passed++;
|
||||
}
|
||||
}
|
||||
}
|
||||
final verdict = status == 0 ? 'PASS' : (status == 137 ? 'KILLED' : 'FAIL');
|
||||
final out = StringBuffer()
|
||||
..writeln('memwatch: $verdict: $passed passed, ${failed.length} failed, $skipped skipped; '
|
||||
'peak $peakMb MB; ${seconds}s; exit $status ($target)');
|
||||
String label(int id) {
|
||||
final path = suitePaths[suiteOf[id]] ?? '';
|
||||
final name = names[id] ?? 'test $id';
|
||||
return path.isEmpty || name.startsWith('loading ') ? _relative(name, appRoot) : '$path: $name';
|
||||
}
|
||||
|
||||
var budget = maxDetailChars;
|
||||
for (final (index, id) in failed.indexed) {
|
||||
if (index >= maxListedFailures) {
|
||||
out.writeln('... and ${failed.length - index} more failures (see full log)');
|
||||
break;
|
||||
}
|
||||
out.writeln('FAILED ${label(id)}');
|
||||
if (index >= maxDetailedFailures || budget <= 0) continue;
|
||||
var detail = _detail(outputs[id] ?? _Output(), appPackage, appRoot);
|
||||
if (detail.length > budget) detail = '${detail.substring(0, budget)}\n ... (summary detail budget reached)';
|
||||
budget -= detail.length;
|
||||
if (detail.isNotEmpty) out.writeln(detail);
|
||||
}
|
||||
if (status == 137) {
|
||||
for (final id in names.keys.where((id) => !finished.contains(id)).take(5)) {
|
||||
out.writeln('RUNNING when killed: ${label(id)}');
|
||||
}
|
||||
}
|
||||
if (!sawEvent && status != 0) {
|
||||
out.writeln('no test results were recorded; last log lines:');
|
||||
for (final line in logTail.take(20)) {
|
||||
out.writeln(' ${_clip(line)}');
|
||||
}
|
||||
}
|
||||
out
|
||||
..writeln('full output: $logPath')
|
||||
..write('json results: $jsonPath');
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
/// The failure cause first (Flutter exception dumps, then test errors, minus
|
||||
/// framework frames), capped; then the tail of the test's ordinary prints.
|
||||
String _detail(_Output output, String appPackage, String appRoot) {
|
||||
final errors = output.exceptions.isEmpty
|
||||
? output.errors
|
||||
: output.errors.where((e) => !e.startsWith('Test failed. See exception logs above.'));
|
||||
final lines = <String>[];
|
||||
var dropped = 0;
|
||||
for (final block in [...output.exceptions, ...errors]) {
|
||||
var frames = 0;
|
||||
for (final raw in const LineSplitter().convert(block)) {
|
||||
final line = raw.trimRight();
|
||||
if (_isNoise(line)) continue;
|
||||
if (_isFrame(line)) {
|
||||
if (!_isAppFrame(line, appPackage, appRoot) || frames >= maxFramesPerBlock) continue;
|
||||
frames++;
|
||||
}
|
||||
if (line.trim().isEmpty && (lines.isEmpty || lines.last.isEmpty)) continue;
|
||||
if (lines.length >= maxLinesPerFailure) {
|
||||
dropped++;
|
||||
continue;
|
||||
}
|
||||
lines.add(line.trim().isEmpty ? '' : _relative(line, appRoot));
|
||||
}
|
||||
}
|
||||
while (lines.isNotEmpty && lines.last.isEmpty) {
|
||||
lines.removeLast();
|
||||
}
|
||||
final buffer = StringBuffer();
|
||||
for (final line in lines) {
|
||||
if (buffer.length >= maxCharsPerFailure) {
|
||||
dropped++;
|
||||
continue;
|
||||
}
|
||||
buffer.writeln(line.isEmpty ? '' : ' ${_clip(line)}');
|
||||
}
|
||||
if (dropped > 0) buffer.writeln(' ... ($dropped more error lines in the full log)');
|
||||
final printed = output.prints.expand(const LineSplitter().convert).toList();
|
||||
if (printed.isNotEmpty) {
|
||||
final tail = printed.length > maxPrintedLines ? printed.sublist(printed.length - maxPrintedLines) : printed;
|
||||
buffer.writeln(' printed output (last ${tail.length} of ${printed.length} lines):');
|
||||
for (final line in tail) {
|
||||
buffer.writeln(' | ${_clip(line)}');
|
||||
}
|
||||
}
|
||||
return buffer.toString().trimRight();
|
||||
}
|
||||
|
||||
final _frame = RegExp(r'^(#\d+\s|package:|dart:|file:///|\S+\.dart\s+\d+)');
|
||||
|
||||
bool _isFrame(String line) => _frame.hasMatch(line.trimLeft());
|
||||
|
||||
bool _isNoise(String line) {
|
||||
final trimmed = line.trim();
|
||||
return trimmed == '<asynchronous suspension>' ||
|
||||
trimmed.startsWith('===== asynchronous gap') ||
|
||||
trimmed.startsWith('(elided ');
|
||||
}
|
||||
|
||||
bool _isAppFrame(String line, String appPackage, String appRoot) {
|
||||
if (line.contains('packages/flutter/') || line.contains('/.pub-cache/') || line.contains('/flutter/bin/cache/')) {
|
||||
return false;
|
||||
}
|
||||
if (appPackage.isNotEmpty && line.contains('package:$appPackage/')) return true;
|
||||
if (appRoot.isNotEmpty && line.contains(appRoot)) return true;
|
||||
// A terse frame with a relative path (test/x_test.dart 12:3) is app code.
|
||||
return RegExp(r'^\S+\.dart\s+\d+').hasMatch(line.trimLeft()) && !RegExp(r'^(package|dart):').hasMatch(line.trimLeft());
|
||||
}
|
||||
|
||||
/// Strips the app root (with any file:// scheme) so paths read as test/....
|
||||
String _relative(String text, String appRoot) {
|
||||
if (appRoot.isEmpty) return text;
|
||||
return text.replaceAll('file://$appRoot/', '').replaceAll('$appRoot/', '');
|
||||
}
|
||||
|
||||
String _clip(String line) => line.length <= maxLineChars ? line : '${line.substring(0, maxLineChars)}...';
|
||||
|
||||
List<String> _readLines(String path) {
|
||||
final file = File(path);
|
||||
if (!file.existsSync()) return const [];
|
||||
return file.readAsLinesSync();
|
||||
}
|
||||
|
||||
String _appPackage() {
|
||||
final pubspec = File('pubspec.yaml');
|
||||
if (!pubspec.existsSync()) return '';
|
||||
return RegExp(r'^name:\s*(\S+)', multiLine: true).firstMatch(pubspec.readAsStringSync())?.group(1) ?? '';
|
||||
}
|
||||
|
||||
void main(List<String> args) {
|
||||
if (args.length != 6) {
|
||||
stderr.writeln('usage: summary.dart <exit> <peak MB> <seconds> <target> <log> <json>');
|
||||
exit(2);
|
||||
}
|
||||
final log = _readLines(args[4]);
|
||||
stdout.writeln(summarize(
|
||||
_readLines(args[5]),
|
||||
status: int.parse(args[0]),
|
||||
peakMb: int.parse(args[1]),
|
||||
seconds: int.parse(args[2]),
|
||||
target: args[3],
|
||||
logPath: args[4],
|
||||
jsonPath: args[5],
|
||||
logTail: log.length > 20 ? log.sublist(log.length - 20) : log,
|
||||
appPackage: _appPackage(),
|
||||
appRoot: Directory.current.path,
|
||||
));
|
||||
}
|
||||
Reference in New Issue
Block a user