Files
pipelines/app/tool/memwatch/summary.dart
T
alan 941d2a7148 memwatch: quiet bounded test summary off a terminal; bounded analyze wrapper
run.sh now records flutter's JSON results beside the log and prints a
summary from tool/memwatch/summary.dart: one verdict line (pass/fail/skip
counts, peak MB, duration, exit status), only failing tests with their
error and app stack frames (framework frames dropped, per-test and total
caps, tail of printed output), the tests still running when killed, and
the log and JSON paths. Quiet is the default when stdout is not a
terminal; MEMWATCH_OUTPUT=full also prints the whole log. The watchdog
loop and exit status are unchanged.

tool/analyze.sh bounds flutter analyze to a verdict line and the first
40 issues. Tests: test/tool/memwatch/summary_test.dart and the
end-to-end tool/memwatch/quiet_test.sh over fixture suites.
2026-11-04 13:10:21 +00:00

246 lines
8.8 KiB
Dart

// Bounded summary of one memwatch run, read from flutter test's JSON file
// reporter. run.sh calls it after the watchdog has finished:
//
// dart tool/memwatch/summary.dart <exit> <peak MB> <seconds> <target> <log> <json>
//
// It prints one verdict line (counts, peak memory, duration, exit status),
// then only the failing tests with their bounded error output, then the log
// paths. Passing tests print nothing. The process exit status is always 0:
// run.sh owns the real exit status.
import 'dart:convert';
import 'dart:io';
/// Failures shown with their output; later failures are listed by name only.
const maxDetailedFailures = 8;
/// Failure names listed at all; the rest are counted.
const maxListedFailures = 30;
/// Error lines and characters kept per failing test.
const maxLinesPerFailure = 40;
const maxCharsPerFailure = 4000;
/// Trailing lines of a failing test's ordinary print() output.
const maxPrintedLines = 8;
/// Characters of failure detail across the whole summary.
const maxDetailChars = 12000;
/// Characters kept per output line.
const maxLineChars = 300;
/// App stack frames kept per error block; framework frames are dropped.
const maxFramesPerBlock = 6;
/// What one test reported: Flutter exception dumps, test errors and other prints.
class _Output {
final exceptions = <String>[];
final errors = <String>[];
final prints = <String>[];
}
String summarize(
Iterable<String> events, {
required int status,
required int peakMb,
required int seconds,
required String target,
required String logPath,
required String jsonPath,
List<String> logTail = const [],
String appPackage = '',
String appRoot = '',
}) {
final names = <int, String>{};
final suiteOf = <int, int>{};
final suitePaths = <int, String>{};
final outputs = <int, _Output>{};
final finished = <int>{};
final failed = <int>[];
var passed = 0, skipped = 0, sawEvent = false;
for (final line in events) {
final Object? event;
try {
event = jsonDecode(line);
} on FormatException {
continue;
}
if (event is! Map) continue;
sawEvent = true;
final id = event['testID'];
switch (event['type']) {
case 'suite':
final suite = event['suite'] as Map;
suitePaths[suite['id'] as int] = _relative('${suite['path'] ?? ''}', appRoot);
case 'testStart':
final test = event['test'] as Map;
names[test['id'] as int] = '${test['name'] ?? ''}';
suiteOf[test['id'] as int] = test['suiteID'] as int? ?? -1;
case 'print' when id is int:
final message = '${event['message'] ?? ''}';
final output = outputs.putIfAbsent(id, _Output.new);
(message.contains('EXCEPTION CAUGHT BY') ? output.exceptions : output.prints).add(message);
case 'error' when id is int:
outputs.putIfAbsent(id, _Output.new).errors.add('${event['error'] ?? ''}\n${event['stackTrace'] ?? ''}');
// An error reported after a test finished (for example in tearDown) still fails it.
if (!failed.contains(id)) failed.add(id);
case 'testDone' when id is int:
finished.add(id);
if (event['result'] != 'success') {
if (!failed.contains(id)) failed.add(id);
} else if (event['skipped'] == true) {
skipped++;
} else if (event['hidden'] != true) {
passed++;
}
}
}
final verdict = status == 0 ? 'PASS' : (status == 137 ? 'KILLED' : 'FAIL');
final out = StringBuffer()
..writeln('memwatch: $verdict: $passed passed, ${failed.length} failed, $skipped skipped; '
'peak $peakMb MB; ${seconds}s; exit $status ($target)');
String label(int id) {
final path = suitePaths[suiteOf[id]] ?? '';
final name = names[id] ?? 'test $id';
return path.isEmpty || name.startsWith('loading ') ? _relative(name, appRoot) : '$path: $name';
}
var budget = maxDetailChars;
for (final (index, id) in failed.indexed) {
if (index >= maxListedFailures) {
out.writeln('... and ${failed.length - index} more failures (see full log)');
break;
}
out.writeln('FAILED ${label(id)}');
if (index >= maxDetailedFailures || budget <= 0) continue;
var detail = _detail(outputs[id] ?? _Output(), appPackage, appRoot);
if (detail.length > budget) detail = '${detail.substring(0, budget)}\n ... (summary detail budget reached)';
budget -= detail.length;
if (detail.isNotEmpty) out.writeln(detail);
}
if (status == 137) {
for (final id in names.keys.where((id) => !finished.contains(id)).take(5)) {
out.writeln('RUNNING when killed: ${label(id)}');
}
}
if (!sawEvent && status != 0) {
out.writeln('no test results were recorded; last log lines:');
for (final line in logTail.take(20)) {
out.writeln(' ${_clip(line)}');
}
}
out
..writeln('full output: $logPath')
..write('json results: $jsonPath');
return out.toString();
}
/// The failure cause first (Flutter exception dumps, then test errors, minus
/// framework frames), capped; then the tail of the test's ordinary prints.
String _detail(_Output output, String appPackage, String appRoot) {
final errors = output.exceptions.isEmpty
? output.errors
: output.errors.where((e) => !e.startsWith('Test failed. See exception logs above.'));
final lines = <String>[];
var dropped = 0;
for (final block in [...output.exceptions, ...errors]) {
var frames = 0;
for (final raw in const LineSplitter().convert(block)) {
final line = raw.trimRight();
if (_isNoise(line)) continue;
if (_isFrame(line)) {
if (!_isAppFrame(line, appPackage, appRoot) || frames >= maxFramesPerBlock) continue;
frames++;
}
if (line.trim().isEmpty && (lines.isEmpty || lines.last.isEmpty)) continue;
if (lines.length >= maxLinesPerFailure) {
dropped++;
continue;
}
lines.add(line.trim().isEmpty ? '' : _relative(line, appRoot));
}
}
while (lines.isNotEmpty && lines.last.isEmpty) {
lines.removeLast();
}
final buffer = StringBuffer();
for (final line in lines) {
if (buffer.length >= maxCharsPerFailure) {
dropped++;
continue;
}
buffer.writeln(line.isEmpty ? '' : ' ${_clip(line)}');
}
if (dropped > 0) buffer.writeln(' ... ($dropped more error lines in the full log)');
final printed = output.prints.expand(const LineSplitter().convert).toList();
if (printed.isNotEmpty) {
final tail = printed.length > maxPrintedLines ? printed.sublist(printed.length - maxPrintedLines) : printed;
buffer.writeln(' printed output (last ${tail.length} of ${printed.length} lines):');
for (final line in tail) {
buffer.writeln(' | ${_clip(line)}');
}
}
return buffer.toString().trimRight();
}
final _frame = RegExp(r'^(#\d+\s|package:|dart:|file:///|\S+\.dart\s+\d+)');
bool _isFrame(String line) => _frame.hasMatch(line.trimLeft());
bool _isNoise(String line) {
final trimmed = line.trim();
return trimmed == '<asynchronous suspension>' ||
trimmed.startsWith('===== asynchronous gap') ||
trimmed.startsWith('(elided ');
}
bool _isAppFrame(String line, String appPackage, String appRoot) {
if (line.contains('packages/flutter/') || line.contains('/.pub-cache/') || line.contains('/flutter/bin/cache/')) {
return false;
}
if (appPackage.isNotEmpty && line.contains('package:$appPackage/')) return true;
if (appRoot.isNotEmpty && line.contains(appRoot)) return true;
// A terse frame with a relative path (test/x_test.dart 12:3) is app code.
return RegExp(r'^\S+\.dart\s+\d+').hasMatch(line.trimLeft()) && !RegExp(r'^(package|dart):').hasMatch(line.trimLeft());
}
/// Strips the app root (with any file:// scheme) so paths read as test/....
String _relative(String text, String appRoot) {
if (appRoot.isEmpty) return text;
return text.replaceAll('file://$appRoot/', '').replaceAll('$appRoot/', '');
}
String _clip(String line) => line.length <= maxLineChars ? line : '${line.substring(0, maxLineChars)}...';
List<String> _readLines(String path) {
final file = File(path);
if (!file.existsSync()) return const [];
return file.readAsLinesSync();
}
String _appPackage() {
final pubspec = File('pubspec.yaml');
if (!pubspec.existsSync()) return '';
return RegExp(r'^name:\s*(\S+)', multiLine: true).firstMatch(pubspec.readAsStringSync())?.group(1) ?? '';
}
void main(List<String> args) {
if (args.length != 6) {
stderr.writeln('usage: summary.dart <exit> <peak MB> <seconds> <target> <log> <json>');
exit(2);
}
final log = _readLines(args[4]);
stdout.writeln(summarize(
_readLines(args[5]),
status: int.parse(args[0]),
peakMb: int.parse(args[1]),
seconds: int.parse(args[2]),
target: args[3],
logPath: args[4],
jsonPath: args[5],
logTail: log.length > 20 ? log.sublist(log.length - 20) : log,
appPackage: _appPackage(),
appRoot: Directory.current.path,
));
}