941d2a7148
run.sh now records flutter's JSON results beside the log and prints a summary from tool/memwatch/summary.dart: one verdict line (pass/fail/skip counts, peak MB, duration, exit status), only failing tests with their error and app stack frames (framework frames dropped, per-test and total caps, tail of printed output), the tests still running when killed, and the log and JSON paths. Quiet is the default when stdout is not a terminal; MEMWATCH_OUTPUT=full also prints the whole log. The watchdog loop and exit status are unchanged. tool/analyze.sh bounds flutter analyze to a verdict line and the first 40 issues. Tests: test/tool/memwatch/summary_test.dart and the end-to-end tool/memwatch/quiet_test.sh over fixture suites.
246 lines
8.8 KiB
Dart
246 lines
8.8 KiB
Dart
// Bounded summary of one memwatch run, read from flutter test's JSON file
|
|
// reporter. run.sh calls it after the watchdog has finished:
|
|
//
|
|
// dart tool/memwatch/summary.dart <exit> <peak MB> <seconds> <target> <log> <json>
|
|
//
|
|
// It prints one verdict line (counts, peak memory, duration, exit status),
|
|
// then only the failing tests with their bounded error output, then the log
|
|
// paths. Passing tests print nothing. The process exit status is always 0:
|
|
// run.sh owns the real exit status.
|
|
import 'dart:convert';
|
|
import 'dart:io';
|
|
|
|
/// Failures shown with their output; later failures are listed by name only.
|
|
const maxDetailedFailures = 8;
|
|
|
|
/// Failure names listed at all; the rest are counted.
|
|
const maxListedFailures = 30;
|
|
|
|
/// Error lines and characters kept per failing test.
|
|
const maxLinesPerFailure = 40;
|
|
const maxCharsPerFailure = 4000;
|
|
|
|
/// Trailing lines of a failing test's ordinary print() output.
|
|
const maxPrintedLines = 8;
|
|
|
|
/// Characters of failure detail across the whole summary.
|
|
const maxDetailChars = 12000;
|
|
|
|
/// Characters kept per output line.
|
|
const maxLineChars = 300;
|
|
|
|
/// App stack frames kept per error block; framework frames are dropped.
|
|
const maxFramesPerBlock = 6;
|
|
|
|
/// What one test reported: Flutter exception dumps, test errors and other prints.
|
|
class _Output {
|
|
final exceptions = <String>[];
|
|
final errors = <String>[];
|
|
final prints = <String>[];
|
|
}
|
|
|
|
String summarize(
|
|
Iterable<String> events, {
|
|
required int status,
|
|
required int peakMb,
|
|
required int seconds,
|
|
required String target,
|
|
required String logPath,
|
|
required String jsonPath,
|
|
List<String> logTail = const [],
|
|
String appPackage = '',
|
|
String appRoot = '',
|
|
}) {
|
|
final names = <int, String>{};
|
|
final suiteOf = <int, int>{};
|
|
final suitePaths = <int, String>{};
|
|
final outputs = <int, _Output>{};
|
|
final finished = <int>{};
|
|
final failed = <int>[];
|
|
var passed = 0, skipped = 0, sawEvent = false;
|
|
for (final line in events) {
|
|
final Object? event;
|
|
try {
|
|
event = jsonDecode(line);
|
|
} on FormatException {
|
|
continue;
|
|
}
|
|
if (event is! Map) continue;
|
|
sawEvent = true;
|
|
final id = event['testID'];
|
|
switch (event['type']) {
|
|
case 'suite':
|
|
final suite = event['suite'] as Map;
|
|
suitePaths[suite['id'] as int] = _relative('${suite['path'] ?? ''}', appRoot);
|
|
case 'testStart':
|
|
final test = event['test'] as Map;
|
|
names[test['id'] as int] = '${test['name'] ?? ''}';
|
|
suiteOf[test['id'] as int] = test['suiteID'] as int? ?? -1;
|
|
case 'print' when id is int:
|
|
final message = '${event['message'] ?? ''}';
|
|
final output = outputs.putIfAbsent(id, _Output.new);
|
|
(message.contains('EXCEPTION CAUGHT BY') ? output.exceptions : output.prints).add(message);
|
|
case 'error' when id is int:
|
|
outputs.putIfAbsent(id, _Output.new).errors.add('${event['error'] ?? ''}\n${event['stackTrace'] ?? ''}');
|
|
// An error reported after a test finished (for example in tearDown) still fails it.
|
|
if (!failed.contains(id)) failed.add(id);
|
|
case 'testDone' when id is int:
|
|
finished.add(id);
|
|
if (event['result'] != 'success') {
|
|
if (!failed.contains(id)) failed.add(id);
|
|
} else if (event['skipped'] == true) {
|
|
skipped++;
|
|
} else if (event['hidden'] != true) {
|
|
passed++;
|
|
}
|
|
}
|
|
}
|
|
final verdict = status == 0 ? 'PASS' : (status == 137 ? 'KILLED' : 'FAIL');
|
|
final out = StringBuffer()
|
|
..writeln('memwatch: $verdict: $passed passed, ${failed.length} failed, $skipped skipped; '
|
|
'peak $peakMb MB; ${seconds}s; exit $status ($target)');
|
|
String label(int id) {
|
|
final path = suitePaths[suiteOf[id]] ?? '';
|
|
final name = names[id] ?? 'test $id';
|
|
return path.isEmpty || name.startsWith('loading ') ? _relative(name, appRoot) : '$path: $name';
|
|
}
|
|
|
|
var budget = maxDetailChars;
|
|
for (final (index, id) in failed.indexed) {
|
|
if (index >= maxListedFailures) {
|
|
out.writeln('... and ${failed.length - index} more failures (see full log)');
|
|
break;
|
|
}
|
|
out.writeln('FAILED ${label(id)}');
|
|
if (index >= maxDetailedFailures || budget <= 0) continue;
|
|
var detail = _detail(outputs[id] ?? _Output(), appPackage, appRoot);
|
|
if (detail.length > budget) detail = '${detail.substring(0, budget)}\n ... (summary detail budget reached)';
|
|
budget -= detail.length;
|
|
if (detail.isNotEmpty) out.writeln(detail);
|
|
}
|
|
if (status == 137) {
|
|
for (final id in names.keys.where((id) => !finished.contains(id)).take(5)) {
|
|
out.writeln('RUNNING when killed: ${label(id)}');
|
|
}
|
|
}
|
|
if (!sawEvent && status != 0) {
|
|
out.writeln('no test results were recorded; last log lines:');
|
|
for (final line in logTail.take(20)) {
|
|
out.writeln(' ${_clip(line)}');
|
|
}
|
|
}
|
|
out
|
|
..writeln('full output: $logPath')
|
|
..write('json results: $jsonPath');
|
|
return out.toString();
|
|
}
|
|
|
|
/// The failure cause first (Flutter exception dumps, then test errors, minus
|
|
/// framework frames), capped; then the tail of the test's ordinary prints.
|
|
String _detail(_Output output, String appPackage, String appRoot) {
|
|
final errors = output.exceptions.isEmpty
|
|
? output.errors
|
|
: output.errors.where((e) => !e.startsWith('Test failed. See exception logs above.'));
|
|
final lines = <String>[];
|
|
var dropped = 0;
|
|
for (final block in [...output.exceptions, ...errors]) {
|
|
var frames = 0;
|
|
for (final raw in const LineSplitter().convert(block)) {
|
|
final line = raw.trimRight();
|
|
if (_isNoise(line)) continue;
|
|
if (_isFrame(line)) {
|
|
if (!_isAppFrame(line, appPackage, appRoot) || frames >= maxFramesPerBlock) continue;
|
|
frames++;
|
|
}
|
|
if (line.trim().isEmpty && (lines.isEmpty || lines.last.isEmpty)) continue;
|
|
if (lines.length >= maxLinesPerFailure) {
|
|
dropped++;
|
|
continue;
|
|
}
|
|
lines.add(line.trim().isEmpty ? '' : _relative(line, appRoot));
|
|
}
|
|
}
|
|
while (lines.isNotEmpty && lines.last.isEmpty) {
|
|
lines.removeLast();
|
|
}
|
|
final buffer = StringBuffer();
|
|
for (final line in lines) {
|
|
if (buffer.length >= maxCharsPerFailure) {
|
|
dropped++;
|
|
continue;
|
|
}
|
|
buffer.writeln(line.isEmpty ? '' : ' ${_clip(line)}');
|
|
}
|
|
if (dropped > 0) buffer.writeln(' ... ($dropped more error lines in the full log)');
|
|
final printed = output.prints.expand(const LineSplitter().convert).toList();
|
|
if (printed.isNotEmpty) {
|
|
final tail = printed.length > maxPrintedLines ? printed.sublist(printed.length - maxPrintedLines) : printed;
|
|
buffer.writeln(' printed output (last ${tail.length} of ${printed.length} lines):');
|
|
for (final line in tail) {
|
|
buffer.writeln(' | ${_clip(line)}');
|
|
}
|
|
}
|
|
return buffer.toString().trimRight();
|
|
}
|
|
|
|
final _frame = RegExp(r'^(#\d+\s|package:|dart:|file:///|\S+\.dart\s+\d+)');
|
|
|
|
bool _isFrame(String line) => _frame.hasMatch(line.trimLeft());
|
|
|
|
bool _isNoise(String line) {
|
|
final trimmed = line.trim();
|
|
return trimmed == '<asynchronous suspension>' ||
|
|
trimmed.startsWith('===== asynchronous gap') ||
|
|
trimmed.startsWith('(elided ');
|
|
}
|
|
|
|
bool _isAppFrame(String line, String appPackage, String appRoot) {
|
|
if (line.contains('packages/flutter/') || line.contains('/.pub-cache/') || line.contains('/flutter/bin/cache/')) {
|
|
return false;
|
|
}
|
|
if (appPackage.isNotEmpty && line.contains('package:$appPackage/')) return true;
|
|
if (appRoot.isNotEmpty && line.contains(appRoot)) return true;
|
|
// A terse frame with a relative path (test/x_test.dart 12:3) is app code.
|
|
return RegExp(r'^\S+\.dart\s+\d+').hasMatch(line.trimLeft()) && !RegExp(r'^(package|dart):').hasMatch(line.trimLeft());
|
|
}
|
|
|
|
/// Strips the app root (with any file:// scheme) so paths read as test/....
|
|
String _relative(String text, String appRoot) {
|
|
if (appRoot.isEmpty) return text;
|
|
return text.replaceAll('file://$appRoot/', '').replaceAll('$appRoot/', '');
|
|
}
|
|
|
|
String _clip(String line) => line.length <= maxLineChars ? line : '${line.substring(0, maxLineChars)}...';
|
|
|
|
List<String> _readLines(String path) {
|
|
final file = File(path);
|
|
if (!file.existsSync()) return const [];
|
|
return file.readAsLinesSync();
|
|
}
|
|
|
|
String _appPackage() {
|
|
final pubspec = File('pubspec.yaml');
|
|
if (!pubspec.existsSync()) return '';
|
|
return RegExp(r'^name:\s*(\S+)', multiLine: true).firstMatch(pubspec.readAsStringSync())?.group(1) ?? '';
|
|
}
|
|
|
|
void main(List<String> args) {
|
|
if (args.length != 6) {
|
|
stderr.writeln('usage: summary.dart <exit> <peak MB> <seconds> <target> <log> <json>');
|
|
exit(2);
|
|
}
|
|
final log = _readLines(args[4]);
|
|
stdout.writeln(summarize(
|
|
_readLines(args[5]),
|
|
status: int.parse(args[0]),
|
|
peakMb: int.parse(args[1]),
|
|
seconds: int.parse(args[2]),
|
|
target: args[3],
|
|
logPath: args[4],
|
|
jsonPath: args[5],
|
|
logTail: log.length > 20 ? log.sublist(log.length - 20) : log,
|
|
appPackage: _appPackage(),
|
|
appRoot: Directory.current.path,
|
|
));
|
|
}
|