Improve testing for leak tracking. (#140553)

packages/flutter/test/flutter_test_config.dart

@@ -44,6 +44,7 @@ Future<void> testExecutable(FutureOr<void> Function() testMain) {
     LeakTesting.settings = LeakTesting
       .settings
       .withIgnored(
+        createdByTestHelpers: true,
         allNotGCed: true,
       );
   }

packages/flutter_test/test/utils/memory_leak_tests.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter/cupertino.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:leak_tracker_flutter_testing/leak_tracker_flutter_testing.dart';
+
+/// Objects that should not be GCed during test run.
+final List<InstrumentedDisposable> _retainer = <InstrumentedDisposable>[];
+
+/// Test cases for memory leaks.
+///
+/// They are separate from test execution to allow
+/// excluding them from test helpers.
+final List<LeakTestCase> memoryLeakTests = <LeakTestCase>[
+  LeakTestCase(
+    name: 'no leaks',
+    body: (PumpWidgetsCallback? pumpWidgets,
+        RunAsyncCallback<dynamic>? runAsync) async {
+      await pumpWidgets!(Container());
+    },
+  ),
+  LeakTestCase(
+    name: 'not disposed disposable',
+    body: (PumpWidgetsCallback? pumpWidgets,
+        RunAsyncCallback<dynamic>? runAsync) async {
+      InstrumentedDisposable();
+    },
+    notDisposedTotal: 1,
+  ),
+  LeakTestCase(
+    name: 'not GCed disposable',
+    body: (PumpWidgetsCallback? pumpWidgets,
+        RunAsyncCallback<dynamic>? runAsync) async {
+      _retainer.add(InstrumentedDisposable()..dispose());
+    },
+    notGCedTotal: 1,
+  ),
+  LeakTestCase(
+    name: 'leaking widget',
+    body: (PumpWidgetsCallback? pumpWidgets,
+        RunAsyncCallback<dynamic>? runAsync) async {
+      StatelessLeakingWidget();
+    },
+    notDisposedTotal: 1,
+    notGCedTotal: 1,
+  ),
+  LeakTestCase(
+    name: 'dispose in tear down',
+    body: (PumpWidgetsCallback? pumpWidgets,
+        RunAsyncCallback<dynamic>? runAsync) async {
+      final InstrumentedDisposable myClass = InstrumentedDisposable();
+      addTearDown(myClass.dispose);
+    },
+  ),
+  LeakTestCase(
+    name: 'pumped leaking widget',
+    body: (PumpWidgetsCallback? pumpWidgets,
+        RunAsyncCallback<dynamic>? runAsync) async {
+      await pumpWidgets!(StatelessLeakingWidget());
+    },
+    notDisposedTotal: 1,
+    notGCedTotal: 1,
+  ),
+  LeakTestCase(
+    name: 'leaking widget in runAsync',
+    body: (PumpWidgetsCallback? pumpWidgets,
+        RunAsyncCallback<dynamic>? runAsync) async {
+      await runAsync!(() async {
+        StatelessLeakingWidget();
+      });
+    },
+    notDisposedTotal: 1,
+    notGCedTotal: 1,
+  ),
+  LeakTestCase(
+    name: 'pumped in runAsync',
+    body: (PumpWidgetsCallback? pumpWidgets,
+        RunAsyncCallback<dynamic>? runAsync) async {
+      await runAsync!(() async {
+        await pumpWidgets!(StatelessLeakingWidget());
+      });
+    },
+    notDisposedTotal: 1,
+    notGCedTotal: 1,
+  ),
+];
+
+String memoryLeakTestsFilePath() {
+  return RegExp(r'(\/[^\/]*.dart):')
+        .firstMatch(StackTrace.current.toString())!
+        .group(1).toString();
+}

packages/flutter_test/test/widget_tester_leaks_test.dart

@@ -2,199 +2,62 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'package:flutter/cupertino.dart';
-import 'package:flutter/material.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:leak_tracker_flutter_testing/leak_tracker_flutter_testing.dart';
 
-late final String _test1TrackingOnNoLeaks;
-late final String _test2TrackingOffLeaks;
-late final String _test3TrackingOnLeaks;
-late final String _test4TrackingOnWithCreationStackTrace;
-late final String _test5TrackingOnWithDisposalStackTrace;
-late final String _test6TrackingOnNoLeaks;
-late final String _test7TrackingOnNoLeaks;
-late final String _test8TrackingOnNotDisposed;
+import 'utils/memory_leak_tests.dart';
 
-void main() {
-  LeakTesting.enable();
-  LeakTesting.collectedLeaksReporter = (Leaks leaks) => verifyLeaks(leaks);
-  LeakTesting.settings = LeakTesting.settings.copyWith(ignore: false);
-
-  // It is important that the test file starts with group, to test that leaks are collected for all tests after group too.
-  group('Group', () {
-    testWidgets('test', (_) async {
-      StatelessLeakingWidget();
-    });
-  });
-
-  testWidgets(_test1TrackingOnNoLeaks = 'test1, tracking-on, no leaks', (WidgetTester widgetTester) async {
-    expect(LeakTracking.isStarted, true);
-    expect(LeakTracking.phase.name, _test1TrackingOnNoLeaks);
-    expect(LeakTracking.phase.ignoreLeaks, false);
-    await widgetTester.pumpWidget(Container());
-  });
-
-  testWidgets(
-    _test2TrackingOffLeaks = 'test2, tracking-off, leaks',
-    experimentalLeakTesting: LeakTesting.settings.withIgnoredAll(), // this test is not tracked by design
-  (WidgetTester widgetTester) async {
-    await widgetTester.pumpWidget(StatelessLeakingWidget());
-  });
+class _TestExecution {
+  _TestExecution(
+      {required this.settings, required this.settingName, required this.test});
 
-  testWidgets(_test3TrackingOnLeaks = 'test3, tracking-on, leaks', (WidgetTester widgetTester) async {
-    expect(LeakTracking.isStarted, true);
-    expect(LeakTracking.phase.name, _test3TrackingOnLeaks);
-    expect(LeakTracking.phase.ignoreLeaks, false);
-    await widgetTester.pumpWidget(StatelessLeakingWidget());
-  });
-
-  testWidgets(
-    _test4TrackingOnWithCreationStackTrace = 'test4, tracking-on, with creation stack trace',
-    experimentalLeakTesting: LeakTesting.settings.withCreationStackTrace(),
-  (WidgetTester widgetTester) async {
-      expect(LeakTracking.isStarted, true);
-      expect(LeakTracking.phase.name, _test4TrackingOnWithCreationStackTrace);
-      expect(LeakTracking.phase.ignoreLeaks, false);
-      await widgetTester.pumpWidget(StatelessLeakingWidget());
-    },
-  );
-
-  testWidgets(
-    _test5TrackingOnWithDisposalStackTrace = 'test5, tracking-on, with disposal stack trace',
-  experimentalLeakTesting: LeakTesting.settings.withDisposalStackTrace(),
-    (WidgetTester widgetTester) async {
-      expect(LeakTracking.isStarted, true);
-      expect(LeakTracking.phase.name, _test5TrackingOnWithDisposalStackTrace);
-      expect(LeakTracking.phase.ignoreLeaks, false);
-      await widgetTester.pumpWidget(StatelessLeakingWidget());
-    },
-  );
-
-  testWidgets(_test6TrackingOnNoLeaks = 'test6, tracking-on, no leaks', (_) async {
-    InstrumentedDisposable().dispose();
-  });
-
-  testWidgets(_test7TrackingOnNoLeaks = 'test7, tracking-on, tear down, no leaks', (_) async {
-    final InstrumentedDisposable myClass = InstrumentedDisposable();
-    addTearDown(myClass.dispose);
-  });
+  final String settingName;
+  final LeakTesting settings;
+  final LeakTestCase test;
 
-  testWidgets(_test8TrackingOnNotDisposed = 'test8, tracking-on, not disposed leak', (_) async {
-    expect(LeakTracking.isStarted, true);
-    expect(LeakTracking.phase.name, _test8TrackingOnNotDisposed);
-    expect(LeakTracking.phase.ignoreLeaks, false);
-    InstrumentedDisposable();
-  });
+  String get name => '${test.name}, $settingName';
 }
 
-int _leakReporterInvocationCount = 0;
-
-void verifyLeaks(Leaks leaks) {
-  _leakReporterInvocationCount += 1;
-  expect(_leakReporterInvocationCount, 1);
+final List<_TestExecution> _testExecutions = <_TestExecution>[];
 
-  try {
-    expect(leaks, isLeakFree);
-  } on TestFailure catch (e) {
-    expect(e.message, contains('https://github.com/dart-lang/leak_tracker'));
+void main() {
+  LeakTesting.collectedLeaksReporter = _verifyLeaks;
+  LeakTesting.enable();
 
-    expect(e.message, isNot(contains(_test1TrackingOnNoLeaks)));
-    expect(e.message, isNot(contains(_test2TrackingOffLeaks)));
-    expect(e.message, contains('test: $_test3TrackingOnLeaks'));
-    expect(e.message, contains('test: $_test4TrackingOnWithCreationStackTrace'));
-    expect(e.message, contains('test: $_test5TrackingOnWithDisposalStackTrace'));
-    expect(e.message, isNot(contains(_test6TrackingOnNoLeaks)));
-    expect(e.message, isNot(contains(_test7TrackingOnNoLeaks)));
-    expect(e.message, contains('test: $_test8TrackingOnNotDisposed'));
+  LeakTesting.settings = LeakTesting.settings
+      .withTrackedAll()
+      .withTracked(allNotDisposed: true, allNotGCed: true)
+      .withIgnored(
+    createdByTestHelpers: true,
+    testHelperExceptions: <RegExp>[
+      RegExp(RegExp.escape(memoryLeakTestsFilePath()))
+    ],
+  );
+
+  for (final LeakTestCase test in memoryLeakTests) {
+    for (final MapEntry<String,
+            LeakTesting Function(LeakTesting settings)> settingsCase
+        in leakTestingSettingsCases.entries) {
+      final LeakTesting settings = settingsCase.value(LeakTesting.settings);
+      if (settings.leakDiagnosticConfig.collectRetainingPathForNotGCed) {
+        // Retaining path requires vm to be started, so skipping.
+        continue;
       }
-
-  _verifyLeaks(
-    leaks,
-    _test3TrackingOnLeaks,
-    notDisposed: 1,
-    notGCed: 1,
-    expectedContextKeys: <LeakType, List<String>>{
-      LeakType.notGCed: <String>[],
-      LeakType.notDisposed: <String>[],
-    },
-  );
-  _verifyLeaks(
-    leaks,
-    _test4TrackingOnWithCreationStackTrace,
-    notDisposed: 1,
-    notGCed: 1,
-    expectedContextKeys: <LeakType, List<String>>{
-      LeakType.notGCed: <String>['start'],
-      LeakType.notDisposed: <String>['start'],
-    },
-  );
-  _verifyLeaks(
-    leaks,
-    _test5TrackingOnWithDisposalStackTrace,
-    notDisposed: 1,
-    notGCed: 1,
-    expectedContextKeys: <LeakType, List<String>>{
-      LeakType.notGCed: <String>['disposal'],
-      LeakType.notDisposed: <String>[],
-    },
-  );
-  _verifyLeaks(
-    leaks,
-    _test8TrackingOnNotDisposed,
-    notDisposed: 1,
-    expectedContextKeys: <LeakType, List<String>>{},
-  );
-}
-
-/// Verifies [allLeaks] contain expected number of leaks for the test [testDescription].
-///
-/// [notDisposed] and [notGCed] set number for expected leaks by leak type.
-/// The method will fail if the leaks context does not contain [expectedContextKeys].
-void _verifyLeaks(
-  Leaks allLeaks,
-  String testDescription, {
-  int notDisposed = 0,
-  int notGCed = 0,
-  Map<LeakType, List<String>> expectedContextKeys = const <LeakType, List<String>>{},
-}) {
-  final Leaks testLeaks = Leaks(
-    allLeaks.byType.map(
-      (LeakType key, List<LeakReport> value) =>
-          MapEntry<LeakType, List<LeakReport>>(key, value.where((LeakReport leak) => leak.phase == testDescription).toList()),
-    ),
-  );
-
-  for (final LeakType type in expectedContextKeys.keys) {
-    final List<LeakReport> leaks = testLeaks.byType[type]!;
-    final List<String> expectedKeys = expectedContextKeys[type]!..sort();
-    for (final LeakReport leak in leaks) {
-      final List<String> actualKeys = leak.context?.keys.toList() ?? <String>[];
-      expect(actualKeys..sort(), equals(expectedKeys), reason: '$testDescription, $type');
+      final _TestExecution execution = _TestExecution(
+          settingName: settingsCase.key, test: test, settings: settings);
+      _testExecutions.add(execution);
+      testWidgets(execution.name, experimentalLeakTesting: settings,
+          (WidgetTester tester) async {
+        await test.body(tester.pumpWidget, tester.runAsync);
+      });
     }
   }
-
-  _verifyLeakList(
-    testLeaks.notDisposed,
-    notDisposed,
-    testDescription,
-  );
-  _verifyLeakList(
-    testLeaks.notGCed,
-    notGCed,
-    testDescription,
-  );
 }
 
-void _verifyLeakList(
-  List<LeakReport> list,
-  int expectedCount,
-  String testDescription,
-) {
-  expect(list.length, expectedCount, reason: testDescription);
-
-  for (final LeakReport leak in list) {
-    expect(leak.trackedClass, contains(InstrumentedDisposable.library));
-    expect(leak.trackedClass, contains('$InstrumentedDisposable'));
+void _verifyLeaks(Leaks leaks) {
+  for (final _TestExecution execution in _testExecutions) {
+    final Leaks testLeaks = leaks.byPhase[execution.name] ?? Leaks.empty();
+    execution.test.verifyLeaks(testLeaks, execution.settings,
+        testDescription: execution.name);
   }
 }