Migrate gallery/transitions_perf test to e2e (#62064)

dev/benchmarks/macrobenchmarks/test/util.dart

@@ -13,13 +13,6 @@ import 'package:macrobenchmarks/common.dart';
 import 'package:e2e/e2e.dart';
 import 'package:macrobenchmarks/main.dart' as app;
 
-/// The maximum amount of time considered safe to spend for a frame's build
-/// phase. Anything past that is in the danger of missing the frame as 60FPS.
-///
-/// Changing this doesn't re-evaluate existing summary.
-Duration kBuildBudget = const Duration(milliseconds: 16);
-// TODO(CareF): Automatically calculate the refresh budget (#61958)
-
 typedef ControlCallback = Future<void> Function(WidgetController controller);
 
 void macroPerfTestE2E(
@@ -31,10 +24,6 @@ void macroPerfTestE2E(
   ControlCallback body,
   ControlCallback setup,
 }) {
-  assert(() {
-    debugPrint(kDebugWarning);
-    return true;
-  }());
   final WidgetsBinding _binding = E2EWidgetsFlutterBinding.ensureInitialized();
   assert(_binding is E2EWidgetsFlutterBinding);
   final E2EWidgetsFlutterBinding binding = _binding as E2EWidgetsFlutterBinding;
@@ -82,140 +71,28 @@ void macroPerfTestE2E(
   }, semanticsEnabled: false, timeout: Timeout(timeout));
 }
 
+bool _firstRun = true;
+
+// TODO(CareF): move this to e2e after FrameTimingSummarizer goes into stable
+// branch (#63537)
+/// watches the [FrameTiming] of `action` and report it to the e2e binding.
 Future<void> watchPerformance(
   E2EWidgetsFlutterBinding binding,
-  Future<void> action(),
-) async {
+  Future<void> action(), {
+  String reportKey = 'performance',
+}) async {
+  assert(() {
+    if (_firstRun) {
+      debugPrint(kDebugWarning);
+      _firstRun = false;
+    }
+    return true;
+  }());
   final List<FrameTiming> frameTimings = <FrameTiming>[];
   final TimingsCallback watcher = frameTimings.addAll;
   binding.addTimingsCallback(watcher);
   await action();
   binding.removeTimingsCallback(watcher);
-  // TODO(CareF): determine if it's running on firebase and report metric online
   final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings);
-  binding.reportData = <String, dynamic>{'performance': frameTimes.summary};
-}
-
-/// This class and summarizes a list of [FrameTiming] for the performance
-/// metrics.
-class FrameTimingSummarizer {
-  factory FrameTimingSummarizer(List<FrameTiming> data) {
-    assert(data != null);
-    assert(data.isNotEmpty);
-    final List<Duration> frameBuildTime = List<Duration>.unmodifiable(
-      data.map<Duration>((FrameTiming datum) => datum.buildDuration),
-    );
-    final List<Duration> frameBuildTimeSorted = List<Duration>.from(frameBuildTime)..sort();
-    final List<Duration> frameRasterizerTime = List<Duration>.unmodifiable(
-      data.map<Duration>((FrameTiming datum) => datum.rasterDuration),
-    );
-    final List<Duration> frameRasterizerTimeSorted = List<Duration>.from(frameRasterizerTime)..sort();
-    final Duration Function(Duration, Duration) add = (Duration a, Duration b) => a + b;
-    return FrameTimingSummarizer._(
-      frameBuildTime: frameBuildTime,
-      frameRasterizerTime: frameRasterizerTime,
-      // This avarage calculation is microsecond precision, which is fine
-      // because typical values of these times are milliseconds.
-      averageFrameBuildTime: frameBuildTime.reduce(add) ~/ data.length,
-      p90FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.90),
-      p99FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.99),
-      worstFrameBuildTime: frameBuildTimeSorted.last,
-      missedFrameBuildBudget: _countExceed(frameBuildTimeSorted, kBuildBudget),
-      averageFrameRasterizerTime: frameRasterizerTime.reduce(add) ~/ data.length,
-      p90FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90),
-      p99FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90),
-      worstFrameRasterizerTime: frameRasterizerTimeSorted.last,
-      missedFrameRasterizerBudget: _countExceed(frameRasterizerTimeSorted, kBuildBudget),
-    );
-  }
-
-  const FrameTimingSummarizer._({
-    @required this.frameBuildTime,
-    @required this.frameRasterizerTime,
-    @required this.averageFrameBuildTime,
-    @required this.p90FrameBuildTime,
-    @required this.p99FrameBuildTime,
-    @required this.worstFrameBuildTime,
-    @required this.missedFrameBuildBudget,
-    @required this.averageFrameRasterizerTime,
-    @required this.p90FrameRasterizerTime,
-    @required this.p99FrameRasterizerTime,
-    @required this.worstFrameRasterizerTime,
-    @required this.missedFrameRasterizerBudget
-  });
-
-  /// List of frame build time in microseconds
-  final List<Duration> frameBuildTime;
-
-  /// List of frame rasterizer time in microseconds
-  final List<Duration> frameRasterizerTime;
-
-  /// The average value of [frameBuildTime] in milliseconds.
-  final Duration averageFrameBuildTime;
-
-  /// The 90-th percentile value of [frameBuildTime] in milliseconds
-  final Duration p90FrameBuildTime;
-
-  /// The 99-th percentile value of [frameBuildTime] in milliseconds
-  final Duration p99FrameBuildTime;
-
-  /// The largest value of [frameBuildTime] in milliseconds
-  final Duration worstFrameBuildTime;
-
-  /// Number of items in [frameBuildTime] that's greater than [kBuildBudget]
-  final int missedFrameBuildBudget;
-
-  /// The average value of [frameRasterizerTime] in milliseconds.
-  final Duration averageFrameRasterizerTime;
-
-  /// The 90-th percentile value of [frameRasterizerTime] in milliseconds.
-  final Duration p90FrameRasterizerTime;
-
-  /// The 99-th percentile value of [frameRasterizerTime] in milliseconds.
-  final Duration p99FrameRasterizerTime;
-
-  /// The largest value of [frameRasterizerTime] in milliseconds.
-  final Duration worstFrameRasterizerTime;
-
-  /// Number of items in [frameRasterizerTime] that's greater than [kBuildBudget]
-  final int missedFrameRasterizerBudget;
-
-  Map<String, dynamic> get summary => <String, dynamic>{
-    'average_frame_build_time_millis':
-        averageFrameBuildTime.inMicroseconds / 1E3,
-    '90th_percentile_frame_build_time_millis':
-        p90FrameBuildTime.inMicroseconds / 1E3,
-    '99th_percentile_frame_build_time_millis':
-        p99FrameBuildTime.inMicroseconds / 1E3,
-    'worst_frame_build_time_millis':
-        worstFrameBuildTime.inMicroseconds / 1E3,
-    'missed_frame_build_budget_count': missedFrameBuildBudget,
-    'average_frame_rasterizer_time_millis':
-        averageFrameRasterizerTime.inMicroseconds / 1E3,
-    '90th_percentile_frame_rasterizer_time_millis':
-        p90FrameRasterizerTime.inMicroseconds / 1E3,
-    '99th_percentile_frame_rasterizer_time_millis':
-        p99FrameRasterizerTime.inMicroseconds / 1E3,
-    'worst_frame_rasterizer_time_millis':
-        worstFrameRasterizerTime.inMicroseconds / 1E3,
-    'missed_frame_rasterizer_budget_count': missedFrameRasterizerBudget,
-    'frame_count': frameBuildTime.length,
-    'frame_build_times': frameBuildTime
-        .map<int>((Duration datum) => datum.inMicroseconds).toList(),
-    'frame_rasterizer_times': frameRasterizerTime
-        .map<int>((Duration datum) => datum.inMicroseconds).toList(),
-  };
-}
-
-// The following helper functions require data sorted
-
-// return the 100*p-th percentile of the data
-T _findPercentile<T>(List<T> data, double p) {
-  assert(p >= 0 && p <= 1);
-  return data[((data.length - 1) * p).round()];
-}
-
-// return the number of items in data that > threshold
-int _countExceed<T extends Comparable<T>>(List<T> data, T threshold) {
-  return data.length - data.indexWhere((T datum) => datum.compareTo(threshold) > 0);
+  binding.reportData = <String, dynamic>{reportKey: frameTimes.summary};
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/gallery.dart';
+import 'package:flutter_devicelab/framework/adb.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<void> main() async {
+  deviceOperatingSystem = DeviceOperatingSystem.android;
+  await task(createGalleryTransitionE2ETest());
+}

dev/devicelab/lib/tasks/gallery.dart

@@ -15,11 +15,34 @@ TaskFunction createGalleryTransitionTest({ bool semanticsEnabled = false }) {
   return GalleryTransitionTest(semanticsEnabled: semanticsEnabled);
 }
 
+TaskFunction createGalleryTransitionE2ETest({ bool semanticsEnabled = false }) {
+  return GalleryTransitionTest(
+    semanticsEnabled: semanticsEnabled,
+    testFile: 'transitions_perf_e2e',
+    needFullTimeline: false,
+    timelineSummaryFile: 'e2e_perf_summary',
+    transitionDurationFile: null,
+    driverFile: 'transitions_perf_e2e_test',
+  );
+}
+
 class GalleryTransitionTest {
 
-  GalleryTransitionTest({ this.semanticsEnabled = false });
+  GalleryTransitionTest({
+    this.semanticsEnabled = false,
+    this.testFile = 'transitions_perf',
+    this.needFullTimeline = true,
+    this.timelineSummaryFile = 'transitions.timeline_summary',
+    this.transitionDurationFile = 'transition_durations.timeline',
+    this.driverFile,
+  });
 
   final bool semanticsEnabled;
+  final bool needFullTimeline;
+  final String testFile;
+  final String timelineSummaryFile;
+  final String transitionDurationFile;
+  final String driverFile;
 
   Future<TaskResult> call() async {
     final Device device = await devices.workingDevice;
@@ -31,41 +54,41 @@ class GalleryTransitionTest {
       await flutter('packages', options: <String>['get']);
 
       final String testDriver = semanticsEnabled
-          ? 'transitions_perf_with_semantics.dart'
-          : 'transitions_perf.dart';
+          ? '${testFile}_with_semantics.dart'
+          : '$testFile.dart';
 
       await flutter('drive', options: <String>[
         '--profile',
-        '--trace-startup',
+        if (needFullTimeline)
+          '--trace-startup',
         '-t',
         'test_driver/$testDriver',
+        if (driverFile != null)
+          ...<String>['--driver', 'test_driver/$driverFile.dart'],
         '-d',
         deviceId,
       ]);
     });
 
-    // Route paths contains slashes, which Firebase doesn't accept in keys, so we
-    // remove them.
-    final Map<String, dynamic> original = json.decode(
-      file('${galleryDirectory.path}/build/transition_durations.timeline.json').readAsStringSync(),
-    ) as Map<String, dynamic>;
-    final Map<String, List<int>> transitions = <String, List<int>>{};
-    for (final String key in original.keys) {
-      transitions[key.replaceAll('/', '')] = List<int>.from(original[key] as List<dynamic>);
-    }
-
     final Map<String, dynamic> summary = json.decode(
-      file('${galleryDirectory.path}/build/transitions.timeline_summary.json').readAsStringSync(),
+      file('${galleryDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(),
     ) as Map<String, dynamic>;
 
-    final Map<String, dynamic> data = <String, dynamic>{
-      'transitions': transitions,
-      'missed_transition_count': _countMissedTransitions(transitions),
-      ...summary,
-    };
+    if (transitionDurationFile != null) {
+      final Map<String, dynamic> original = json.decode(
+        file('${galleryDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(),
+      ) as Map<String, dynamic>;
+      final Map<String, List<int>> transitions = <String, List<int>>{};
+      for (final String key in original.keys) {
+        transitions[key] = List<int>.from(original[key] as List<dynamic>);
+      }
+      summary['transitions'] = transitions;
+      summary['missed_transition_count'] = _countMissedTransitions(transitions);
+    }
 
-    return TaskResult.success(data, benchmarkScoreKeys: <String>[
-      'missed_transition_count',
+    return TaskResult.success(summary, benchmarkScoreKeys: <String>[
+      if (transitionDurationFile != null)
+        'missed_transition_count',
       'average_frame_build_time_millis',
       'worst_frame_build_time_millis',
       '90th_percentile_frame_build_time_millis',

dev/devicelab/manifest.yaml

@@ -411,15 +411,6 @@ tasks:
     stage: devicelab
     required_agent_capabilities: ["linux/android"]
 
-  # flutter_gallery_instrumentation_test:
-  #   description: >
-  #     Same as flutter_gallery__transition_perf but uses Android instrumentation
-  #     framework, and therefore does not require a host computer to run. This
-  #     test can run on off-the-shelf infrastructures, such as Firebase Test Lab.
-  #   stage: devicelab
-  #   required_agent_capabilities: ["linux/android"]
-  #   flaky: true
-
   linux_chrome_dev_mode:
     description: >
       Run flutter web on the devicelab and hot restart.
@@ -763,6 +754,14 @@ tasks:
     stage: devicelab
     required_agent_capabilities: ["linux/android"]
 
+  flutter_gallery__transition_perf_e2e:
+    description: >
+      Measures the performance of screen transitions in Flutter Gallery on
+      Android with e2e.
+    stage: devicelab
+    required_agent_capabilities: ["linux/android"]
+    flaky: true
+
   flutter_gallery_sksl_warmup__transition_perf:
     description: >
       Measures the runtime performance of Flutter gallery transitions on Android

dev/integration_tests/flutter_gallery/android/app/src/androidTest/java/io/flutter/demo/gallery/FlutterGalleryInstrumentationTest.java

-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package io.flutter.demo.gallery;
-
-import android.support.test.filters.LargeTest;
-import android.support.test.rule.ActivityTestRule;
-import android.support.test.runner.AndroidJUnit4;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
-
-@RunWith(AndroidJUnit4.class)
-@LargeTest
-public class FlutterGalleryInstrumentationTest {
-  @Rule
-  public ActivityTestRule<MainActivity> mActivityRule =
-      new ActivityTestRule<>(MainActivity.class);
-
-  private MainActivity activity;
-
-  @Before
-  public void setUp() {
-    activity = mActivityRule.getActivity();
-  }
-
-  @Test
-  public void activityLoaded() throws Exception {
-    FlutterGalleryInstrumentation instrumentation = activity.getInstrumentation();
-    instrumentation.waitForTestToFinish();
-    assertThat(instrumentation.isTestSuccessful(), is(true));
-  }
-}

dev/integration_tests/flutter_gallery/android/app/src/androidTest/java/io/flutter/demo/gallery/MainActivityTest.java

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.demo.gallery;
+
+import androidx.test.rule.ActivityTestRule;
+import dev.flutter.plugins.e2e.FlutterTestRunner;
+import org.junit.Rule;
+import org.junit.runner.RunWith;
+
+@RunWith(FlutterTestRunner.class)
+public class MainActivityTest {
+  @Rule
+  public ActivityTestRule<MainActivity> rule = new ActivityTestRule<>(MainActivity.class, true, false);
+}

dev/integration_tests/flutter_gallery/lib/demo_lists.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Demos for which timeline data will be collected using
+// FlutterDriver.traceAction().
+//
+// Warning: The number of tests executed with timeline collection enabled
+// significantly impacts heap size of the running app. When run with
+// --trace-startup, as we do in this test, the VM stores trace events in an
+// endless buffer instead of a ring buffer.
+//
+// These names must match GalleryItem titles from kAllGalleryDemos
+// in dev/integration_tests/flutter_gallery/lib/gallery/demos.dart
+const List<String> kProfiledDemos = <String>[
+  'Shrine@Studies',
+  'Contact profile@Studies',
+  'Animation@Studies',
+  'Bottom navigation@Material',
+  'Buttons@Material',
+  'Cards@Material',
+  'Chips@Material',
+  'Dialogs@Material',
+  'Pickers@Material',
+];
+
+// There are 3 places where the Gallery demos are traversed.
+// 1- In widget tests such as dev/integration_tests/flutter_gallery/test/smoke_test.dart
+// 2- In driver tests such as dev/integration_tests/flutter_gallery/test_driver/transitions_perf_test.dart
+// 3- In on-device instrumentation tests such as dev/integration_tests/flutter_gallery/test/live_smoketest.dart
+//
+// If you change navigation behavior in the Gallery or in the framework, make
+// sure all 3 are covered.
+
+// Demos that will be backed out of within FlutterDriver.runUnsynchronized();
+//
+// These names must match GalleryItem titles from kAllGalleryDemos
+// in dev/integration_tests/flutter_gallery/lib/gallery/demos.dart
+const List<String> kUnsynchronizedDemos = <String>[
+  'Progress indicators@Material',
+  'Activity Indicator@Cupertino',
+  'Video@Media',
+];

dev/integration_tests/flutter_gallery/pubspec.yaml

@@ -51,6 +51,7 @@ dev_dependencies:
   flutter_goldens:
     sdk: flutter
   test: 1.16.0-nullsafety.1
+  e2e: 0.7.0
 
   _fe_analyzer_shared: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   analyzer: 0.39.17 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -275,4 +276,4 @@ flutter:
       - asset: packages/flutter_gallery_assets/fonts/merriweather/Merriweather-Regular.ttf
       - asset: packages/flutter_gallery_assets/fonts/merriweather/Merriweather-Light.ttf
 
-# PUBSPEC CHECKSUM: 6b77
+# PUBSPEC CHECKSUM: 7fc2

dev/integration_tests/flutter_gallery/test_driver/e2e_utils.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:ui';
+
+import 'package:flutter/scheduler.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+import 'package:e2e/e2e.dart';
+
+bool _firstRun = true;
+
+// TODO(CareF): move this to e2e after FrameTimingSummarizer goes into stable
+// branch (#63537)
+/// watches the [FrameTiming] of `action` and report it to the e2e binding.
+Future<void> watchPerformance(
+  E2EWidgetsFlutterBinding binding,
+  Future<void> action(), {
+  String reportKey = 'performance',
+}) async {
+  assert(() {
+    if (_firstRun) {
+      debugPrint(kDebugWarning);
+      _firstRun = false;
+    }
+    return true;
+  }());
+  final List<FrameTiming> frameTimings = <FrameTiming>[];
+  final TimingsCallback watcher = frameTimings.addAll;
+  binding.addTimingsCallback(watcher);
+  await action();
+  binding.removeTimingsCallback(watcher);
+  final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings);
+  binding.reportData = <String, dynamic>{reportKey: frameTimes.summary};
+}

dev/integration_tests/flutter_gallery/test_driver/transitions_perf_e2e.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:ui';
+
+import 'package:flutter/cupertino.dart';
+import 'package:flutter/widgets.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+import 'package:e2e/e2e.dart';
+
+import 'package:flutter_gallery/gallery/app.dart' show GalleryApp;
+import 'package:flutter_gallery/gallery/demos.dart';
+import 'package:flutter_gallery/demo_lists.dart';
+
+import 'e2e_utils.dart';
+
+const List<String> kSkippedDemos = <String>[];
+
+// All of the gallery demos, identified as "title@category".
+//
+// These names are reported by the test app, see _handleMessages()
+// in transitions_perf.dart.
+List<String> _allDemos = kAllGalleryDemos.map(
+  (GalleryDemo demo) => '${demo.title}@${demo.category.name}',
+).toList();
+
+/// Scrolls each demo menu item into view, launches it, then returns to the
+/// home screen twice.
+Future<void> runDemos(List<String> demos, WidgetController controller) async {
+  final Finder demoList = find.byType(Scrollable);
+  String currentDemoCategory;
+
+  for (final String demo in demos) {
+    if (kSkippedDemos.contains(demo))
+      continue;
+
+    final String demoName = demo.substring(0, demo.indexOf('@'));
+    final String demoCategory = demo.substring(demo.indexOf('@') + 1);
+    print('> $demo');
+    await controller.pump(const Duration(milliseconds: 250));
+
+    if (currentDemoCategory == null) {
+      await controller.tap(find.text(demoCategory));
+      await controller.pumpAndSettle();
+    } else if (currentDemoCategory != demoCategory) {
+      await controller.tap(find.byTooltip('Back'));
+      await controller.pumpAndSettle();
+      await controller.tap(find.text(demoCategory));
+      await controller.pumpAndSettle();
+      // Scroll back to the top
+      await controller.drag(demoList, const Offset(0.0, 10000.0));
+      await controller.pumpAndSettle(const Duration(milliseconds: 100));
+    }
+    currentDemoCategory = demoCategory;
+
+    final Finder demoItem = find.text(demoName);
+    await controller.scrollUntilVisible(demoItem, 48.0);
+    await controller.pumpAndSettle();
+
+    Future<void> pageBack() {
+      Finder backButton = find.byTooltip('Back');
+      if (backButton.evaluate().isEmpty) {
+        backButton = find.byType(CupertinoNavigationBarBackButton);
+      }
+      return controller.tap(backButton);
+    }
+
+    for (int i = 0; i < 2; i += 1) {
+      await controller.tap(demoItem); // Launch the demo
+
+      if (kUnsynchronizedDemos.contains(demo)) {
+        // These tests have animation, pumpAndSettle cannot be used.
+        // This time is questionable. 300ms is the tested reasonable result.
+        await controller.pump(const Duration(milliseconds: 300));
+        await controller.pump();
+        await pageBack();
+      } else {
+        await controller.pumpAndSettle();
+        // page back
+        await pageBack();
+      }
+      await controller.pumpAndSettle();
+    }
+
+    print('< Success');
+  }
+
+  // Return to the home screen
+  await controller.tap(find.byTooltip('Back'));
+  await controller.pumpAndSettle();
+}
+
+void main([List<String> args = const <String>[]]) {
+  final bool withSemantics = args.contains('--with_semantics');
+  final E2EWidgetsFlutterBinding binding =
+      E2EWidgetsFlutterBinding.ensureInitialized() as E2EWidgetsFlutterBinding;
+  binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;
+  group('flutter gallery transitions on e2e', () {
+    testWidgets('find.bySemanticsLabel', (WidgetTester tester) async {
+      runApp(const GalleryApp(testMode: true));
+      await tester.pumpAndSettle();
+      final int id = tester.getSemantics(find.bySemanticsLabel('Material')).id;
+      expect(id, greaterThan(-1));
+    }, skip: !withSemantics, semanticsEnabled: true);
+
+    testWidgets(
+      'all demos',
+      (WidgetTester tester) async {
+        runApp(const GalleryApp(testMode: true));
+        await tester.pumpAndSettle();
+        // Collect timeline data for just a limited set of demos to avoid OOMs.
+        await watchPerformance(binding, () async {
+          await runDemos(kProfiledDemos, tester);
+        });
+
+        // Execute the remaining tests.
+        final Set<String> unprofiledDemos = Set<String>.from(_allDemos)
+          ..removeAll(kProfiledDemos);
+        await runDemos(unprofiledDemos.toList(), tester);
+      },
+      timeout: const Timeout(Duration(minutes: 5)),
+      semanticsEnabled: withSemantics,
+    );
+  });
+}

dev/integration_tests/flutter_gallery/test_driver/transitions_perf_e2e_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:e2e/e2e_driver.dart' as driver;
+
+Future<void> main() => driver.e2eDriver(
+  timeout: const Duration(minutes: 5),
+  responseDataCallback: (Map<String, dynamic> data) async {
+    await driver.writeResponseData(
+      data['performance'] as Map<String, dynamic>,
+      testOutputFilename: 'e2e_perf_summary',
+    );
+  }
+);

dev/integration_tests/flutter_gallery/test_driver/transitions_perf_e2e_with_semantics.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'transitions_perf_e2e.dart' as transitions_perf;
+
+void main() {
+  transitions_perf.main(<String>['--with_semantics']);
+}

dev/integration_tests/flutter_gallery/test_driver/transitions_perf_test.dart

@@ -11,47 +11,9 @@ import 'package:flutter_driver/flutter_driver.dart';
 import 'package:path/path.dart' as path;
 import 'package:test/test.dart' hide TypeMatcher, isInstanceOf;
 
-const FileSystem _fs = LocalFileSystem();
+import 'package:flutter_gallery/demo_lists.dart';
 
-// Demos for which timeline data will be collected using
-// FlutterDriver.traceAction().
-//
-// Warning: The number of tests executed with timeline collection enabled
-// significantly impacts heap size of the running app. When run with
-// --trace-startup, as we do in this test, the VM stores trace events in an
-// endless buffer instead of a ring buffer.
-//
-// These names must match GalleryItem titles from kAllGalleryDemos
-// in dev/integration_tests/flutter_gallery/lib/gallery/demos.dart
-const List<String> kProfiledDemos = <String>[
-  'Shrine@Studies',
-  'Contact profile@Studies',
-  'Animation@Studies',
-  'Bottom navigation@Material',
-  'Buttons@Material',
-  'Cards@Material',
-  'Chips@Material',
-  'Dialogs@Material',
-  'Pickers@Material',
-];
-
-// There are 3 places where the Gallery demos are traversed.
-// 1- In widget tests such as dev/integration_tests/flutter_gallery/test/smoke_test.dart
-// 2- In driver tests such as dev/integration_tests/flutter_gallery/test_driver/transitions_perf_test.dart
-// 3- In on-device instrumentation tests such as dev/integration_tests/flutter_gallery/test/live_smoketest.dart
-//
-// If you change navigation behavior in the Gallery or in the framework, make
-// sure all 3 are covered.
-
-// Demos that will be backed out of within FlutterDriver.runUnsynchronized();
-//
-// These names must match GalleryItem titles from kAllGalleryDemos
-// in dev/integration_tests/flutter_gallery/lib/gallery/demos.dart
-const List<String> kUnsynchronizedDemos = <String>[
-  'Progress indicators@Material',
-  'Activity Indicator@Cupertino',
-  'Video@Media',
-];
+const FileSystem _fs = LocalFileSystem();
 
 const List<String> kSkippedDemos = <String>[];