Make gallery tests more robust (#15957)

examples/flutter_gallery/lib/demo/material/backdrop_demo.dart

@@ -14,6 +14,8 @@ class Category {
   const Category({ this.title, this.assets });
   final String title;
   final List<String> assets;
+  @override
+  String toString() => '$runtimeType("$title")';
 }
 
 const List<Category> allCategories = const <Category>[
@@ -178,10 +180,13 @@ class BackdropPanel extends StatelessWidget {
               alignment: AlignmentDirectional.centerStart,
               child: new DefaultTextStyle(
                 style: theme.textTheme.subhead,
+                child: new Tooltip(
+                  message: 'Tap to dismiss',
                   child: title,
                 ),
               ),
             ),
+          ),
           const Divider(height: 1.0),
           new Expanded(child: child),
         ],

examples/flutter_gallery/lib/gallery/item.dart

@@ -118,7 +118,7 @@ List<GalleryItem> _buildGalleryItems() {
     ),
     new GalleryItem(
       title: 'Data tables',
-      subtitle: 'Data tables',
+      subtitle: 'Rows and columns',
       category: 'Material Components',
       routeName: DataTableDemo.routeName,
       buildRoute: (BuildContext context) => new DataTableDemo(),

examples/flutter_gallery/test/live_smoketest.dart

@@ -11,30 +11,55 @@ import 'package:flutter/scheduler.dart';
 import 'package:flutter_test/flutter_test.dart';
 
 import 'package:flutter_gallery/gallery/app.dart';
+import 'package:flutter_gallery/gallery/item.dart';
 
-/// Reports success or failure to the native code.
+// Reports success or failure to the native code.
 const MethodChannel _kTestChannel = const MethodChannel('io.flutter.demo.gallery/TestLifecycleListener');
 
+// The titles for all of the Gallery demos.
+final List<String> _kAllDemos = kAllGalleryItems.map((GalleryItem item) => item.title).toList();
+
+// We don't want to wait for animations to complete before tapping the
+// back button in the demos with these titles.
+const List<String> _kUnsynchronizedDemos = const <String>[
+  'Progress indicators',
+  'Activity Indicator',
+  'Video',
+];
+
+// These demos can't be backed out of by tapping a button whose
+// tooltip is 'Back'.
+const List<String> _kSkippedDemos = const <String>[
+  'Backdrop',
+  'Pull to refresh',
+];
+
 Future<Null> main() async {
   try {
-    runApp(const GalleryApp());
+    // Verify that _kUnsynchronizedDemos and _kSkippedDemos identify
+    // demos that actually exist.
+    if (!new Set<String>.from(_kAllDemos).containsAll(_kUnsynchronizedDemos))
+      fail('Unrecognized demo names in _kUnsynchronizedDemos: $_kUnsynchronizedDemos');
+    if (!new Set<String>.from(_kAllDemos).containsAll(_kSkippedDemos))
+      fail('Unrecognized demo names in _kSkippedDemos: $_kSkippedDemos');
 
-    const Duration kWaitBetweenActions = const Duration(milliseconds: 250);
+    runApp(const GalleryApp());
     final _LiveWidgetController controller = new _LiveWidgetController();
-
-    for (Demo demo in demos) {
-      print('Testing "${demo.title}" demo');
-      final Finder menuItem = find.text(demo.title);
+    for (String demo in _kAllDemos) {
+      print('Testing "$demo" demo');
+      final Finder menuItem = find.text(demo);
       await controller.scrollIntoView(menuItem, alignment: 0.5);
-      await new Future<Null>.delayed(kWaitBetweenActions);
+
+      if (_kSkippedDemos.contains(demo)) {
+        print('> skipped $demo');
+        continue;
+      }
 
       for (int i = 0; i < 2; i += 1) {
         await controller.tap(menuItem); // Launch the demo
-        await new Future<Null>.delayed(kWaitBetweenActions);
-        controller.frameSync = demo.synchronized;
+        controller.frameSync = !_kUnsynchronizedDemos.contains(demo);
         await controller.tap(find.byTooltip('Back'));
         controller.frameSync = true;
-        await new Future<Null>.delayed(kWaitBetweenActions);
       }
       print('Success');
     }
@@ -45,70 +70,6 @@ Future<Null> main() async {
   }
 }
 
-class Demo {
-  const Demo(this.title, {this.synchronized = true});
-
-  /// The title of the demo.
-  final String title;
-
-  /// True if frameSync should be enabled for this test.
-  final bool synchronized;
-}
-
-// Warning: this list must be kept in sync with the value of
-// kAllGalleryItems.map((GalleryItem item) => item.title).toList();
-const List<Demo> demos = const <Demo>[
-  // Demos
-  const Demo('Shrine'),
-  const Demo('Contact profile'),
-  const Demo('Animation'),
-
-  // Material Components
-  const Demo('Bottom navigation'),
-  const Demo('Buttons'),
-  const Demo('Cards'),
-  const Demo('Chips'),
-  const Demo('Date and time pickers'),
-  const Demo('Dialog'),
-  const Demo('Drawer'),
-  const Demo('Expand/collapse list control'),
-  const Demo('Expansion panels'),
-  const Demo('Floating action button'),
-  const Demo('Grid'),
-  const Demo('Icons'),
-  const Demo('Leave-behind list items'),
-  const Demo('List'),
-  const Demo('Menus'),
-  const Demo('Modal bottom sheet'),
-  const Demo('Page selector'),
-  const Demo('Persistent bottom sheet'),
-  const Demo('Progress indicators', synchronized: false),
-  const Demo('Pull to refresh'),
-  const Demo('Scrollable tabs'),
-  const Demo('Selection controls'),
-  const Demo('Sliders'),
-  const Demo('Snackbar'),
-  const Demo('Tabs'),
-  const Demo('Text fields'),
-  const Demo('Tooltips'),
-
-  // Cupertino Components
-  const Demo('Activity Indicator', synchronized: false),
-  const Demo('Buttons'),
-  const Demo('Dialogs'),
-  const Demo('Navigation'),
-  const Demo('Sliders'),
-  const Demo('Switches'),
-
-  // Media
-  const Demo('Animated images'),
-
-  // Style
-  const Demo('Colors'),
-  const Demo('Typography'),
-];
-
-
 class _LiveWidgetController {
 
   final WidgetController _controller = new WidgetController(WidgetsBinding.instance);

examples/flutter_gallery/test/smoke_test.dart

@@ -102,11 +102,19 @@ Future<Null> smokeDemo(WidgetTester tester, String routeName) async {
   await tester.pump();
   await tester.pump(const Duration(milliseconds: 400));
 
+  // This demo's back button isn't initially visible.
+  if (routeName == '/material/backdrop') {
+    await tester.tap(find.byTooltip('Tap to dismiss'));
+    await tester.pumpAndSettle();
+  }
+
   // Go back
   await tester.pageBack();
+  await tester.pumpAndSettle();
   await tester.pump(); // Start the pop "back" operation.
   await tester.pump(); // Complete the willPop() Future.
   await tester.pump(const Duration(milliseconds: 400)); // Wait until it has finished.
+
   return null;
 }
 
@@ -126,8 +134,6 @@ Future<Null> runSmokeTest(WidgetTester tester) async {
     final Finder finder = findGalleryItemByRouteName(tester, routeName);
     Scrollable.ensureVisible(tester.element(finder), alignment: 0.5);
     await tester.pumpAndSettle();
-    if (routeName == '/material/backdrop')
-      continue;
     await smokeDemo(tester, routeName);
     tester.binding.debugAssertNoTransientCallbacks('A transient callback was still active after leaving route $routeName');
   }

examples/flutter_gallery/test_driver/memory_nav_test.dart

@@ -23,18 +23,14 @@ void main() {
       final SerializableFinder menuItem = find.text('Text fields');
       driver.waitFor(menuItem).then<Null>((Null value) async {
         scroll = false;
-        await new Future<Null>.delayed(kWaitBetweenActions);
         for (int i = 0; i < 15; i++) {
           await driver.tap(menuItem);
-          await new Future<Null>.delayed(kWaitBetweenActions);
           await driver.tap(find.byTooltip('Back'));
-          await new Future<Null>.delayed(kWaitBetweenActions);
         }
         completer.complete();
       });
       while (scroll) {
         await driver.scroll(find.text('Flutter Gallery'), 0.0, -500.0, const Duration(milliseconds: 80));
-        await new Future<Null>.delayed(kWaitBetweenActions);
       }
       await completer.future;
     }, timeout: const Timeout(const Duration(minutes: 1)));

examples/flutter_gallery/test_driver/transitions_perf.dart

@@ -2,10 +2,21 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:async';
+import 'dart:convert' show JsonEncoder;
+
 import 'package:flutter_driver/driver_extension.dart';
+import 'package:flutter_gallery/gallery/item.dart';
 import 'package:flutter_gallery/main.dart' as app;
 
+Future<String> _handleMessages(String message) async {
+  assert(message == 'demoNames');
+  return const JsonEncoder.withIndent('  ').convert(
+    kAllGalleryItems.map((GalleryItem item) => item.title).toList(),
+  );
+}
+
 void main() {
-  enableFlutterDriverExtension();
+  enableFlutterDriverExtension(handler: _handleMessages);
   app.main();
 }

examples/flutter_gallery/test_driver/transitions_perf_test.dart

@@ -3,7 +3,7 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert' show JsonEncoder;
+import 'dart:convert' show JsonEncoder, JsonDecoder;
 
 import 'package:file/file.dart';
 import 'package:file/local.dart';
@@ -11,81 +11,46 @@ import 'package:flutter_driver/flutter_driver.dart';
 import 'package:path/path.dart' as path;
 import 'package:test/test.dart';
 
-class Demo {
-  const Demo(this.title, {this.synchronized = true, this.profiled = false});
-
-  /// The title of the demo.
-  final String title;
-
-  /// True if frameSync should be enabled for this test.
-  final bool synchronized;
-
-  // True if timeline data should be collected for this test.
-  //
-  // Warning: The number of tests executed with timeline collection enabled
-  // significantly impacts heap size of the running app. When run with
-  // --trace-startup, as we do in this test, the VM stores trace events in an
-  // endless buffer instead of a ring buffer.
-  final bool profiled;
-}
+const FileSystem _fs = const LocalFileSystem();
 
-// Warning: this list must be kept in sync with the value of
-// kAllGalleryItems.map((GalleryItem item) => item.title).toList();
-const List<Demo> demos = const <Demo>[
-  // Demos
-  const Demo('Shrine', profiled: true),
-  const Demo('Contact profile', profiled: true),
-  const Demo('Animation', profiled: true),
-
-  // Material Components
-  const Demo('Bottom navigation', profiled: true),
-  const Demo('Buttons', profiled: true),
-  const Demo('Cards', profiled: true),
-  const Demo('Chips', profiled: true),
-  const Demo('Date and time pickers', profiled: true),
-  const Demo('Dialog', profiled: true),
-  const Demo('Drawer'),
-  const Demo('Expand/collapse list control'),
-  const Demo('Expansion panels'),
-  const Demo('Floating action button'),
-  const Demo('Grid'),
-  const Demo('Icons'),
-  const Demo('Leave-behind list items'),
-  const Demo('List'),
-  const Demo('Menus'),
-  const Demo('Modal bottom sheet'),
-  const Demo('Page selector'),
-  const Demo('Persistent bottom sheet'),
-  const Demo('Progress indicators', synchronized: false),
-  const Demo('Pull to refresh'),
-  const Demo('Scrollable tabs'),
-  const Demo('Selection controls'),
-  const Demo('Sliders'),
-  const Demo('Snackbar'),
-  const Demo('Tabs'),
-  const Demo('Text fields'),
-  const Demo('Tooltips'),
-
-  // Cupertino Components
-  const Demo('Activity Indicator', synchronized: false),
-  const Demo('Buttons'),
-  const Demo('Dialogs'),
-  const Demo('Navigation'),
-  const Demo('Pickers'),
-  const Demo('Sliders'),
-  const Demo('Switches'),
-
-  // Media
-  const Demo('Animated images'),
-
-  // Style
-  const Demo('Colors'),
-  const Demo('Typography'),
+// Demos for which timeline data will be collected using
+// FlutterDriver.traceAction().
+//
+// Warning: The number of tests executed with timeline collection enabled
+// significantly impacts heap size of the running app. When run with
+// --trace-startup, as we do in this test, the VM stores trace events in an
+// endless buffer instead of a ring buffer.
+//
+// These names must match GalleryItem titles from  kAllGalleryItems
+// in examples/flutter_gallery/lib/gallery.item.dart
+const List<String> kProfiledDemos = const <String>[
+  'Shrine',
+  'Contact profile',
+  'Animation',
+  'Bottom navigation',
+  'Buttons',
+  'Cards',
+  'Chips',
+  'Date and time pickers',
+  'Dialog',
 ];
 
-const FileSystem _fs = const LocalFileSystem();
+// Demos that will be backed out of within FlutterDriver.runUnsynchronized();
+//
+// These names must match GalleryItem titles from  kAllGalleryItems
+// in examples/flutter_gallery/lib/gallery.item.dart
+const List<String> kUnsynchronizedDemos = const <String>[
+  'Progress indicators',
+  'Activity Indicator',
+  'Video',
+];
 
-const Duration kWaitBetweenActions = const Duration(milliseconds: 250);
+// All of the gallery demo titles in the order they appear on the
+// gallery home page.
+//
+// These names are reported by the test app, see _handleMessages()
+// in transitions_perf.dart.
+List<String> _allDemos = <String>[];
 
 /// Extracts event data from [events] recorded by timeline, validates it, turns
 /// it into a histogram, and saves to a JSON file.
@@ -155,25 +120,29 @@ Future<Null> saveDurationsHistogram(List<Map<String, dynamic>> events, String ou
 
 /// Scrolls each demo menu item into view, launches it, then returns to the
 /// home screen twice.
-Future<Null> runDemos(Iterable<Demo> demos, FlutterDriver driver) async {
-  for (Demo demo in demos) {
-    print('Testing "${demo.title}" demo');
-    final SerializableFinder menuItem = find.text(demo.title);
-    await driver.scrollIntoView(menuItem, alignment: 0.5);
-    await new Future<Null>.delayed(kWaitBetweenActions);
+Future<Null> runDemos(List<String> demos, FlutterDriver driver) async {
+  for (String demo in demos) {
+    print('Testing "$demo" demo');
+    final SerializableFinder menuItem = find.text(demo);
+    await driver.scrollUntilVisible(find.byType('CustomScrollView'), menuItem,
+      dyScroll: -48.0,
+      alignment: 0.5,
+    );
 
     for (int i = 0; i < 2; i += 1) {
       await driver.tap(menuItem); // Launch the demo
-      await new Future<Null>.delayed(kWaitBetweenActions);
-      if (demo.synchronized) {
-        await driver.tap(find.byTooltip('Back'));
-      } else {
+
+      // This demo's back button isn't initially visible.
+      if (demo == 'Backdrop')
+        await driver.tap(find.byTooltip('Tap to dismiss'));
+
+      if (kUnsynchronizedDemos.contains(demo)) {
         await driver.runUnsynchronized<Future<Null>>(() async {
-          await new Future<Null>.delayed(kWaitBetweenActions);
           await driver.tap(find.byTooltip('Back'));
         });
+      } else {
+        await driver.tap(find.byTooltip('Back'));
       }
-      await new Future<Null>.delayed(kWaitBetweenActions);
     }
     print('Success');
   }
@@ -184,10 +153,16 @@ void main([List<String> args = const <String>[]]) {
     FlutterDriver driver;
     setUpAll(() async {
       driver = await FlutterDriver.connect();
+
       if (args.contains('--with_semantics')) {
         print('Enabeling semantics...');
         await driver.setSemantics(true);
       }
+
+      // See _handleMessages() in transitions_perf.dart.
+      _allDemos = const JsonDecoder().convert(await driver.requestData('demoNames'));
+      if (_allDemos.isEmpty)
+        throw 'no demo names found';
     });
 
     tearDownAll(() async {
@@ -197,14 +172,15 @@ void main([List<String> args = const <String>[]]) {
 
     test('all demos', () async {
       // Collect timeline data for just a limited set of demos to avoid OOMs.
-      final Timeline timeline = await driver.traceAction(() async {
-        final Iterable<Demo> profiledDemos = demos.where((Demo demo) => demo.profiled);
-        await runDemos(profiledDemos, driver);
+      final Timeline timeline = await driver.traceAction(
+        () async {
+          await runDemos(kProfiledDemos, driver);
         },
         streams: const <TimelineStream>[
           TimelineStream.dart,
           TimelineStream.embedder,
-      ]);
+        ],
+      );
 
       // Save the duration (in microseconds) of the first timeline Frame event
       // that follows a 'Start Transition' event. The Gallery app adds a
@@ -214,9 +190,15 @@ void main([List<String> args = const <String>[]]) {
       final String histogramPath = path.join(testOutputsDirectory, 'transition_durations.timeline.json');
       await saveDurationsHistogram(timeline.json['traceEvents'], histogramPath);
 
+      // Scroll back to the top
+      await driver.scrollUntilVisible(find.byType('CustomScrollView'), find.text(_allDemos[0]),
+        dyScroll: 200.0,
+        alignment: 0.0
+      );
+
       // Execute the remaining tests.
-      final Iterable<Demo> unprofiledDemos = demos.where((Demo demo) => !demo.profiled);
-      await runDemos(unprofiledDemos, driver);
+      final Set<String> unprofiledDemos = new Set<String>.from(_allDemos)..removeAll(kProfiledDemos);
+      await runDemos(unprofiledDemos.toList(), driver);
 
     }, timeout: const Timeout(const Duration(minutes: 5)));
   });

packages/flutter_driver/lib/src/driver/driver.dart

@@ -409,10 +409,71 @@ class FlutterDriver {
 
   /// Scrolls the Scrollable ancestor of the widget located by [finder]
   /// until the widget is completely visible.
+  ///
+  /// If the widget located by [finder] is contained by a scrolling widget
+  /// that lazily creates its children, like [ListView] or [CustomScrollView],
+  /// then this method may fail because [finder] doesn't actually exist.
+  /// The [scrollUntilVisible] method can be used in this case.
   Future<Null> scrollIntoView(SerializableFinder finder, { double alignment: 0.0, Duration timeout }) async {
     return await _sendCommand(new ScrollIntoView(finder, alignment: alignment, timeout: timeout)).then((Map<String, dynamic> _) => null);
   }
 
+  /// Repeatedly [scroll] the widget located by [scrollable] by [dxScroll] and
+  /// [dyScroll] until [item] is visible, and then use [scrollIntoView] to
+  /// ensure the item's final position matches [alignment].
+  ///
+  /// The [scrollable] must locate the scrolling widget that contains [item].
+  /// Typically `find.byType('ListView') or `find.byType('CustomScrollView')`.
+  ///
+  /// Atleast one of [dxScroll] and [dyScroll] must be non-zero.
+  ///
+  /// If [item] is below the currently visible items, then specify a negative
+  /// value for [dyScroll] that's a small enough increment to expose [item]
+  /// without potentially scrolling it up and completely out of view. Similarly
+  /// if [item] is above, then specify a positve value for [dyScroll].
+  ///
+  /// If [item] is to the right of the the currently visible items, then
+  /// specify a negative value for [dxScroll] that's a small enough increment to
+  /// expose [item] without potentially scrolling it up and completely out of
+  /// view. Similarly if [item] is to the left, then specify a positve value
+  /// for [dyScroll].
+  ///
+  /// The [timeout] value should be long enough to accommodate as many scrolls
+  /// as needed to bring an item into view. The default is 10 seconds.
+  Future<Null> scrollUntilVisible(SerializableFinder scrollable, SerializableFinder item, {
+    double alignment: 0.0,
+    double dxScroll: 0.0,
+    double dyScroll: 0.0,
+    Duration timeout: const Duration(seconds: 10),
+  }) async {
+    assert(scrollable != null);
+    assert(item != null);
+    assert(alignment != null);
+    assert(dxScroll != null);
+    assert(dyScroll != null);
+    assert(dxScroll != 0.0 || dyScroll != 0.0);
+    assert(timeout != null);
+
+    // If the item is already visible then we're done.
+    bool isVisible = false;
+    try {
+      await waitFor(item, timeout: const Duration(milliseconds: 100));
+      isVisible = true;
+    } on DriverError {
+      // Assume that that waitFor timed out because the item isn't visible.
+    }
+
+    if (!isVisible) {
+      waitFor(item, timeout: timeout).then((Null _) { isVisible = true; });
+      while (!isVisible) {
+        await scroll(scrollable, dxScroll, dyScroll, const Duration(milliseconds: 100));
+        await new Future<Null>.delayed(const Duration(milliseconds: 500));
+      }
+    }
+
+    return scrollIntoView(item, alignment: alignment);
+  }
+
   /// Returns the text in the `Text` widget located by [finder].
   Future<String> getText(SerializableFinder finder, { Duration timeout }) async {
     return GetTextResult.fromJson(await _sendCommand(new GetText(finder, timeout: timeout))).text;