transitions_perf_test.dart 8.01 KB
Newer Older
1 2 3 4 5
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
6
import 'dart:convert' show JsonEncoder, JsonDecoder;
7

8
import 'package:file/file.dart';
9
import 'package:file/local.dart';
10
import 'package:flutter_driver/flutter_driver.dart';
11
import 'package:path/path.dart' as path;
12
import 'package:test/test.dart' hide TypeMatcher, isInstanceOf;
13

14
const FileSystem _fs = LocalFileSystem();
15

16 17 18 19 20 21 22 23
// Demos for which timeline data will be collected using
// FlutterDriver.traceAction().
//
// Warning: The number of tests executed with timeline collection enabled
// significantly impacts heap size of the running app. When run with
// --trace-startup, as we do in this test, the VM stores trace events in an
// endless buffer instead of a ring buffer.
//
24 25
// These names must match GalleryItem titles from kAllGalleryDemos
// in examples/flutter_gallery/lib/gallery/demos.dart
26
const List<String> kProfiledDemos = <String>[
27 28 29
  'Shrine@Studies',
  'Contact profile@Studies',
  'Animation@Studies',
30 31 32 33 34 35
  'Bottom navigation@Material',
  'Buttons@Material',
  'Cards@Material',
  'Chips@Material',
  'Dialogs@Material',
  'Pickers@Material',
36 37
];

38 39 40 41 42 43 44 45
// There are 3 places where the Gallery demos are traversed.
// 1- In widget tests such as examples/flutter_gallery/test/smoke_test.dart
// 2- In driver tests such as examples/flutter_gallery/test_driver/transitions_perf_test.dart
// 3- In on-device instrumentation tests such as examples/flutter_gallery/test/live_smoketest.dart
//
// If you change navigation behavior in the Gallery or in the framework, make
// sure all 3 are covered.

46 47
// Demos that will be backed out of within FlutterDriver.runUnsynchronized();
//
48 49
// These names must match GalleryItem titles from kAllGalleryDemos
// in examples/flutter_gallery/lib/gallery/demos.dart
50
const List<String> kUnsynchronizedDemos = <String>[
51 52 53 54 55
  'Progress indicators@Material',
  'Activity Indicator@Cupertino',
  'Video@Media',
];

56
const List<String> kSkippedDemos = <String>[];
57

58
// All of the gallery demos, identified as "title@category".
59
//
60 61
// These names are reported by the test app, see _handleMessages()
// in transitions_perf.dart.
62
List<String> _allDemos = <String>[];
63

64 65
/// Extracts event data from [events] recorded by timeline, validates it, turns
/// it into a histogram, and saves to a JSON file.
66
Future<void> saveDurationsHistogram(List<Map<String, dynamic>> events, String outputPath) async {
67
  final Map<String, List<int>> durations = <String, List<int>>{};
68 69 70 71 72 73 74 75 76 77
  Map<String, dynamic> startEvent;

  // Save the duration of the first frame after each 'Start Transition' event.
  for (Map<String, dynamic> event in events) {
    final String eventName = event['name'];
    if (eventName == 'Start Transition') {
      assert(startEvent == null);
      startEvent = event;
    } else if (startEvent != null && eventName == 'Frame') {
      final String routeName = startEvent['args']['to'];
78
      durations[routeName] ??= <int>[];
79 80 81 82 83 84 85 86
      durations[routeName].add(event['dur']);
      startEvent = null;
    }
  }

  // Verify that the durations data is valid.
  if (durations.keys.isEmpty)
    throw 'no "Start Transition" timeline events found';
87
  final Map<String, int> unexpectedValueCounts = <String, int>{};
88 89 90 91 92 93 94
  durations.forEach((String routeName, List<int> values) {
    if (values.length != 2) {
      unexpectedValueCounts[routeName] = values.length;
    }
  });

  if (unexpectedValueCounts.isNotEmpty) {
95
    final StringBuffer error = StringBuffer('Some routes recorded wrong number of values (expected 2 values/route):\n\n');
96 97 98 99
    unexpectedValueCounts.forEach((String routeName, int count) {
      error.writeln(' - $routeName recorded $count values.');
    });
    error.writeln('\nFull event sequence:');
100
    final Iterator<Map<String, dynamic>> eventIter = events.iterator;
101 102
    String lastEventName = '';
    String lastRouteName = '';
103
    while (eventIter.moveNext()) {
104
      final String eventName = eventIter.current['name'];
105 106 107 108

      if (!<String>['Start Transition', 'Frame'].contains(eventName))
        continue;

109
      final String routeName = eventName == 'Start Transition'
110 111 112 113 114 115 116 117 118 119 120 121 122
        ? eventIter.current['args']['to']
        : '';

      if (eventName == lastEventName && routeName == lastRouteName) {
        error.write('.');
      } else {
        error.write('\n - $eventName $routeName .');
      }

      lastEventName = eventName;
      lastRouteName = routeName;
    }
    throw error;
123 124 125
  }

  // Save the durations Map to a file.
126
  final File file = await _fs.file(outputPath).create(recursive: true);
127
  await file.writeAsString(const JsonEncoder.withIndent('  ').convert(durations));
128 129
}

130 131
/// Scrolls each demo menu item into view, launches it, then returns to the
/// home screen twice.
132
Future<void> runDemos(List<String> demos, FlutterDriver driver) async {
133 134 135
  final SerializableFinder demoList = find.byValueKey('GalleryDemoList');
  String currentDemoCategory;

136
  for (String demo in demos) {
137 138 139 140 141 142
    if (kSkippedDemos.contains(demo))
      continue;

    final String demoName = demo.substring(0, demo.indexOf('@'));
    final String demoCategory = demo.substring(demo.indexOf('@') + 1);
    print('> $demo');
143 144 145 146 147 148

    if (currentDemoCategory == null) {
      await driver.tap(find.text(demoCategory));
    } else if (currentDemoCategory != demoCategory) {
      await driver.tap(find.byTooltip('Back'));
      await driver.tap(find.text(demoCategory));
149 150
      // Scroll back to the top
      await driver.scroll(demoList, 0.0, 10000.0, const Duration(milliseconds: 100));
151 152
    }
    currentDemoCategory = demoCategory;
153

154
    final SerializableFinder demoItem = find.text(demoName);
155 156 157 158 159
    await driver.scrollUntilVisible(demoList, demoItem,
      dyScroll: -48.0,
      alignment: 0.5,
      timeout: const Duration(seconds: 30),
    );
160

161 162
    for (int i = 0; i < 2; i += 1) {
      await driver.tap(demoItem); // Launch the demo
163 164

      if (kUnsynchronizedDemos.contains(demo)) {
165
        await driver.runUnsynchronized<void>(() async {
166
          await driver.tap(find.pageBack());
167
        });
168
      } else {
169
        await driver.tap(find.pageBack());
170 171
      }
    }
172

173
    print('< Success');
174
  }
175 176 177

  // Return to the home screen
  await driver.tap(find.byTooltip('Back'));
178 179
}

180
void main([List<String> args = const <String>[]]) {
181 182 183 184
  group('flutter gallery transitions', () {
    FlutterDriver driver;
    setUpAll(() async {
      driver = await FlutterDriver.connect();
185

186 187 188 189
      if (args.contains('--with_semantics')) {
        print('Enabeling semantics...');
        await driver.setSemantics(true);
      }
190 191

      // See _handleMessages() in transitions_perf.dart.
192
      _allDemos = List<String>.from(const JsonDecoder().convert(await driver.requestData('demoNames')));
193 194
      if (_allDemos.isEmpty)
        throw 'no demo names found';
195 196 197 198
    });

    tearDownAll(() async {
      if (driver != null)
199
        await driver.close();
200 201 202
    });

    test('all demos', () async {
203

204
      // Collect timeline data for just a limited set of demos to avoid OOMs.
205 206 207 208 209 210 211 212 213
      final Timeline timeline = await driver.traceAction(
        () async {
          await runDemos(kProfiledDemos, driver);
        },
        streams: const <TimelineStream>[
          TimelineStream.dart,
          TimelineStream.embedder,
        ],
      );
214 215 216 217

      // Save the duration (in microseconds) of the first timeline Frame event
      // that follows a 'Start Transition' event. The Gallery app adds a
      // 'Start Transition' event when a demo is launched (see GalleryItem).
218
      final TimelineSummary summary = TimelineSummary.summarize(timeline);
219
      await summary.writeSummaryToFile('transitions', pretty: true);
220
      final String histogramPath = path.join(testOutputsDirectory, 'transition_durations.timeline.json');
221
      await saveDurationsHistogram(
222
          List<Map<String, dynamic>>.from(timeline.json['traceEvents']),
223
          histogramPath);
224 225

      // Execute the remaining tests.
226
      final Set<String> unprofiledDemos = Set<String>.from(_allDemos)..removeAll(kProfiledDemos);
227
      await runDemos(unprofiledDemos.toList(), driver);
228

229
    }, timeout: const Timeout(Duration(minutes: 5)));
230 231
  });
}