transitions_perf_test.dart 8.39 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
import 'dart:convert' show JsonEncoder, json;
6

7
import 'package:file/file.dart';
8
import 'package:file/local.dart';
9
import 'package:flutter_driver/flutter_driver.dart';
10
import 'package:flutter_gallery/demo_lists.dart';
11
import 'package:path/path.dart' as path;
12
import 'package:test/test.dart' hide TypeMatcher, isInstanceOf;
13

14
const FileSystem _fs = LocalFileSystem();
15

16 17 18 19 20 21 22 23 24
/// The demos we don't run as part of the integration test.
///
/// Demo names are formatted as 'DEMO_NAME@DEMO_CATEGORY' (see
/// `demo_lists.dart` for more examples).
const List<String> kSkippedDemos = <String>[
  // This demo is flaky on CI due to hitting the network.
  // See: https://github.com/flutter/flutter/issues/100497
  'Video@Media',
];
25

26
// All of the gallery demos, identified as "title@category".
27
//
28 29
// These names are reported by the test app, see _handleMessages()
// in transitions_perf.dart.
30
List<String> _allDemos = <String>[];
31

32 33
/// Extracts event data from [events] recorded by timeline, validates it, turns
/// it into a histogram, and saves to a JSON file.
34
Future<void> saveDurationsHistogram(List<Map<String, dynamic>> events, String outputPath) async {
35
  final Map<String, List<int>> durations = <String, List<int>>{};
36 37
  Map<String, dynamic>? startEvent;
  int? frameStart;
38 39

  // Save the duration of the first frame after each 'Start Transition' event.
40
  for (final Map<String, dynamic> event in events) {
41
    final String eventName = event['name'] as String;
42 43 44 45
    if (eventName == 'Start Transition') {
      assert(startEvent == null);
      startEvent = event;
    } else if (startEvent != null && eventName == 'Frame') {
46 47
      final String phase = event['ph'] as String;
      final int timestamp = event['ts'] as int;
48
      if (phase == 'B' || phase == 'b') {
49 50 51
        assert(frameStart == null);
        frameStart = timestamp;
      } else {
52
        assert(phase == 'E' || phase == 'e');
53
        final String routeName = (startEvent['args'] as Map<String, dynamic>)['to'] as String;
54
        durations[routeName] ??= <int>[];
55
        durations[routeName]!.add(timestamp - frameStart!);
56 57 58
        startEvent = null;
        frameStart = null;
      }
59 60 61 62
    }
  }

  // Verify that the durations data is valid.
63
  if (durations.keys.isEmpty) {
64
    throw 'no "Start Transition" timeline events found';
65
  }
66
  final Map<String, int> unexpectedValueCounts = <String, int>{};
67 68 69 70 71 72 73
  durations.forEach((String routeName, List<int> values) {
    if (values.length != 2) {
      unexpectedValueCounts[routeName] = values.length;
    }
  });

  if (unexpectedValueCounts.isNotEmpty) {
74
    final StringBuffer error = StringBuffer('Some routes recorded wrong number of values (expected 2 values/route):\n\n');
75 76
    // When run with --trace-startup, the VM stores trace events in an endless buffer instead of a ring buffer.
    error.write('You must add the --trace-startup parameter to run the test. \n\n');
77 78 79 80
    unexpectedValueCounts.forEach((String routeName, int count) {
      error.writeln(' - $routeName recorded $count values.');
    });
    error.writeln('\nFull event sequence:');
81
    final Iterator<Map<String, dynamic>> eventIter = events.iterator;
82 83
    String lastEventName = '';
    String lastRouteName = '';
84
    while (eventIter.moveNext()) {
85
      final String eventName = eventIter.current['name'] as String;
86

87
      if (!<String>['Start Transition', 'Frame'].contains(eventName)) {
88
        continue;
89
      }
90

91
      final String routeName = eventName == 'Start Transition'
92
        ? (eventIter.current['args'] as Map<String, dynamic>)['to'] as String
93 94 95 96 97 98 99 100 101 102 103 104
        : '';

      if (eventName == lastEventName && routeName == lastRouteName) {
        error.write('.');
      } else {
        error.write('\n - $eventName $routeName .');
      }

      lastEventName = eventName;
      lastRouteName = routeName;
    }
    throw error;
105 106 107
  }

  // Save the durations Map to a file.
108
  final File file = await _fs.file(outputPath).create(recursive: true);
109
  await file.writeAsString(const JsonEncoder.withIndent('  ').convert(durations));
110 111
}

112 113
/// Scrolls each demo menu item into view, launches it, then returns to the
/// home screen twice.
114
Future<void> runDemos(List<String> demos, FlutterDriver driver) async {
115
  final SerializableFinder demoList = find.byValueKey('GalleryDemoList');
116
  String? currentDemoCategory;
117

118
  for (final String demo in demos) {
119
    if (kSkippedDemos.contains(demo)) {
120
      continue;
121
    }
122 123 124 125

    final String demoName = demo.substring(0, demo.indexOf('@'));
    final String demoCategory = demo.substring(demo.indexOf('@') + 1);
    print('> $demo');
126

127
    final SerializableFinder demoCategoryItem = find.text(demoCategory);
128
    if (currentDemoCategory == null) {
129 130
      await driver.scrollIntoView(demoCategoryItem);
      await driver.tap(demoCategoryItem);
131 132
    } else if (currentDemoCategory != demoCategory) {
      await driver.tap(find.byTooltip('Back'));
133 134
      await driver.scrollIntoView(demoCategoryItem);
      await driver.tap(demoCategoryItem);
135 136
      // Scroll back to the top
      await driver.scroll(demoList, 0.0, 10000.0, const Duration(milliseconds: 100));
137 138
    }
    currentDemoCategory = demoCategory;
139

140
    final SerializableFinder demoItem = find.text(demoName);
141 142 143 144
    await driver.scrollUntilVisible(demoList, demoItem,
      dyScroll: -48.0,
      alignment: 0.5,
    );
145

146 147
    for (int i = 0; i < 2; i += 1) {
      await driver.tap(demoItem); // Launch the demo
148 149

      if (kUnsynchronizedDemos.contains(demo)) {
150
        await driver.runUnsynchronized<void>(() async {
151
          await driver.tap(find.pageBack());
152
        });
153
      } else {
154
        await driver.tap(find.pageBack());
155 156
      }
    }
157

158
    print('< Success');
159
  }
160 161 162

  // Return to the home screen
  await driver.tap(find.byTooltip('Back'));
163 164
}

165
void main([List<String> args = const <String>[]]) {
166
  final bool withSemantics = args.contains('--with_semantics');
167
  final bool hybrid = args.contains('--hybrid');
168
  group('flutter gallery transitions', () {
169
    late FlutterDriver driver;
170 171
    setUpAll(() async {
      driver = await FlutterDriver.connect();
172

173 174
      // Wait for the first frame to be rasterized.
      await driver.waitUntilFirstFrameRasterized();
175
      if (withSemantics) {
176
        print('Enabling semantics...');
177 178
        await driver.setSemantics(true);
      }
179 180

      // See _handleMessages() in transitions_perf.dart.
181
      _allDemos = List<String>.from(json.decode(await driver.requestData('demoNames')) as List<dynamic>);
182
      if (_allDemos.isEmpty) {
183
        throw 'no demo names found';
184
      }
185 186 187
    });

    tearDownAll(() async {
188
        await driver.close();
189 190
    });

191 192 193 194
    test('find.bySemanticsLabel', () async {
      // Assert that we can use semantics related finders in profile mode.
      final int id = await driver.getSemanticsId(find.bySemanticsLabel('Material'));
      expect(id, greaterThan(-1));
195 196 197 198
    },
        skip: !withSemantics, // [intended] test only makes sense when semantics are turned on.
        timeout: Timeout.none,
    );
199

200
    test('all demos', () async {
201
      // Collect timeline data for just a limited set of demos to avoid OOMs.
202 203
      final Timeline timeline = await driver.traceAction(
        () async {
204 205 206 207 208
          if (hybrid) {
            await driver.requestData('profileDemos');
          } else {
            await runDemos(kProfiledDemos, driver);
          }
209 210 211 212
        },
        streams: const <TimelineStream>[
          TimelineStream.dart,
          TimelineStream.embedder,
213
          TimelineStream.gc,
214 215
        ],
      );
216 217 218 219

      // Save the duration (in microseconds) of the first timeline Frame event
      // that follows a 'Start Transition' event. The Gallery app adds a
      // 'Start Transition' event when a demo is launched (see GalleryItem).
220
      final TimelineSummary summary = TimelineSummary.summarize(timeline);
221
      await summary.writeTimelineToFile('transitions', pretty: true);
222
      final String histogramPath = path.join(testOutputsDirectory, 'transition_durations.timeline.json');
223
      await saveDurationsHistogram(
224
          List<Map<String, dynamic>>.from(timeline.json['traceEvents'] as List<dynamic>),
225
          histogramPath);
226 227

      // Execute the remaining tests.
228
      if (hybrid) {
229
        await driver.requestData('restDemos');
230 231 232 233
      } else {
        final Set<String> unprofiledDemos = Set<String>.from(_allDemos)..removeAll(kProfiledDemos);
        await runDemos(unprofiledDemos.toList(), driver);
      }
234

235
    }, timeout: Timeout.none);
236 237
  });
}