gallery.dart 14.6 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io';
7
import 'dart:math' as math;
8

9
import '../framework/devices.dart';
10
import '../framework/framework.dart';
11
import '../framework/task_result.dart';
12
import '../framework/utils.dart';
13 14 15 16 17 18 19 20
import 'build_test_task.dart';

final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');

/// Temp function during gallery tests transition to build+test model.
///
/// https://github.com/flutter/flutter/issues/103542
TaskFunction createGalleryTransitionBuildTest(List<String> args, {bool semanticsEnabled = false}) {
21
  return GalleryTransitionBuildTest(args, semanticsEnabled: semanticsEnabled).call;
22
}
23

24
TaskFunction createGalleryTransitionTest({bool semanticsEnabled = false}) {
25
  return GalleryTransitionTest(semanticsEnabled: semanticsEnabled).call;
26 27
}

28 29 30
TaskFunction createGalleryTransitionE2EBuildTest(
  List<String> args, {
  bool semanticsEnabled = false,
31
  bool? enableImpeller,
32 33 34 35 36 37 38 39 40 41
}) {
  return GalleryTransitionBuildTest(
    args,
    testFile: semanticsEnabled ? 'transitions_perf_e2e_with_semantics' : 'transitions_perf_e2e',
    needFullTimeline: false,
    timelineSummaryFile: 'e2e_perf_summary',
    transitionDurationFile: null,
    timelineTraceFile: null,
    driverFile: 'transitions_perf_e2e_test',
    enableImpeller: enableImpeller,
42
  ).call;
43 44
}

45 46
TaskFunction createGalleryTransitionE2ETest({
  bool semanticsEnabled = false,
47
  bool? enableImpeller,
48
}) {
49
  return GalleryTransitionTest(
50 51 52
    testFile: semanticsEnabled
        ? 'transitions_perf_e2e_with_semantics'
        : 'transitions_perf_e2e',
53 54 55
    needFullTimeline: false,
    timelineSummaryFile: 'e2e_perf_summary',
    transitionDurationFile: null,
56
    timelineTraceFile: null,
57
    driverFile: 'transitions_perf_e2e_test',
58
    enableImpeller: enableImpeller,
59
  ).call;
60 61
}

62 63 64 65 66 67 68 69
TaskFunction createGalleryTransitionHybridBuildTest(
  List<String> args, {
  bool semanticsEnabled = false,
}) {
  return GalleryTransitionBuildTest(
    args,
    semanticsEnabled: semanticsEnabled,
    driverFile: semanticsEnabled ? 'transitions_perf_hybrid_with_semantics_test' : 'transitions_perf_hybrid_test',
70
  ).call;
71 72
}

73
TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) {
74 75 76 77 78
  return GalleryTransitionTest(
    semanticsEnabled: semanticsEnabled,
    driverFile: semanticsEnabled
        ? 'transitions_perf_hybrid_with_semantics_test'
        : 'transitions_perf_hybrid_test',
79
  ).call;
80 81
}

82 83 84
class GalleryTransitionTest {

  GalleryTransitionTest({
85 86 87 88
    this.semanticsEnabled = false,
    this.testFile = 'transitions_perf',
    this.needFullTimeline = true,
    this.timelineSummaryFile = 'transitions.timeline_summary',
89
    this.timelineTraceFile = 'transitions.timeline',
90 91
    this.transitionDurationFile = 'transition_durations.timeline',
    this.driverFile,
92 93
    this.measureCpuGpu = true,
    this.measureMemory = true,
94
    this.enableImpeller,
95
  });
96 97

  final bool semanticsEnabled;
98
  final bool needFullTimeline;
99 100
  final bool measureCpuGpu;
  final bool measureMemory;
101
  final bool? enableImpeller;
102 103
  final String testFile;
  final String timelineSummaryFile;
104 105 106
  final String? timelineTraceFile;
  final String? transitionDurationFile;
  final String? driverFile;
107

108 109 110 111 112 113
  Future<TaskResult> call() async {
    final Device device = await devices.workingDevice;
    await device.unlock();
    final String deviceId = device.deviceId;
    final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
    await inDirectory<void>(galleryDirectory, () async {
114
      String? applicationBinaryPath;
115 116 117 118 119 120 121
      if (deviceOperatingSystem == DeviceOperatingSystem.android) {
        section('BUILDING APPLICATION');
        await flutter(
          'build',
          options: <String>[
            'apk',
            '--no-android-gradle-daemon',
122 123 124
            '--profile',
            '-t',
            'test_driver/$testFile.dart',
125 126 127 128 129
            '--target-platform',
            'android-arm,android-arm64',
          ],
        );
        applicationBinaryPath = 'build/app/outputs/flutter-apk/app-profile.apk';
130
      }
131

132 133 134 135 136
      final String testDriver = driverFile ?? (semanticsEnabled
          ? '${testFile}_with_semantics_test'
          : '${testFile}_test');
      section('DRIVE START');
      await flutter('drive', options: <String>[
137
        '--no-dds',
138
        '--profile',
139 140
        if (enableImpeller != null && enableImpeller!) '--enable-impeller',
        if (enableImpeller != null && !enableImpeller!) '--no-enable-impeller',
141 142
        if (needFullTimeline)
          '--trace-startup',
143 144 145 146 147 148 149 150 151 152 153
        if (applicationBinaryPath != null)
          '--use-application-binary=$applicationBinaryPath'
        else
          ...<String>[
            '-t',
            'test_driver/$testFile.dart',
          ],
        '--driver',
        'test_driver/$testDriver.dart',
        '-d',
        deviceId,
154 155
        '-v',
        '--verbose-system-logs'
156 157 158 159
      ]);
    });

    final String testOutputDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build';
160
    final Map<String, dynamic> summary = json.decode(
161
      file('$testOutputDirectory/$timelineSummaryFile.json').readAsStringSync(),
162
    ) as Map<String, dynamic>;
163

164 165
    if (transitionDurationFile != null) {
      final Map<String, dynamic> original = json.decode(
166
        file('$testOutputDirectory/$transitionDurationFile.json').readAsStringSync(),
167 168 169 170 171 172 173 174
      ) as Map<String, dynamic>;
      final Map<String, List<int>> transitions = <String, List<int>>{};
      for (final String key in original.keys) {
        transitions[key] = List<int>.from(original[key] as List<dynamic>);
      }
      summary['transitions'] = transitions;
      summary['missed_transition_count'] = _countMissedTransitions(transitions);
    }
175

176
    final bool isAndroid = deviceOperatingSystem == DeviceOperatingSystem.android;
177
    return TaskResult.success(summary,
178 179
      detailFiles: <String>[
        if (transitionDurationFile != null)
180
          '$testOutputDirectory/$transitionDurationFile.json',
181
        if (timelineTraceFile != null)
182
          '$testOutputDirectory/$timelineTraceFile.json',
183
      ],
184 185 186 187 188 189 190 191
      benchmarkScoreKeys: <String>[
        if (transitionDurationFile != null)
          'missed_transition_count',
        'average_frame_build_time_millis',
        'worst_frame_build_time_millis',
        '90th_percentile_frame_build_time_millis',
        '99th_percentile_frame_build_time_millis',
        'average_frame_rasterizer_time_millis',
192
        'stddev_frame_rasterizer_time_millis',
193 194 195
        'worst_frame_rasterizer_time_millis',
        '90th_percentile_frame_rasterizer_time_millis',
        '99th_percentile_frame_rasterizer_time_millis',
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211
        'average_layer_cache_count',
        '90th_percentile_layer_cache_count',
        '99th_percentile_layer_cache_count',
        'worst_layer_cache_count',
        'average_layer_cache_memory',
        '90th_percentile_layer_cache_memory',
        '99th_percentile_layer_cache_memory',
        'worst_layer_cache_memory',
        'average_picture_cache_count',
        '90th_percentile_picture_cache_count',
        '99th_percentile_picture_cache_count',
        'worst_picture_cache_count',
        'average_picture_cache_memory',
        '90th_percentile_picture_cache_memory',
        '99th_percentile_picture_cache_memory',
        'worst_picture_cache_memory',
212 213 214 215 216 217 218 219 220 221 222
        if (measureCpuGpu && !isAndroid) ...<String>[
          // See https://github.com/flutter/flutter/issues/68888
          if (summary['average_cpu_usage'] != null) 'average_cpu_usage',
          if (summary['average_gpu_usage'] != null) 'average_gpu_usage',
        ],
        if (measureMemory && !isAndroid) ...<String>[
          // See https://github.com/flutter/flutter/issues/68888
          if (summary['average_memory_usage'] != null) 'average_memory_usage',
          if (summary['90th_percentile_memory_usage'] != null) '90th_percentile_memory_usage',
          if (summary['99th_percentile_memory_usage'] != null) '99th_percentile_memory_usage',
        ],
223 224
      ],
    );
225 226
  }
}
227

228 229 230 231 232 233 234 235 236 237 238 239
class GalleryTransitionBuildTest extends BuildTestTask {
  GalleryTransitionBuildTest(
    super.args, {
    this.semanticsEnabled = false,
    this.testFile = 'transitions_perf',
    this.needFullTimeline = true,
    this.timelineSummaryFile = 'transitions.timeline_summary',
    this.timelineTraceFile = 'transitions.timeline',
    this.transitionDurationFile = 'transition_durations.timeline',
    this.driverFile,
    this.measureCpuGpu = true,
    this.measureMemory = true,
240
    this.enableImpeller,
241 242 243 244 245 246
  }) : super(workingDirectory: galleryDirectory);

  final bool semanticsEnabled;
  final bool needFullTimeline;
  final bool measureCpuGpu;
  final bool measureMemory;
247
  final bool? enableImpeller;
248 249 250 251 252 253 254 255
  final String testFile;
  final String timelineSummaryFile;
  final String? timelineTraceFile;
  final String? transitionDurationFile;
  final String? driverFile;

  final String testOutputDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build';

256 257
  @override
  void copyArtifacts() {
258 259 260 261
    if (applicationBinaryPath == null) {
      return;
    }
    if (deviceOperatingSystem == DeviceOperatingSystem.android) {
262 263 264 265
      copy(
        file('${galleryDirectory.path}/build/app/outputs/flutter-apk/app-profile.apk'),
        Directory(applicationBinaryPath!),
      );
266 267 268 269 270
    } else if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
      recursiveCopy(
        Directory('${galleryDirectory.path}/build/ios/iphoneos'),
        Directory(applicationBinaryPath!),
      );
271 272 273
    }
  }

274 275
  @override
  List<String> getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) {
276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
    if (deviceOperatingSystem == DeviceOperatingSystem.android) {
      return <String>[
        'apk',
        '--no-android-gradle-daemon',
        '--profile',
        '-t',
        'test_driver/$testFile.dart',
        '--target-platform',
        'android-arm,android-arm64',
      ];
    } else if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
      return <String>[
        'ios',
        '--codesign',
        '--profile',
        '-t',
        'test_driver/$testFile.dart',
      ];
    }
    throw Exception('$deviceOperatingSystem has no build configuration');
296 297 298 299 300 301
  }

  @override
  List<String> getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) {
    final String testDriver = driverFile ?? (semanticsEnabled ? '${testFile}_with_semantics_test' : '${testFile}_test');
    return <String>[
302
      '--no-dds',
303
      '--profile',
304 305
      if (enableImpeller != null && enableImpeller!) '--enable-impeller',
      if (enableImpeller != null && !enableImpeller!) '--no-enable-impeller',
306 307 308
      if (needFullTimeline) '--trace-startup',
      '-t',
      'test_driver/$testFile.dart',
309
      if (applicationBinaryPath != null) '--use-application-binary=${getApplicationBinaryPath()}',
310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384
      '--driver',
      'test_driver/$testDriver.dart',
      '-d',
      deviceId,
    ];
  }

  @override
  Future<TaskResult> parseTaskResult() async {
    final Map<String, dynamic> summary = json.decode(
      file('$testOutputDirectory/$timelineSummaryFile.json').readAsStringSync(),
    ) as Map<String, dynamic>;

    if (transitionDurationFile != null) {
      final Map<String, dynamic> original = json.decode(
        file('$testOutputDirectory/$transitionDurationFile.json').readAsStringSync(),
      ) as Map<String, dynamic>;
      final Map<String, List<int>> transitions = <String, List<int>>{};
      for (final String key in original.keys) {
        transitions[key] = List<int>.from(original[key] as List<dynamic>);
      }
      summary['transitions'] = transitions;
      summary['missed_transition_count'] = _countMissedTransitions(transitions);
    }

    final bool isAndroid = deviceOperatingSystem == DeviceOperatingSystem.android;
    return TaskResult.success(
      summary,
      detailFiles: <String>[
        if (transitionDurationFile != null) '$testOutputDirectory/$transitionDurationFile.json',
        if (timelineTraceFile != null) '$testOutputDirectory/$timelineTraceFile.json',
      ],
      benchmarkScoreKeys: <String>[
        if (transitionDurationFile != null) 'missed_transition_count',
        'average_frame_build_time_millis',
        'worst_frame_build_time_millis',
        '90th_percentile_frame_build_time_millis',
        '99th_percentile_frame_build_time_millis',
        'average_frame_rasterizer_time_millis',
        'worst_frame_rasterizer_time_millis',
        '90th_percentile_frame_rasterizer_time_millis',
        '99th_percentile_frame_rasterizer_time_millis',
        'average_layer_cache_count',
        '90th_percentile_layer_cache_count',
        '99th_percentile_layer_cache_count',
        'worst_layer_cache_count',
        'average_layer_cache_memory',
        '90th_percentile_layer_cache_memory',
        '99th_percentile_layer_cache_memory',
        'worst_layer_cache_memory',
        'average_picture_cache_count',
        '90th_percentile_picture_cache_count',
        '99th_percentile_picture_cache_count',
        'worst_picture_cache_count',
        'average_picture_cache_memory',
        '90th_percentile_picture_cache_memory',
        '99th_percentile_picture_cache_memory',
        'worst_picture_cache_memory',
        if (measureCpuGpu && !isAndroid) ...<String>[
          // See https://github.com/flutter/flutter/issues/68888
          if (summary['average_cpu_usage'] != null) 'average_cpu_usage',
          if (summary['average_gpu_usage'] != null) 'average_gpu_usage',
        ],
        if (measureMemory && !isAndroid) ...<String>[
          // See https://github.com/flutter/flutter/issues/68888
          if (summary['average_memory_usage'] != null) 'average_memory_usage',
          if (summary['90th_percentile_memory_usage'] != null) '90th_percentile_memory_usage',
          if (summary['99th_percentile_memory_usage'] != null) '99th_percentile_memory_usage',
        ],
      ],
    );
  }

  @override
  String getApplicationBinaryPath() {
385 386 387 388 389 390
    if (deviceOperatingSystem == DeviceOperatingSystem.android) {
      return '$applicationBinaryPath/app-profile.apk';
    } else if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
      return '$applicationBinaryPath/Flutter Gallery.app';
    } else {
      return applicationBinaryPath!;
391 392 393 394
    }
  }
}

395
int _countMissedTransitions(Map<String, List<int>> transitions) {
396
  const int kTransitionBudget = 100000; // µs
397 398
  int count = 0;
  transitions.forEach((String demoName, List<int> durations) {
399
    final int longestDuration = durations.reduce(math.max);
400 401
    if (longestDuration > kTransitionBudget) {
      print('$demoName missed transition time budget ($longestDuration µs > $kTransitionBudget µs)');
402 403 404 405 406
      count++;
    }
  });
  return count;
}