gallery.dart 13.5 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io';
7
import 'dart:math' as math;
8

9
import '../framework/devices.dart';
10
import '../framework/framework.dart';
11
import '../framework/task_result.dart';
12
import '../framework/utils.dart';
13 14 15 16 17 18 19 20 21 22
import 'build_test_task.dart';

final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');

/// Temp function during gallery tests transition to build+test model.
///
/// https://github.com/flutter/flutter/issues/103542
TaskFunction createGalleryTransitionBuildTest(List<String> args, {bool semanticsEnabled = false}) {
  return GalleryTransitionBuildTest(args, semanticsEnabled: semanticsEnabled);
}
23

24 25
TaskFunction createGalleryTransitionTest({bool semanticsEnabled = false}) {
  return GalleryTransitionTest(semanticsEnabled: semanticsEnabled);
26 27
}

28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
TaskFunction createGalleryTransitionE2EBuildTest(
  List<String> args, {
  bool semanticsEnabled = false,
  bool enableImpeller = false,
}) {
  return GalleryTransitionBuildTest(
    args,
    testFile: semanticsEnabled ? 'transitions_perf_e2e_with_semantics' : 'transitions_perf_e2e',
    needFullTimeline: false,
    timelineSummaryFile: 'e2e_perf_summary',
    transitionDurationFile: null,
    timelineTraceFile: null,
    driverFile: 'transitions_perf_e2e_test',
    enableImpeller: enableImpeller,
  );
}

45 46 47 48
TaskFunction createGalleryTransitionE2ETest({
  bool semanticsEnabled = false,
  bool enableImpeller = false,
}) {
49
  return GalleryTransitionTest(
50 51 52
    testFile: semanticsEnabled
        ? 'transitions_perf_e2e_with_semantics'
        : 'transitions_perf_e2e',
53 54 55
    needFullTimeline: false,
    timelineSummaryFile: 'e2e_perf_summary',
    transitionDurationFile: null,
56
    timelineTraceFile: null,
57
    driverFile: 'transitions_perf_e2e_test',
58
    enableImpeller: enableImpeller,
59 60 61
  );
}

62 63 64 65 66 67 68 69 70 71 72
TaskFunction createGalleryTransitionHybridBuildTest(
  List<String> args, {
  bool semanticsEnabled = false,
}) {
  return GalleryTransitionBuildTest(
    args,
    semanticsEnabled: semanticsEnabled,
    driverFile: semanticsEnabled ? 'transitions_perf_hybrid_with_semantics_test' : 'transitions_perf_hybrid_test',
  );
}

73
TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) {
74 75 76 77 78 79 80 81
  return GalleryTransitionTest(
    semanticsEnabled: semanticsEnabled,
    driverFile: semanticsEnabled
        ? 'transitions_perf_hybrid_with_semantics_test'
        : 'transitions_perf_hybrid_test',
  );
}

82 83 84
class GalleryTransitionTest {

  GalleryTransitionTest({
85 86 87 88
    this.semanticsEnabled = false,
    this.testFile = 'transitions_perf',
    this.needFullTimeline = true,
    this.timelineSummaryFile = 'transitions.timeline_summary',
89
    this.timelineTraceFile = 'transitions.timeline',
90 91
    this.transitionDurationFile = 'transition_durations.timeline',
    this.driverFile,
92 93
    this.measureCpuGpu = true,
    this.measureMemory = true,
94
    this.enableImpeller = false,
95
  });
96 97

  final bool semanticsEnabled;
98
  final bool needFullTimeline;
99 100
  final bool measureCpuGpu;
  final bool measureMemory;
101
  final bool enableImpeller;
102 103
  final String testFile;
  final String timelineSummaryFile;
104 105 106
  final String? timelineTraceFile;
  final String? transitionDurationFile;
  final String? driverFile;
107

108 109 110 111 112 113
  Future<TaskResult> call() async {
    final Device device = await devices.workingDevice;
    await device.unlock();
    final String deviceId = device.deviceId;
    final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
    await inDirectory<void>(galleryDirectory, () async {
114
      String? applicationBinaryPath;
115 116 117 118 119 120 121
      if (deviceOperatingSystem == DeviceOperatingSystem.android) {
        section('BUILDING APPLICATION');
        await flutter(
          'build',
          options: <String>[
            'apk',
            '--no-android-gradle-daemon',
122 123 124
            '--profile',
            '-t',
            'test_driver/$testFile.dart',
125 126 127 128 129
            '--target-platform',
            'android-arm,android-arm64',
          ],
        );
        applicationBinaryPath = 'build/app/outputs/flutter-apk/app-profile.apk';
130
      }
131

132 133 134 135 136
      final String testDriver = driverFile ?? (semanticsEnabled
          ? '${testFile}_with_semantics_test'
          : '${testFile}_test');
      section('DRIVE START');
      await flutter('drive', options: <String>[
137
        '--no-dds',
138
        '--profile',
139
        if (enableImpeller) '--enable-impeller',
140 141
        if (needFullTimeline)
          '--trace-startup',
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
        if (applicationBinaryPath != null)
          '--use-application-binary=$applicationBinaryPath'
        else
          ...<String>[
            '-t',
            'test_driver/$testFile.dart',
          ],
        '--driver',
        'test_driver/$testDriver.dart',
        '-d',
        deviceId,
      ]);
    });

    final String testOutputDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build';
157
    final Map<String, dynamic> summary = json.decode(
158
      file('$testOutputDirectory/$timelineSummaryFile.json').readAsStringSync(),
159
    ) as Map<String, dynamic>;
160

161 162
    if (transitionDurationFile != null) {
      final Map<String, dynamic> original = json.decode(
163
        file('$testOutputDirectory/$transitionDurationFile.json').readAsStringSync(),
164 165 166 167 168 169 170 171
      ) as Map<String, dynamic>;
      final Map<String, List<int>> transitions = <String, List<int>>{};
      for (final String key in original.keys) {
        transitions[key] = List<int>.from(original[key] as List<dynamic>);
      }
      summary['transitions'] = transitions;
      summary['missed_transition_count'] = _countMissedTransitions(transitions);
    }
172

173
    final bool isAndroid = deviceOperatingSystem == DeviceOperatingSystem.android;
174
    return TaskResult.success(summary,
175 176
      detailFiles: <String>[
        if (transitionDurationFile != null)
177
          '$testOutputDirectory/$transitionDurationFile.json',
178
        if (timelineTraceFile != null)
179
          '$testOutputDirectory/$timelineTraceFile.json',
180
      ],
181 182 183 184 185 186 187 188 189 190 191
      benchmarkScoreKeys: <String>[
        if (transitionDurationFile != null)
          'missed_transition_count',
        'average_frame_build_time_millis',
        'worst_frame_build_time_millis',
        '90th_percentile_frame_build_time_millis',
        '99th_percentile_frame_build_time_millis',
        'average_frame_rasterizer_time_millis',
        'worst_frame_rasterizer_time_millis',
        '90th_percentile_frame_rasterizer_time_millis',
        '99th_percentile_frame_rasterizer_time_millis',
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
        'average_layer_cache_count',
        '90th_percentile_layer_cache_count',
        '99th_percentile_layer_cache_count',
        'worst_layer_cache_count',
        'average_layer_cache_memory',
        '90th_percentile_layer_cache_memory',
        '99th_percentile_layer_cache_memory',
        'worst_layer_cache_memory',
        'average_picture_cache_count',
        '90th_percentile_picture_cache_count',
        '99th_percentile_picture_cache_count',
        'worst_picture_cache_count',
        'average_picture_cache_memory',
        '90th_percentile_picture_cache_memory',
        '99th_percentile_picture_cache_memory',
        'worst_picture_cache_memory',
208 209 210 211 212 213 214 215 216 217 218
        if (measureCpuGpu && !isAndroid) ...<String>[
          // See https://github.com/flutter/flutter/issues/68888
          if (summary['average_cpu_usage'] != null) 'average_cpu_usage',
          if (summary['average_gpu_usage'] != null) 'average_gpu_usage',
        ],
        if (measureMemory && !isAndroid) ...<String>[
          // See https://github.com/flutter/flutter/issues/68888
          if (summary['average_memory_usage'] != null) 'average_memory_usage',
          if (summary['90th_percentile_memory_usage'] != null) '90th_percentile_memory_usage',
          if (summary['99th_percentile_memory_usage'] != null) '99th_percentile_memory_usage',
        ],
219 220
      ],
    );
221 222
  }
}
223

224 225 226 227 228 229 230 231 232 233 234 235
class GalleryTransitionBuildTest extends BuildTestTask {
  GalleryTransitionBuildTest(
    super.args, {
    this.semanticsEnabled = false,
    this.testFile = 'transitions_perf',
    this.needFullTimeline = true,
    this.timelineSummaryFile = 'transitions.timeline_summary',
    this.timelineTraceFile = 'transitions.timeline',
    this.transitionDurationFile = 'transition_durations.timeline',
    this.driverFile,
    this.measureCpuGpu = true,
    this.measureMemory = true,
236
    this.enableImpeller = false,
237 238 239 240 241 242
  }) : super(workingDirectory: galleryDirectory);

  final bool semanticsEnabled;
  final bool needFullTimeline;
  final bool measureCpuGpu;
  final bool measureMemory;
243
  final bool enableImpeller;
244 245 246 247 248 249 250 251
  final String testFile;
  final String timelineSummaryFile;
  final String? timelineTraceFile;
  final String? transitionDurationFile;
  final String? driverFile;

  final String testOutputDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build';

252 253 254 255 256 257 258 259 260 261
  @override
  void copyArtifacts() {
    if(applicationBinaryPath != null) {
      copy(
        file('${galleryDirectory.path}/build/app/outputs/flutter-apk/app-profile.apk'),
        Directory(applicationBinaryPath!),
      );
    }
  }

262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
  @override
  List<String> getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) {
    return <String>[
      'apk',
      '--no-android-gradle-daemon',
      '--profile',
      '-t',
      'test_driver/$testFile.dart',
      '--target-platform',
      'android-arm,android-arm64',
    ];
  }

  @override
  List<String> getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) {
    final String testDriver = driverFile ?? (semanticsEnabled ? '${testFile}_with_semantics_test' : '${testFile}_test');
    return <String>[
279
      '--no-dds',
280
      '--profile',
281
      if (enableImpeller) '--enable-impeller',
282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361
      if (needFullTimeline) '--trace-startup',
      '-t',
      'test_driver/$testFile.dart',
      '--use-application-binary=${getApplicationBinaryPath()}',
      '--driver',
      'test_driver/$testDriver.dart',
      '-d',
      deviceId,
    ];
  }

  @override
  Future<TaskResult> parseTaskResult() async {
    final Map<String, dynamic> summary = json.decode(
      file('$testOutputDirectory/$timelineSummaryFile.json').readAsStringSync(),
    ) as Map<String, dynamic>;

    if (transitionDurationFile != null) {
      final Map<String, dynamic> original = json.decode(
        file('$testOutputDirectory/$transitionDurationFile.json').readAsStringSync(),
      ) as Map<String, dynamic>;
      final Map<String, List<int>> transitions = <String, List<int>>{};
      for (final String key in original.keys) {
        transitions[key] = List<int>.from(original[key] as List<dynamic>);
      }
      summary['transitions'] = transitions;
      summary['missed_transition_count'] = _countMissedTransitions(transitions);
    }

    final bool isAndroid = deviceOperatingSystem == DeviceOperatingSystem.android;
    return TaskResult.success(
      summary,
      detailFiles: <String>[
        if (transitionDurationFile != null) '$testOutputDirectory/$transitionDurationFile.json',
        if (timelineTraceFile != null) '$testOutputDirectory/$timelineTraceFile.json',
      ],
      benchmarkScoreKeys: <String>[
        if (transitionDurationFile != null) 'missed_transition_count',
        'average_frame_build_time_millis',
        'worst_frame_build_time_millis',
        '90th_percentile_frame_build_time_millis',
        '99th_percentile_frame_build_time_millis',
        'average_frame_rasterizer_time_millis',
        'worst_frame_rasterizer_time_millis',
        '90th_percentile_frame_rasterizer_time_millis',
        '99th_percentile_frame_rasterizer_time_millis',
        'average_layer_cache_count',
        '90th_percentile_layer_cache_count',
        '99th_percentile_layer_cache_count',
        'worst_layer_cache_count',
        'average_layer_cache_memory',
        '90th_percentile_layer_cache_memory',
        '99th_percentile_layer_cache_memory',
        'worst_layer_cache_memory',
        'average_picture_cache_count',
        '90th_percentile_picture_cache_count',
        '99th_percentile_picture_cache_count',
        'worst_picture_cache_count',
        'average_picture_cache_memory',
        '90th_percentile_picture_cache_memory',
        '99th_percentile_picture_cache_memory',
        'worst_picture_cache_memory',
        if (measureCpuGpu && !isAndroid) ...<String>[
          // See https://github.com/flutter/flutter/issues/68888
          if (summary['average_cpu_usage'] != null) 'average_cpu_usage',
          if (summary['average_gpu_usage'] != null) 'average_gpu_usage',
        ],
        if (measureMemory && !isAndroid) ...<String>[
          // See https://github.com/flutter/flutter/issues/68888
          if (summary['average_memory_usage'] != null) 'average_memory_usage',
          if (summary['90th_percentile_memory_usage'] != null) '90th_percentile_memory_usage',
          if (summary['99th_percentile_memory_usage'] != null) '99th_percentile_memory_usage',
        ],
      ],
    );
  }

  @override
  String getApplicationBinaryPath() {
    if (applicationBinaryPath != null) {
362
      return '${applicationBinaryPath!}/app-profile.apk';
363 364 365 366 367 368
    }

    return 'build/app/outputs/flutter-apk/app-profile.apk';
  }
}

369
int _countMissedTransitions(Map<String, List<int>> transitions) {
370
  const int kTransitionBudget = 100000; // µs
371 372
  int count = 0;
  transitions.forEach((String demoName, List<int> durations) {
373
    final int longestDuration = durations.reduce(math.max);
374 375
    if (longestDuration > kTransitionBudget) {
      print('$demoName missed transition time budget ($longestDuration µs > $kTransitionBudget µs)');
376 377 378 379 380
      count++;
    }
  });
  return count;
}