flutter_tool_startup.dart 4.87 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:io';

import 'package:path/path.dart' as path;

import '../framework/task_result.dart';
import '../framework/utils.dart';

/// Run each benchmark this many times and compute average, min, max.
const int _kRunsPerBenchmark = 10;

Future<TaskResult> flutterToolStartupBenchmarkTask() async {
  final Directory projectParentDirectory =
      Directory.systemTemp.createTempSync('flutter_tool_startup_benchmark');
  final Directory projectDirectory =
      dir(path.join(projectParentDirectory.path, 'benchmark'));
  await inDirectory<void>(flutterDirectory, () async {
    await flutter('update-packages');
    await flutter('create', options: <String>[projectDirectory.path]);
    // Remove 'test' directory so we don't time the actual testing, but only the launching of the flutter tool
    rmTree(dir(path.join(projectDirectory.path, 'test')));
  });

  final Map<String, dynamic> data = <String, dynamic>{
    // `flutter test` in dir with no `test` folder.
    ...(await _Benchmark(
      projectDirectory,
      'test startup',
      'test',
    ).run())
        .asMap('flutter_tool_startup_test'),

    // `flutter test -d foo_device` in dir woth no `test` folder.
    ...(await _Benchmark(
      projectDirectory,
      'test startup with specified device',
      'test',
      options: <String>['-d', 'foo_device'],
    ).run())
        .asMap('flutter_tool_startup_test_with_specified_device'),

    // `flutter test -v` where no android sdk will be found (at least currently).
    ...(await _Benchmark(
      projectDirectory,
      'test startup no android sdk',
      'test',
      options: <String>['-v'],
      environment: <String, String>{
        'ANDROID_HOME': 'dummy value',
        'ANDROID_SDK_ROOT': 'dummy value',
        'PATH': pathWithoutWhereHits(<String>['adb', 'aapt']),
      },
    ).run())
        .asMap('flutter_tool_startup_test_no_android_sdk'),

    // `flutter -h`.
    ...(await _Benchmark(
      projectDirectory,
      'help startup',
      '-h',
    ).run())
        .asMap('flutter_tool_startup_help'),
  };

  // Cleanup.
  rmTree(projectParentDirectory);

  return TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
}

String pathWithoutWhereHits(List<String> whats) {
  final String pathEnvironment = Platform.environment['PATH'] ?? '';
  List<String> paths;
  if (Platform.isWindows) {
    paths = pathEnvironment.split(';');
  } else {
    paths = pathEnvironment.split(':');
  }
  // This isn't great but will probably work for our purposes.
  final List<String> extensions = <String>['', '.exe', '.bat', '.com'];

  final List<String> notFound = <String>[];
  for (final String path in paths) {
    bool found = false;
    for (final String extension in extensions) {
      for (final String what in whats) {
        final File f = File('$path${Platform.pathSeparator}$what$extension');
        if (f.existsSync()) {
          found = true;
          break;
        }
      }
      if (found) {
        break;
      }
    }
    if (!found) {
      notFound.add(path);
    }
  }

  if (Platform.isWindows) {
    return notFound.join(';');
  } else {
    return notFound.join(':');
  }
}

class _BenchmarkResult {
  const _BenchmarkResult(this.mean, this.min, this.max);

  final int mean; // Milliseconds

  final int min; // Milliseconds

  final int max; // Milliseconds

  Map<String, dynamic> asMap(String name) {
    return <String, dynamic>{
      name: mean,
      '${name}_minimum': min,
      '${name}_maximum': max,
    };
  }
}

class _Benchmark {
  _Benchmark(this.directory, this.title, this.command,
      {this.options = const <String>[], this.environment});

  final Directory directory;

  final String title;

  final String command;

  final List<String> options;

  final Map<String, String>? environment;

  Future<int> execute(int iteration, int targetIterations) async {
    section('Benchmark $title - ${iteration + 1} / $targetIterations');
    final Stopwatch stopwatch = Stopwatch();
    await inDirectory<void>(directory, () async {
      stopwatch.start();
      // canFail is set to true, as e.g. `flutter test` in a dir with no `test`
      // directory sets a non-zero return value.
      await flutter(command,
          options: options, canFail: true, environment: environment);
      stopwatch.stop();
    });
    return stopwatch.elapsedMilliseconds;
  }

  /// Runs `benchmark` several times and reports the results.
  Future<_BenchmarkResult> run() async {
    final List<int> results = <int>[];
    int sum = 0;
    for (int i = 0; i < _kRunsPerBenchmark; i++) {
      final int thisRuntime = await execute(i, _kRunsPerBenchmark);
      results.add(thisRuntime);
      sum += thisRuntime;
    }
    results.sort();
    return _BenchmarkResult(sum ~/ results.length, results.first, results.last);
  }
}