run.dart 11.4 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6 7 8
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io';

import 'package:args/args.dart';
9
import 'package:flutter_devicelab/framework/ab.dart';
10
import 'package:flutter_devicelab/framework/runner.dart';
11
import 'package:flutter_devicelab/framework/task_result.dart';
12
import 'package:flutter_devicelab/framework/utils.dart';
13
import 'package:path/path.dart' as path;
14

15 16 17 18 19 20 21
/// Runs tasks.
///
/// The tasks are chosen depending on the command-line options.
Future<void> main(List<String> rawArgs) async {
  // This is populated by a callback in the ArgParser.
  final List<String> taskNames = <String>[];
  final ArgParser argParser = createArgParser(taskNames);
22

23 24 25 26 27 28 29 30 31
  ArgResults args;
  try {
    args = argParser.parse(rawArgs); // populates taskNames as a side-effect
  } on FormatException catch (error) {
    stderr.writeln('${error.message}\n');
    stderr.writeln('Usage:\n');
    stderr.writeln(argParser.usage);
    exit(1);
  }
32

33 34
  /// Suppresses standard output, prints only standard error output.
  final bool silent = (args['silent'] as bool?) ?? false;
35

36 37 38 39
  /// The build of the local engine to use.
  ///
  /// Required for A/B test mode.
  final String? localEngine = args['local-engine'] as String?;
40

41 42
  /// The path to the engine "src/" directory.
  final String? localEngineSrcPath = args['local-engine-src-path'] as String?;
43

44 45
  /// The device-id to run test on.
  final String? deviceId = args['device-id'] as String?;
46

47 48
  /// Whether to exit on first test failure.
  final bool exitOnFirstTestFailure = (args['exit'] as bool?) ?? false;
49

50 51
  /// Whether to tell tasks to clean up after themselves.
  final bool terminateStrayDartProcesses = (args['terminate-stray-dart-processes'] as bool?) ?? false;
52

53 54
  /// The git branch being tested on.
  final String? gitBranch = args['git-branch'] as String?;
55

56 57 58 59 60
  /// Name of the LUCI builder this test is currently running on.
  ///
  /// This is only passed on CI runs for Cocoon to be able to uniquely identify
  /// this test run.
  final String? luciBuilder = args['luci-builder'] as String?;
61

62 63
  /// Path to write test results to.
  final String? resultsPath = args['results-file'] as String?;
64

65
  if (args.wasParsed('list')) {
66 67
    for (int i = 0; i < taskNames.length; i++) {
      print('${(i + 1).toString().padLeft(3)} - ${taskNames[i]}');
68
    }
69
    exit(0);
70 71
  }

72
  if (taskNames.isEmpty) {
73
    stderr.writeln('Failed to find tasks to run based on supplied options.');
74
    exit(1);
75 76
  }

77
  if (args.wasParsed('ab')) {
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
    final int runsPerTest = int.parse(args['ab'] as String);
    final String resultsFile = args['ab-result-file'] as String? ?? 'ABresults#.json';
    if (taskNames.length > 1) {
      stderr.writeln('When running in A/B test mode exactly one task must be passed but got ${taskNames.join(', ')}.\n');
      stderr.writeln(argParser.usage);
      exit(1);
    }
    if (localEngine == null) {
      stderr.writeln('When running in A/B test mode --local-engine is required.\n');
      stderr.writeln(argParser.usage);
      exit(1);
    }
    await _runABTest(
      runsPerTest: runsPerTest,
      silent: silent,
      localEngine: localEngine,
      localEngineSrcPath: localEngineSrcPath,
      deviceId: deviceId,
      resultsFile: resultsFile,
      taskName: taskNames.single,
    );
99
  } else {
100
    await runTasks(taskNames,
101
      silent: silent,
102 103
      localEngine: localEngine,
      localEngineSrcPath: localEngineSrcPath,
104
      deviceId: deviceId,
105
      exitOnFirstTestFailure: exitOnFirstTestFailure,
106
      terminateStrayDartProcesses: terminateStrayDartProcesses,
107 108 109
      gitBranch: gitBranch,
      luciBuilder: luciBuilder,
      resultsPath: resultsPath,
110
    );
111 112 113
  }
}

114 115 116 117 118 119 120 121 122
Future<void> _runABTest({
  required int runsPerTest,
  required bool silent,
  required String localEngine,
  required String? localEngineSrcPath,
  required String? deviceId,
  required String resultsFile,
  required String taskName,
}) async {
123 124
  print('$taskName A/B test. Will run $runsPerTest times.');

125
  final ABTest abTest = ABTest(localEngine, taskName);
126 127 128 129
  for (int i = 1; i <= runsPerTest; i++) {
    section('Run #$i');

    print('Running with the default engine (A)');
130
    final TaskResult defaultEngineResult = await runTask(
131 132
      taskName,
      silent: silent,
133
      deviceId: deviceId,
134 135 136 137 138
    );

    print('Default engine result:');
    print(const JsonEncoder.withIndent('  ').convert(defaultEngineResult));

139
    if (!defaultEngineResult.succeeded) {
140 141 142 143 144 145 146
      stderr.writeln('Task failed on the default engine.');
      exit(1);
    }

    abTest.addAResult(defaultEngineResult);

    print('Running with the local engine (B)');
147
    final TaskResult localEngineResult = await runTask(
148 149 150 151
      taskName,
      silent: silent,
      localEngine: localEngine,
      localEngineSrcPath: localEngineSrcPath,
152
      deviceId: deviceId,
153 154 155 156 157
    );

    print('Task localEngineResult:');
    print(const JsonEncoder.withIndent('  ').convert(localEngineResult));

158
    if (!localEngineResult.succeeded) {
159 160 161 162 163
      stderr.writeln('Task failed on the local engine.');
      exit(1);
    }

    abTest.addBResult(localEngineResult);
164

165
    if (silent != true && i < runsPerTest) {
166 167 168
      section('A/B results so far');
      print(abTest.printSummary());
    }
169
  }
170 171
  abTest.finalize();

172
  final File jsonFile = _uniqueFile(resultsFile);
173
  jsonFile.writeAsStringSync(const JsonEncoder.withIndent('  ').convert(abTest.jsonMap));
174

175
  if (silent != true) {
176 177 178 179 180
    section('Raw results');
    print(abTest.rawResults());
  }

  section('Final A/B results');
181
  print(abTest.printSummary());
182 183 184 185 186 187 188 189 190 191 192 193 194

  print('');
  print('Results saved to ${jsonFile.path}');
}

File _uniqueFile(String filenameTemplate) {
  final List<String> parts = filenameTemplate.split('#');
  if (parts.length != 2) {
    return File(filenameTemplate);
  }
  File file = File(parts[0] + parts[1]);
  int i = 1;
  while (file.existsSync()) {
195
    file = File(parts[0] + i.toString() + parts[1]);
196 197 198
    i++;
  }
  return file;
199 200
}

201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
ArgParser createArgParser(List<String> taskNames) {
  return ArgParser()
    ..addMultiOption(
      'task',
      abbr: 't',
      help: 'Either:\n'
          ' - the name of a task defined in manifest.yaml.\n'
          '   Example: complex_layout__start_up.\n'
          ' - the path to a Dart file corresponding to a task,\n'
          '   which resides in bin/tasks.\n'
          '   Example: bin/tasks/complex_layout__start_up.dart.\n'
          '\n'
          'This option may be repeated to specify multiple tasks.',
      callback: (List<String> value) {
        for (final String nameOrPath in value) {
          final List<String> fragments = path.split(nameOrPath);
          final bool isDartFile = fragments.last.endsWith('.dart');

          if (fragments.length == 1 && !isDartFile) {
            // Not a path
            taskNames.add(nameOrPath);
          } else if (!isDartFile || !path.equals(path.dirname(nameOrPath), path.join('bin', 'tasks'))) {
            // Unsupported executable location
            throw FormatException('Invalid value for option -t (--task): $nameOrPath');
          } else {
            taskNames.add(path.withoutExtension(fragments.last));
          }
        }
      },
    )
    ..addOption(
      'device-id',
      abbr: 'd',
      help: 'Target device id (prefixes are allowed, names are not supported).\n'
            'The option will be ignored if the test target does not run on a\n'
            'mobile device. This still respects the device operating system\n'
            'settings in the test case, and will results in error if no device\n'
            'with given ID/ID prefix is found.',
    )
    ..addOption(
      'ab',
      help: 'Runs an A/B test comparing the default engine with the local\n'
            'engine build for one task. This option does not support running\n'
            'multiple tasks. The value is the number of times to run the task.\n'
            'The task is expected to be a benchmark that reports score keys.\n'
            'The A/B test collects the metrics collected by the test and\n'
            'produces a report containing averages, noise, and the speed-up\n'
            'between the two engines. --local-engine is required when running\n'
            'an A/B test.',
      callback: (String? value) {
        if (value != null && int.tryParse(value) == null) {
          throw ArgParserException('Option --ab must be a number, but was "$value".');
        }
      },
    )
    ..addOption(
      'ab-result-file',
      help: 'The filename in which to place the json encoded results of an A/B test.\n'
            'The filename may contain a single # character to be replaced by a sequence\n'
            'number if the name already exists.',
    )
    ..addFlag(
      'exit',
      defaultsTo: true,
      help: 'Exit on the first test failure. Currently flakes are intentionally (though '
            'incorrectly) not considered to be failures.',
    )
    ..addOption(
      'git-branch',
      help: '[Flutter infrastructure] Git branch of the current commit. LUCI\n'
            'checkouts run in detached HEAD state, so the branch must be passed.',
    )
    ..addOption(
      'local-engine',
      help: 'Name of a build output within the engine out directory, if you\n'
            'are building Flutter locally. Use this to select a specific\n'
            'version of the engine if you have built multiple engine targets.\n'
            'This path is relative to --local-engine-src-path/out. This option\n'
            'is required when running an A/B test (see the --ab option).',
    )
    ..addFlag(
      'list',
      abbr: 'l',
      help: "Don't actually run the tasks, but list out the tasks that would\n"
            'have been run, in the order they would have run.',
    )
    ..addOption(
      'local-engine-src-path',
      help: 'Path to your engine src directory, if you are building Flutter\n'
            'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
            'the location based on the value of the --flutter-root option.',
    )
    ..addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.')
    ..addFlag(
      'match-host-platform',
      defaultsTo: true,
      help: 'Only run tests that match the host platform (e.g. do not run a\n'
            'test with a `required_agent_capabilities` value of "mac/android"\n'
            'on a windows host). Each test publishes its '
            '`required_agent_capabilities`\nin the `manifest.yaml` file.',
    )
    ..addOption(
      'results-file',
      help: '[Flutter infrastructure] File path for test results. If passed with\n'
            'task, will write test results to the file.'
    )
    ..addOption(
      'service-account-token-file',
      help: '[Flutter infrastructure] Authentication for uploading results.',
    )
    ..addFlag(
      'silent',
      help: 'Reduce verbosity slightly.',
    )
    ..addFlag(
      'terminate-stray-dart-processes',
      defaultsTo: true,
      help: 'Whether to send a SIGKILL signal to any Dart processes that are still '
            'running when a task is completed. If any Dart processes are terminated '
            'in this way, the test is considered to have failed.',
    )
    ..addMultiOption(
      'test',
      hide: true,
      callback: (List<String> value) {
        if (value.isNotEmpty) {
          throw const FormatException(
            'Invalid option --test. Did you mean --task (-t)?',
          );
330
        }
331 332 333
      },
    );
}