run.dart 11.8 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6 7 8
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io';

import 'package:args/args.dart';
9
import 'package:flutter_devicelab/framework/ab.dart';
10 11
import 'package:flutter_devicelab/framework/manifest.dart';
import 'package:flutter_devicelab/framework/runner.dart';
12
import 'package:flutter_devicelab/framework/task_result.dart';
13
import 'package:flutter_devicelab/framework/utils.dart';
14
import 'package:path/path.dart' as path;
15

16
late ArgResults args;
17

18 19
List<String> _taskNames = <String>[];

20
/// The device-id to run test on.
21
String? deviceId;
22

23
/// The git branch being tested on.
24
String? gitBranch;
25

26 27 28
/// The build of the local engine to use.
///
/// Required for A/B test mode.
29
String? localEngine;
30 31

/// The path to the engine "src/" directory.
32
String? localEngineSrcPath;
33

34 35 36 37
/// Name of the LUCI builder this test is currently running on.
///
/// This is only passed on CI runs for Cocoon to be able to uniquely identify
/// this test run.
38
String? luciBuilder;
39

40
/// Whether to exit on first test failure.
41
bool exitOnFirstTestFailure = false;
42

43
/// Path to write test results to.
44
String? resultsPath;
45

46 47 48
/// File containing a service account token.
///
/// If passed, the test run results will be uploaded to Flutter infrastructure.
49
String? serviceAccountTokenFile;
50 51

/// Suppresses standard output, prints only standard error output.
52
bool silent = false;
53

54 55 56 57
/// Runs tasks.
///
/// The tasks are chosen depending on the command-line options
/// (see [_argParser]).
58
Future<void> main(List<String> rawArgs) async {
59 60
  try {
    args = _argParser.parse(rawArgs);
61
  } on FormatException catch (error) {
62 63 64 65
    stderr.writeln('${error.message}\n');
    stderr.writeln('Usage:\n');
    stderr.writeln(_argParser.usage);
    exitCode = 1;
66
    return;
67 68
  }

69 70 71 72 73 74 75 76 77
  deviceId = args['device-id'] as String?;
  exitOnFirstTestFailure = (args['exit'] as bool?) ?? false;
  gitBranch = args['git-branch'] as String?;
  localEngine = args['local-engine'] as String?;
  localEngineSrcPath = args['local-engine-src-path'] as String?;
  luciBuilder = args['luci-builder'] as String?;
  resultsPath = args['results-file'] as String?;
  serviceAccountTokenFile = args['service-account-token-file'] as String?;
  silent = (args['silent'] as bool?) ?? false;
78

79
  if (!args.wasParsed('task')) {
80 81 82 83 84 85
    if (args.wasParsed('stage') || args.wasParsed('all')) {
      addTasks(
        tasks: loadTaskManifest().tasks,
        args: args,
        taskNames: _taskNames,
      );
86 87 88
    }
  }

89 90 91 92 93 94 95 96
  if (args.wasParsed('list')) {
    for (int i = 0; i < _taskNames.length; i++) {
      print('${(i + 1).toString().padLeft(3)} - ${_taskNames[i]}');
    }
    exitCode = 0;
    return;
  }

97
  if (_taskNames.isEmpty) {
98 99
    stderr.writeln('Failed to find tasks to run based on supplied options.');
    exitCode = 1;
100
    return;
101 102
  }

103 104 105
  if (args.wasParsed('ab')) {
    await _runABTest();
  } else {
106
    await runTasks(_taskNames,
107
      silent: silent,
108 109
      localEngine: localEngine,
      localEngineSrcPath: localEngineSrcPath,
110
      deviceId: deviceId,
111 112 113 114
      exitOnFirstTestFailure: exitOnFirstTestFailure,
      gitBranch: gitBranch,
      luciBuilder: luciBuilder,
      resultsPath: resultsPath,
115
    );
116 117 118
  }
}

119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
Future<void> _runABTest() async {
  final int runsPerTest = int.parse(args['ab'] as String);

  if (_taskNames.length > 1) {
    stderr.writeln('When running in A/B test mode exactly one task must be passed but got ${_taskNames.join(', ')}.\n');
    stderr.writeln(_argParser.usage);
    exit(1);
  }

  if (!args.wasParsed('local-engine')) {
    stderr.writeln('When running in A/B test mode --local-engine is required.\n');
    stderr.writeln(_argParser.usage);
    exit(1);
  }

  final String taskName = _taskNames.single;

  print('$taskName A/B test. Will run $runsPerTest times.');

138
  final ABTest abTest = ABTest(localEngine!, taskName);
139 140 141 142
  for (int i = 1; i <= runsPerTest; i++) {
    section('Run #$i');

    print('Running with the default engine (A)');
143
    final TaskResult defaultEngineResult = await runTask(
144 145
      taskName,
      silent: silent,
146
      deviceId: deviceId,
147 148 149 150 151
    );

    print('Default engine result:');
    print(const JsonEncoder.withIndent('  ').convert(defaultEngineResult));

152
    if (!defaultEngineResult.succeeded) {
153 154 155 156 157 158 159
      stderr.writeln('Task failed on the default engine.');
      exit(1);
    }

    abTest.addAResult(defaultEngineResult);

    print('Running with the local engine (B)');
160
    final TaskResult localEngineResult = await runTask(
161 162 163 164
      taskName,
      silent: silent,
      localEngine: localEngine,
      localEngineSrcPath: localEngineSrcPath,
165
      deviceId: deviceId,
166 167 168 169 170
    );

    print('Task localEngineResult:');
    print(const JsonEncoder.withIndent('  ').convert(localEngineResult));

171
    if (!localEngineResult.succeeded) {
172 173 174 175 176
      stderr.writeln('Task failed on the local engine.');
      exit(1);
    }

    abTest.addBResult(localEngineResult);
177

178
    if (silent != true && i < runsPerTest) {
179 180 181
      section('A/B results so far');
      print(abTest.printSummary());
    }
182
  }
183 184
  abTest.finalize();

185
  final File jsonFile = _uniqueFile(args['ab-result-file'] as String? ?? 'ABresults#.json');
186
  jsonFile.writeAsStringSync(const JsonEncoder.withIndent('  ').convert(abTest.jsonMap));
187

188
  if (silent != true) {
189 190 191 192 193
    section('Raw results');
    print(abTest.rawResults());
  }

  section('Final A/B results');
194
  print(abTest.printSummary());
195 196 197 198 199 200 201 202 203 204 205 206 207

  print('');
  print('Results saved to ${jsonFile.path}');
}

File _uniqueFile(String filenameTemplate) {
  final List<String> parts = filenameTemplate.split('#');
  if (parts.length != 2) {
    return File(filenameTemplate);
  }
  File file = File(parts[0] + parts[1]);
  int i = 1;
  while (file.existsSync()) {
208
    file = File(parts[0] + i.toString() + parts[1]);
209 210 211
    i++;
  }
  return file;
212 213
}

214
void addTasks({
215 216 217
  required List<ManifestTask> tasks,
  required ArgResults args,
  required List<String> taskNames,
218 219 220 221 222 223 224 225 226
}) {
  if (args.wasParsed('continue-from')) {
    final int index = tasks.indexWhere((ManifestTask task) => task.name == args['continue-from']);
    if (index == -1) {
      throw Exception('Invalid task name "${args['continue-from']}"');
    }
    tasks.removeRange(0, index);
  }
  // Only start skipping if user specified a task to continue from
227
  final String stage = args['stage'] as String;
228
  for (final ManifestTask task in tasks) {
229
    final bool isQualifyingStage = stage == null || task.stage == stage;
230
    final bool isQualifyingHost = !(args['match-host-platform'] as bool) || task.isSupportedByHost();
231 232 233
    if (isQualifyingHost && isQualifyingStage) {
      taskNames.add(task.name);
    }
234 235 236 237
  }
}

/// Command-line options for the `run.dart` command.
238
final ArgParser _argParser = ArgParser()
239
  ..addMultiOption(
240 241 242
    'task',
    abbr: 't',
    splitCommas: true,
243
    help: 'Either:\n'
244 245 246 247 248
        ' - the name of a task defined in manifest.yaml.\n'
        '   Example: complex_layout__start_up.\n'
        ' - the path to a Dart file corresponding to a task,\n'
        '   which resides in bin/tasks.\n'
        '   Example: bin/tasks/complex_layout__start_up.dart.\n'
249 250 251
        '\n'
        'This option may be repeated to specify multiple tasks.',
    callback: (List<String> value) {
252
      for (final String nameOrPath in value) {
253 254
        final List<String> fragments = path.split(nameOrPath);
        final bool isDartFile = fragments.last.endsWith('.dart');
255 256 257 258

        if (fragments.length == 1 && !isDartFile) {
          // Not a path
          _taskNames.add(nameOrPath);
259
        } else if (!isDartFile || !path.equals(path.dirname(nameOrPath), path.join('bin', 'tasks'))) {
260
          // Unsupported executable location
261
          throw FormatException('Invalid value for option -t (--task): $nameOrPath');
262 263 264 265 266
        } else {
          _taskNames.add(path.withoutExtension(fragments.last));
        }
      }
    },
267
  )
268 269 270 271 272 273 274 275 276
  ..addOption(
    'device-id',
    abbr: 'd',
    help: 'Target device id (prefixes are allowed, names are not supported).\n'
          'The option will be ignored if the test target does not run on a\n'
          'mobile device. This still respects the device operating system\n'
          'settings in the test case, and will results in error if no device\n'
          'with given ID/ID prefix is found.',
  )
277 278 279 280 281 282 283 284 285 286
  ..addOption(
    'ab',
    help: 'Runs an A/B test comparing the default engine with the local\n'
          'engine build for one task. This option does not support running\n'
          'multiple tasks. The value is the number of times to run the task.\n'
          'The task is expected to be a benchmark that reports score keys.\n'
          'The A/B test collects the metrics collected by the test and\n'
          'produces a report containing averages, noise, and the speed-up\n'
          'between the two engines. --local-engine is required when running\n'
          'an A/B test.',
287
    callback: (String? value) {
288 289 290 291 292
      if (value != null && int.tryParse(value) == null) {
        throw ArgParserException('Option --ab must be a number, but was "$value".');
      }
    },
  )
293 294 295 296 297 298
  ..addOption(
    'ab-result-file',
    help: 'The filename in which to place the json encoded results of an A/B test.\n'
          'The filename may contain a single # character to be replaced by a sequence\n'
          'number if the name already exists.',
  )
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313
  ..addFlag(
    'all',
    abbr: 'a',
    help: 'Runs all tasks defined in manifest.yaml in alphabetical order.',
  )
  ..addOption(
    'continue-from',
    abbr: 'c',
    help: 'With --all or --stage, continue from the given test.',
  )
  ..addFlag(
    'exit',
    defaultsTo: true,
    help: 'Exit on the first test failure.',
  )
314 315 316 317 318
  ..addOption(
    'git-branch',
    help: '[Flutter infrastructure] Git branch of the current commit. LUCI\n'
          'checkouts run in detached HEAD state, so the branch must be passed.',
  )
319 320 321 322 323
  ..addOption(
    'local-engine',
    help: 'Name of a build output within the engine out directory, if you\n'
          'are building Flutter locally. Use this to select a specific\n'
          'version of the engine if you have built multiple engine targets.\n'
324 325
          'This path is relative to --local-engine-src-path/out. This option\n'
          'is required when running an A/B test (see the --ab option).',
326 327 328 329
  )
  ..addFlag(
    'list',
    abbr: 'l',
330
    help: "Don't actually run the tasks, but list out the tasks that would\n"
331 332 333 334 335 336 337 338
          'have been run, in the order they would have run.',
  )
  ..addOption(
    'local-engine-src-path',
    help: 'Path to your engine src directory, if you are building Flutter\n'
          'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
          'the location based on the value of the --flutter-root option.',
  )
339
  ..addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.')
340 341 342 343 344
  ..addFlag(
    'match-host-platform',
    defaultsTo: true,
    help: 'Only run tests that match the host platform (e.g. do not run a\n'
          'test with a `required_agent_capabilities` value of "mac/android"\n'
345
          'on a windows host). Each test publishes its '
346 347
          '`required_agent_capabilities`\nin the `manifest.yaml` file.',
  )
348 349 350 351 352
  ..addOption(
    'results-file',
    help: '[Flutter infrastructure] File path for test results. If passed with\n'
          'task, will write test results to the file.'
  )
353 354 355 356
  ..addOption(
    'service-account-token-file',
    help: '[Flutter infrastructure] Authentication for uploading results.',
  )
357 358 359
  ..addOption(
    'stage',
    abbr: 's',
360 361
    help: 'Name of the stage. Runs all tasks for that stage. The tasks and\n'
          'their stages are read from manifest.yaml.',
362
  )
363
  ..addFlag(
364 365 366
    'silent',
    negatable: true,
    defaultsTo: false,
367
  )
368
  ..addMultiOption(
369 370 371 372 373
    'test',
    hide: true,
    splitCommas: true,
    callback: (List<String> value) {
      if (value.isNotEmpty) {
374
        throw const FormatException(
375 376 377 378 379
          'Invalid option --test. Did you mean --task (-t)?',
        );
      }
    },
  );