run.dart 12.6 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6 7 8
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io';

import 'package:args/args.dart';
9
import 'package:path/path.dart' as path;
10

11
import 'package:flutter_devicelab/framework/ab.dart';
12
import 'package:flutter_devicelab/framework/cocoon.dart';
13 14
import 'package:flutter_devicelab/framework/manifest.dart';
import 'package:flutter_devicelab/framework/runner.dart';
15
import 'package:flutter_devicelab/framework/task_result.dart';
16 17
import 'package:flutter_devicelab/framework/utils.dart';

18 19
ArgResults args;

20 21
List<String> _taskNames = <String>[];

22 23
/// The device-id to run test on.
String deviceId;
24

25 26 27
/// The git branch being tested on.
String gitBranch;

28 29 30 31 32 33 34 35
/// The build of the local engine to use.
///
/// Required for A/B test mode.
String localEngine;

/// The path to the engine "src/" directory.
String localEngineSrcPath;

36 37 38 39 40 41
/// Name of the LUCI builder this test is currently running on.
///
/// This is only passed on CI runs for Cocoon to be able to uniquely identify
/// this test run.
String luciBuilder;

42 43 44
/// Whether to exit on first test failure.
bool exitOnFirstTestFailure;

45 46 47
/// Path to write test results to.
String resultsPath;

48 49 50 51 52 53 54
/// File containing a service account token.
///
/// If passed, the test run results will be uploaded to Flutter infrastructure.
String serviceAccountTokenFile;

/// Suppresses standard output, prints only standard error output.
bool silent;
55

56 57 58 59
/// Runs tasks.
///
/// The tasks are chosen depending on the command-line options
/// (see [_argParser]).
60
Future<void> main(List<String> rawArgs) async {
61 62
  try {
    args = _argParser.parse(rawArgs);
63
  } on FormatException catch (error) {
64 65 66 67
    stderr.writeln('${error.message}\n');
    stderr.writeln('Usage:\n');
    stderr.writeln(_argParser.usage);
    exitCode = 1;
68
    return;
69 70
  }

71 72 73 74 75 76 77 78 79 80
  deviceId = args['device-id'] as String;
  exitOnFirstTestFailure = args['exit'] as bool;
  gitBranch = args['git-branch'] as String;
  localEngine = args['local-engine'] as String;
  localEngineSrcPath = args['local-engine-src-path'] as String;
  luciBuilder = args['luci-builder'] as String;
  resultsPath = args['results-file'] as String;
  serviceAccountTokenFile = args['service-account-token-file'] as String;
  silent = args['silent'] as bool;

81
  if (!args.wasParsed('task')) {
82 83 84 85 86 87
    if (args.wasParsed('stage') || args.wasParsed('all')) {
      addTasks(
        tasks: loadTaskManifest().tasks,
        args: args,
        taskNames: _taskNames,
      );
88 89 90
    }
  }

91 92 93 94 95 96 97 98
  if (args.wasParsed('list')) {
    for (int i = 0; i < _taskNames.length; i++) {
      print('${(i + 1).toString().padLeft(3)} - ${_taskNames[i]}');
    }
    exitCode = 0;
    return;
  }

99
  if (_taskNames.isEmpty) {
100 101
    stderr.writeln('Failed to find tasks to run based on supplied options.');
    exitCode = 1;
102
    return;
103 104
  }

105 106 107 108 109 110 111 112
  if (args.wasParsed('ab')) {
    await _runABTest();
  } else {
    await _runTasks();
  }
}

Future<void> _runTasks() async {
113
  for (final String taskName in _taskNames) {
114
    section('Running task "$taskName"');
115
    final TaskResult result = await runTask(
116 117 118 119
      taskName,
      silent: silent,
      localEngine: localEngine,
      localEngineSrcPath: localEngineSrcPath,
120
      deviceId: deviceId,
121
    );
122 123

    print('Task result:');
124
    print(const JsonEncoder.withIndent('  ').convert(result));
125
    section('Finished task "$taskName"');
126

127 128 129 130 131 132 133 134 135
    if (resultsPath != null) {
      final Cocoon cocoon = Cocoon();
      await cocoon.writeTaskResultToFile(
        builderName: luciBuilder,
        gitBranch: gitBranch,
        result: result,
        resultsPath: resultsPath,
      );
    } else if (serviceAccountTokenFile != null) {
136
      final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
137

138
      /// Cocoon references LUCI tasks by the [luciBuilder] instead of [taskName].
139
      await cocoon.sendTaskResult(builderName: luciBuilder, result: result, gitBranch: gitBranch);
140 141
    }

142
    if (!result.succeeded) {
143
      exitCode = 1;
144
      if (exitOnFirstTestFailure) {
145 146 147 148 149 150
        return;
      }
    }
  }
}

151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169
Future<void> _runABTest() async {
  final int runsPerTest = int.parse(args['ab'] as String);

  if (_taskNames.length > 1) {
    stderr.writeln('When running in A/B test mode exactly one task must be passed but got ${_taskNames.join(', ')}.\n');
    stderr.writeln(_argParser.usage);
    exit(1);
  }

  if (!args.wasParsed('local-engine')) {
    stderr.writeln('When running in A/B test mode --local-engine is required.\n');
    stderr.writeln(_argParser.usage);
    exit(1);
  }

  final String taskName = _taskNames.single;

  print('$taskName A/B test. Will run $runsPerTest times.');

170
  final ABTest abTest = ABTest(localEngine, taskName);
171 172 173 174
  for (int i = 1; i <= runsPerTest; i++) {
    section('Run #$i');

    print('Running with the default engine (A)');
175
    final TaskResult defaultEngineResult = await runTask(
176 177
      taskName,
      silent: silent,
178
      deviceId: deviceId,
179 180 181 182 183
    );

    print('Default engine result:');
    print(const JsonEncoder.withIndent('  ').convert(defaultEngineResult));

184
    if (!defaultEngineResult.succeeded) {
185 186 187 188 189 190 191
      stderr.writeln('Task failed on the default engine.');
      exit(1);
    }

    abTest.addAResult(defaultEngineResult);

    print('Running with the local engine (B)');
192
    final TaskResult localEngineResult = await runTask(
193 194 195 196
      taskName,
      silent: silent,
      localEngine: localEngine,
      localEngineSrcPath: localEngineSrcPath,
197
      deviceId: deviceId,
198 199 200 201 202
    );

    print('Task localEngineResult:');
    print(const JsonEncoder.withIndent('  ').convert(localEngineResult));

203
    if (!localEngineResult.succeeded) {
204 205 206 207 208
      stderr.writeln('Task failed on the local engine.');
      exit(1);
    }

    abTest.addBResult(localEngineResult);
209 210 211 212 213

    if (!silent && i < runsPerTest) {
      section('A/B results so far');
      print(abTest.printSummary());
    }
214
  }
215 216 217 218
  abTest.finalize();

  final File jsonFile = _uniqueFile(args['ab-result-file'] as String ?? 'ABresults#.json');
  jsonFile.writeAsString(const JsonEncoder.withIndent('  ').convert(abTest.jsonMap));
219 220 221 222 223 224 225

  if (!silent) {
    section('Raw results');
    print(abTest.rawResults());
  }

  section('Final A/B results');
226
  print(abTest.printSummary());
227 228 229 230 231 232 233 234 235 236 237 238 239

  print('');
  print('Results saved to ${jsonFile.path}');
}

File _uniqueFile(String filenameTemplate) {
  final List<String> parts = filenameTemplate.split('#');
  if (parts.length != 2) {
    return File(filenameTemplate);
  }
  File file = File(parts[0] + parts[1]);
  int i = 1;
  while (file.existsSync()) {
240
    file = File(parts[0] + i.toString() + parts[1]);
241 242 243
    i++;
  }
  return file;
244 245
}

246 247 248 249 250 251 252 253 254 255 256 257 258
void addTasks({
  List<ManifestTask> tasks,
  ArgResults args,
  List<String> taskNames,
}) {
  if (args.wasParsed('continue-from')) {
    final int index = tasks.indexWhere((ManifestTask task) => task.name == args['continue-from']);
    if (index == -1) {
      throw Exception('Invalid task name "${args['continue-from']}"');
    }
    tasks.removeRange(0, index);
  }
  // Only start skipping if user specified a task to continue from
259
  final String stage = args['stage'] as String;
260
  for (final ManifestTask task in tasks) {
261
    final bool isQualifyingStage = stage == null || task.stage == stage;
262
    final bool isQualifyingHost = !(args['match-host-platform'] as bool) || task.isSupportedByHost();
263 264 265
    if (isQualifyingHost && isQualifyingStage) {
      taskNames.add(task.name);
    }
266 267 268 269
  }
}

/// Command-line options for the `run.dart` command.
270
final ArgParser _argParser = ArgParser()
271
  ..addMultiOption(
272 273 274
    'task',
    abbr: 't',
    splitCommas: true,
275
    help: 'Either:\n'
276 277 278 279 280
        ' - the name of a task defined in manifest.yaml.\n'
        '   Example: complex_layout__start_up.\n'
        ' - the path to a Dart file corresponding to a task,\n'
        '   which resides in bin/tasks.\n'
        '   Example: bin/tasks/complex_layout__start_up.dart.\n'
281 282 283
        '\n'
        'This option may be repeated to specify multiple tasks.',
    callback: (List<String> value) {
284
      for (final String nameOrPath in value) {
285 286
        final List<String> fragments = path.split(nameOrPath);
        final bool isDartFile = fragments.last.endsWith('.dart');
287 288 289 290

        if (fragments.length == 1 && !isDartFile) {
          // Not a path
          _taskNames.add(nameOrPath);
291
        } else if (!isDartFile || !path.equals(path.dirname(nameOrPath), path.join('bin', 'tasks'))) {
292
          // Unsupported executable location
293
          throw FormatException('Invalid value for option -t (--task): $nameOrPath');
294 295 296 297 298
        } else {
          _taskNames.add(path.withoutExtension(fragments.last));
        }
      }
    },
299
  )
300 301 302 303 304 305 306 307 308
  ..addOption(
    'device-id',
    abbr: 'd',
    help: 'Target device id (prefixes are allowed, names are not supported).\n'
          'The option will be ignored if the test target does not run on a\n'
          'mobile device. This still respects the device operating system\n'
          'settings in the test case, and will results in error if no device\n'
          'with given ID/ID prefix is found.',
  )
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
  ..addOption(
    'ab',
    help: 'Runs an A/B test comparing the default engine with the local\n'
          'engine build for one task. This option does not support running\n'
          'multiple tasks. The value is the number of times to run the task.\n'
          'The task is expected to be a benchmark that reports score keys.\n'
          'The A/B test collects the metrics collected by the test and\n'
          'produces a report containing averages, noise, and the speed-up\n'
          'between the two engines. --local-engine is required when running\n'
          'an A/B test.',
    callback: (String value) {
      if (value != null && int.tryParse(value) == null) {
        throw ArgParserException('Option --ab must be a number, but was "$value".');
      }
    },
  )
325 326 327 328 329 330
  ..addOption(
    'ab-result-file',
    help: 'The filename in which to place the json encoded results of an A/B test.\n'
          'The filename may contain a single # character to be replaced by a sequence\n'
          'number if the name already exists.',
  )
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345
  ..addFlag(
    'all',
    abbr: 'a',
    help: 'Runs all tasks defined in manifest.yaml in alphabetical order.',
  )
  ..addOption(
    'continue-from',
    abbr: 'c',
    help: 'With --all or --stage, continue from the given test.',
  )
  ..addFlag(
    'exit',
    defaultsTo: true,
    help: 'Exit on the first test failure.',
  )
346 347 348 349 350
  ..addOption(
    'git-branch',
    help: '[Flutter infrastructure] Git branch of the current commit. LUCI\n'
          'checkouts run in detached HEAD state, so the branch must be passed.',
  )
351 352 353 354 355
  ..addOption(
    'local-engine',
    help: 'Name of a build output within the engine out directory, if you\n'
          'are building Flutter locally. Use this to select a specific\n'
          'version of the engine if you have built multiple engine targets.\n'
356 357
          'This path is relative to --local-engine-src-path/out. This option\n'
          'is required when running an A/B test (see the --ab option).',
358 359 360 361
  )
  ..addFlag(
    'list',
    abbr: 'l',
362
    help: "Don't actually run the tasks, but list out the tasks that would\n"
363 364 365 366 367 368 369 370
          'have been run, in the order they would have run.',
  )
  ..addOption(
    'local-engine-src-path',
    help: 'Path to your engine src directory, if you are building Flutter\n'
          'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
          'the location based on the value of the --flutter-root option.',
  )
371
  ..addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.')
372 373 374 375 376
  ..addFlag(
    'match-host-platform',
    defaultsTo: true,
    help: 'Only run tests that match the host platform (e.g. do not run a\n'
          'test with a `required_agent_capabilities` value of "mac/android"\n'
377
          'on a windows host). Each test publishes its '
378 379
          '`required_agent_capabilities`\nin the `manifest.yaml` file.',
  )
380 381 382 383 384
  ..addOption(
    'results-file',
    help: '[Flutter infrastructure] File path for test results. If passed with\n'
          'task, will write test results to the file.'
  )
385 386 387 388
  ..addOption(
    'service-account-token-file',
    help: '[Flutter infrastructure] Authentication for uploading results.',
  )
389 390 391
  ..addOption(
    'stage',
    abbr: 's',
392 393
    help: 'Name of the stage. Runs all tasks for that stage. The tasks and\n'
          'their stages are read from manifest.yaml.',
394
  )
395
  ..addFlag(
396 397 398
    'silent',
    negatable: true,
    defaultsTo: false,
399
  )
400
  ..addMultiOption(
401 402 403 404 405
    'test',
    hide: true,
    splitCommas: true,
    callback: (List<String> value) {
      if (value.isNotEmpty) {
406
        throw const FormatException(
407 408 409 410 411
          'Invalid option --test. Did you mean --task (-t)?',
        );
      }
    },
  );