run.dart 9.06 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6 7 8 9
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io';

import 'package:args/args.dart';
10
import 'package:path/path.dart' as path;
11

12
import 'package:flutter_devicelab/framework/ab.dart';
13 14 15 16
import 'package:flutter_devicelab/framework/manifest.dart';
import 'package:flutter_devicelab/framework/runner.dart';
import 'package:flutter_devicelab/framework/utils.dart';

17 18
ArgResults args;

19 20
List<String> _taskNames = <String>[];

21 22 23 24 25 26 27 28 29 30 31 32 33 34
/// Suppresses standard output, prints only standard error output.
bool silent;

/// The build of the local engine to use.
///
/// Required for A/B test mode.
String localEngine;

/// The path to the engine "src/" directory.
String localEngineSrcPath;

/// Whether to exit on first test failure.
bool exitOnFirstTestFailure;

35 36 37 38
/// Runs tasks.
///
/// The tasks are chosen depending on the command-line options
/// (see [_argParser]).
39
Future<void> main(List<String> rawArgs) async {
40 41
  try {
    args = _argParser.parse(rawArgs);
42
  } on FormatException catch (error) {
43 44 45 46
    stderr.writeln('${error.message}\n');
    stderr.writeln('Usage:\n');
    stderr.writeln(_argParser.usage);
    exitCode = 1;
47
    return;
48 49
  }

50
  if (!args.wasParsed('task')) {
51 52 53 54 55 56
    if (args.wasParsed('stage') || args.wasParsed('all')) {
      addTasks(
        tasks: loadTaskManifest().tasks,
        args: args,
        taskNames: _taskNames,
      );
57 58 59
    }
  }

60 61 62 63 64 65 66 67
  if (args.wasParsed('list')) {
    for (int i = 0; i < _taskNames.length; i++) {
      print('${(i + 1).toString().padLeft(3)} - ${_taskNames[i]}');
    }
    exitCode = 0;
    return;
  }

68
  if (_taskNames.isEmpty) {
69 70
    stderr.writeln('Failed to find tasks to run based on supplied options.');
    exitCode = 1;
71
    return;
72 73
  }

74 75 76 77
  silent = args['silent'] as bool;
  localEngine = args['local-engine'] as String;
  localEngineSrcPath = args['local-engine-src-path'] as String;
  exitOnFirstTestFailure = args['exit'] as bool;
78

79 80 81 82 83 84 85 86
  if (args.wasParsed('ab')) {
    await _runABTest();
  } else {
    await _runTasks();
  }
}

Future<void> _runTasks() async {
87
  for (final String taskName in _taskNames) {
88
    section('Running task "$taskName"');
89 90 91 92 93 94
    final Map<String, dynamic> result = await runTask(
      taskName,
      silent: silent,
      localEngine: localEngine,
      localEngineSrcPath: localEngineSrcPath,
    );
95 96

    print('Task result:');
97
    print(const JsonEncoder.withIndent('  ').convert(result));
98
    section('Finished task "$taskName"');
99

100
    if (!(result['success'] as bool)) {
101
      exitCode = 1;
102
      if (exitOnFirstTestFailure) {
103 104 105 106 107 108
        return;
      }
    }
  }
}

109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
Future<void> _runABTest() async {
  final int runsPerTest = int.parse(args['ab'] as String);

  if (_taskNames.length > 1) {
    stderr.writeln('When running in A/B test mode exactly one task must be passed but got ${_taskNames.join(', ')}.\n');
    stderr.writeln(_argParser.usage);
    exit(1);
  }

  if (!args.wasParsed('local-engine')) {
    stderr.writeln('When running in A/B test mode --local-engine is required.\n');
    stderr.writeln(_argParser.usage);
    exit(1);
  }

  final String taskName = _taskNames.single;

  print('$taskName A/B test. Will run $runsPerTest times.');

  final ABTest abTest = ABTest();
  for (int i = 1; i <= runsPerTest; i++) {
    section('Run #$i');

    print('Running with the default engine (A)');
    final Map<String, dynamic> defaultEngineResult = await runTask(
      taskName,
      silent: silent,
    );

    print('Default engine result:');
    print(const JsonEncoder.withIndent('  ').convert(defaultEngineResult));

    if (!(defaultEngineResult['success'] as bool)) {
      stderr.writeln('Task failed on the default engine.');
      exit(1);
    }

    abTest.addAResult(defaultEngineResult);

    print('Running with the local engine (B)');
    final Map<String, dynamic> localEngineResult = await runTask(
      taskName,
      silent: silent,
      localEngine: localEngine,
      localEngineSrcPath: localEngineSrcPath,
    );

    print('Task localEngineResult:');
    print(const JsonEncoder.withIndent('  ').convert(localEngineResult));

    if (!(localEngineResult['success'] as bool)) {
      stderr.writeln('Task failed on the local engine.');
      exit(1);
    }

    abTest.addBResult(localEngineResult);
  }
  print(abTest.printSummary());
}

169 170 171 172 173 174 175 176 177 178 179 180 181
void addTasks({
  List<ManifestTask> tasks,
  ArgResults args,
  List<String> taskNames,
}) {
  if (args.wasParsed('continue-from')) {
    final int index = tasks.indexWhere((ManifestTask task) => task.name == args['continue-from']);
    if (index == -1) {
      throw Exception('Invalid task name "${args['continue-from']}"');
    }
    tasks.removeRange(0, index);
  }
  // Only start skipping if user specified a task to continue from
182
  final String stage = args['stage'] as String;
183
  for (final ManifestTask task in tasks) {
184
    final bool isQualifyingStage = stage == null || task.stage == stage;
185
    final bool isQualifyingHost = !(args['match-host-platform'] as bool) || task.isSupportedByHost();
186 187 188
    if (isQualifyingHost && isQualifyingStage) {
      taskNames.add(task.name);
    }
189 190 191 192
  }
}

/// Command-line options for the `run.dart` command.
193
final ArgParser _argParser = ArgParser()
194
  ..addMultiOption(
195 196 197
    'task',
    abbr: 't',
    splitCommas: true,
198 199 200 201 202 203
    help: 'Either:\n'
        ' - the name of a task defined in manifest.yaml. Example: complex_layout__start_up.\n'
        ' - the path to a Dart file corresponding to a task, which resides in bin/tasks. Example: bin/tasks/complex_layout__start_up.dart.\n'
        '\n'
        'This option may be repeated to specify multiple tasks.',
    callback: (List<String> value) {
204
      for (final String nameOrPath in value) {
205 206
        final List<String> fragments = path.split(nameOrPath);
        final bool isDartFile = fragments.last.endsWith('.dart');
207 208 209 210

        if (fragments.length == 1 && !isDartFile) {
          // Not a path
          _taskNames.add(nameOrPath);
211
        } else if (!isDartFile || !path.equals(path.dirname(nameOrPath), path.join('bin', 'tasks'))) {
212
          // Unsupported executable location
213
          throw FormatException('Invalid value for option -t (--task): $nameOrPath');
214 215 216 217 218
        } else {
          _taskNames.add(path.withoutExtension(fragments.last));
        }
      }
    },
219
  )
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
  ..addOption(
    'ab',
    help: 'Runs an A/B test comparing the default engine with the local\n'
          'engine build for one task. This option does not support running\n'
          'multiple tasks. The value is the number of times to run the task.\n'
          'The task is expected to be a benchmark that reports score keys.\n'
          'The A/B test collects the metrics collected by the test and\n'
          'produces a report containing averages, noise, and the speed-up\n'
          'between the two engines. --local-engine is required when running\n'
          'an A/B test.',
    callback: (String value) {
      if (value != null && int.tryParse(value) == null) {
        throw ArgParserException('Option --ab must be a number, but was "$value".');
      }
    },
  )
236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
  ..addFlag(
    'all',
    abbr: 'a',
    help: 'Runs all tasks defined in manifest.yaml in alphabetical order.',
  )
  ..addOption(
    'continue-from',
    abbr: 'c',
    help: 'With --all or --stage, continue from the given test.',
  )
  ..addFlag(
    'exit',
    defaultsTo: true,
    help: 'Exit on the first test failure.',
  )
  ..addOption(
    'local-engine',
    help: 'Name of a build output within the engine out directory, if you\n'
          'are building Flutter locally. Use this to select a specific\n'
          'version of the engine if you have built multiple engine targets.\n'
256 257
          'This path is relative to --local-engine-src-path/out. This option\n'
          'is required when running an A/B test (see the --ab option).',
258 259 260 261
  )
  ..addFlag(
    'list',
    abbr: 'l',
262
    help: "Don't actually run the tasks, but list out the tasks that would\n"
263 264 265 266 267 268 269 270 271 272 273 274 275
          'have been run, in the order they would have run.',
  )
  ..addOption(
    'local-engine-src-path',
    help: 'Path to your engine src directory, if you are building Flutter\n'
          'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
          'the location based on the value of the --flutter-root option.',
  )
  ..addFlag(
    'match-host-platform',
    defaultsTo: true,
    help: 'Only run tests that match the host platform (e.g. do not run a\n'
          'test with a `required_agent_capabilities` value of "mac/android"\n'
276
          'on a windows host). Each test publishes its '
277 278
          '`required_agent_capabilities`\nin the `manifest.yaml` file.',
  )
279 280 281
  ..addOption(
    'stage',
    abbr: 's',
282 283
    help: 'Name of the stage. Runs all tasks for that stage. The tasks and\n'
          'their stages are read from manifest.yaml.',
284
  )
285
  ..addFlag(
286 287 288
    'silent',
    negatable: true,
    defaultsTo: false,
289
  )
290
  ..addMultiOption(
291 292 293 294 295
    'test',
    hide: true,
    splitCommas: true,
    callback: (List<String> value) {
      if (value.isNotEmpty) {
296
        throw const FormatException(
297 298 299 300 301
          'Invalid option --test. Did you mean --task (-t)?',
        );
      }
    },
  );