add --exit and --match-host-platform defaults to devicelab runner (#37832)

dev/devicelab/README.md

@@ -122,6 +122,9 @@ To run all tests defined in `manifest.yaml`, use option `-a` (`--all`):
 ../../bin/cache/dart-sdk/bin/dart bin/run.dart -a
 ```
 
+This defaults to only running tests supported by your host device's platform
+(`--match-host-platform`) and exiting after the first failure (`--exit`).
+
 ## Running specific tests
 
 To run a test, use option `-t` (`--task`):

dev/devicelab/bin/run.dart

@@ -32,21 +32,23 @@ Future<void> main(List<String> rawArgs) async {
   }
 
   if (!args.wasParsed('task')) {
-    if (args.wasParsed('stage')) {
-      final String stageName = args['stage'];
-      final List<ManifestTask> tasks = loadTaskManifest().tasks;
-      for (ManifestTask task in tasks) {
-        if (task.stage == stageName)
-          _taskNames.add(task.name);
-      }
-    } else if (args.wasParsed('all')) {
-      final List<ManifestTask> tasks = loadTaskManifest().tasks;
-      for (ManifestTask task in tasks) {
-        _taskNames.add(task.name);
-      }
+    if (args.wasParsed('stage') || args.wasParsed('all')) {
+      addTasks(
+        tasks: loadTaskManifest().tasks,
+        args: args,
+        taskNames: _taskNames,
+      );
     }
   }
 
+  if (args.wasParsed('list')) {
+    for (int i = 0; i < _taskNames.length; i++) {
+      print('${(i + 1).toString().padLeft(3)} - ${_taskNames[i]}');
+    }
+    exitCode = 0;
+    return;
+  }
+
   if (_taskNames.isEmpty) {
     stderr.writeln('Failed to find tasks to run based on supplied options.');
     exitCode = 1;
@@ -66,12 +68,39 @@ Future<void> main(List<String> rawArgs) async {
       localEngineSrcPath: localEngineSrcPath,
     );
 
-    if (!result['success'])
-      exitCode = 1;
-
     print('Task result:');
     print(const JsonEncoder.withIndent('  ').convert(result));
     section('Finished task "$taskName"');
+
+    if (!result['success']) {
+      exitCode = 1;
+      if (args['exit']) {
+        return;
+      }
+    }
+  }
+}
+
+void addTasks({
+  List<ManifestTask> tasks,
+  ArgResults args,
+  List<String> taskNames,
+}) {
+  if (args.wasParsed('continue-from')) {
+    final int index = tasks.indexWhere((ManifestTask task) => task.name == args['continue-from']);
+    if (index == -1) {
+      throw Exception('Invalid task name "${args['continue-from']}"');
+    }
+    tasks.removeRange(0, index);
+  }
+  // Only start skipping if user specified a task to continue from
+  final String stage = args['stage'];
+  for (ManifestTask task in tasks) {
+    final bool isQualifyingStage = stage == null || task.stage == stage;
+    final bool isQualifyingHost = !args['match-host-platform'] || task.isSupportedByHost();
+    if (isQualifyingHost && isQualifyingStage) {
+      taskNames.add(task.name);
+    }
   }
 }
 
@@ -103,16 +132,58 @@ final ArgParser _argParser = ArgParser()
       }
     },
   )
+  ..addFlag(
+    'all',
+    abbr: 'a',
+    help: 'Runs all tasks defined in manifest.yaml in alphabetical order.',
+  )
+  ..addOption(
+    'continue-from',
+    abbr: 'c',
+    help: 'With --all or --stage, continue from the given test.',
+  )
+  ..addFlag(
+    'exit',
+    defaultsTo: true,
+    help: 'Exit on the first test failure.',
+  )
+  ..addOption(
+    'local-engine',
+    help: 'Name of a build output within the engine out directory, if you\n'
+          'are building Flutter locally. Use this to select a specific\n'
+          'version of the engine if you have built multiple engine targets.\n'
+          'This path is relative to --local-engine-src-path/out.',
+  )
+  ..addFlag(
+    'list',
+    abbr: 'l',
+    help: 'Don\'t actually run the tasks, but list out the tasks that would\n'
+          'have been run, in the order they would have run.',
+  )
+  ..addOption(
+    'local-engine-src-path',
+    help: 'Path to your engine src directory, if you are building Flutter\n'
+          'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
+          'the location based on the value of the --flutter-root option.',
+  )
+  ..addFlag(
+    'match-host-platform',
+    defaultsTo: true,
+    help: 'Only run tests that match the host platform (e.g. do not run a\n'
+          'test with a `required_agent_capabilities` value of "mac/android"\n'
+          'on a windows host). Each test publishes its'
+          '`required_agent_capabilities`\nin the `manifest.yaml` file.',
+  )
   ..addOption(
     'stage',
     abbr: 's',
-    help: 'Name of the stage. Runs all tasks for that stage. '
-          'The tasks and their stages are read from manifest.yaml.',
+    help: 'Name of the stage. Runs all tasks for that stage. The tasks and\n'
+          'their stages are read from manifest.yaml.',
   )
   ..addFlag(
-    'all',
-    abbr: 'a',
-    help: 'Runs all tasks defined in manifest.yaml.',
+    'silent',
+    negatable: true,
+    defaultsTo: false,
   )
   ..addMultiOption(
     'test',
@@ -125,24 +196,6 @@ final ArgParser _argParser = ArgParser()
         );
       }
     },
-  )
-  ..addFlag(
-    'silent',
-    negatable: true,
-    defaultsTo: false,
-  )
-  ..addOption(
-    'local-engine',
-    help: 'Name of a build output within the engine out directory, if you are '
-          'building Flutter locally. Use this to select a specific version of '
-          'the engine if you have built multiple engine targets. This path is '
-          'relative to --local-engine-src-path/out.',
-  )
-  ..addOption(
-    'local-engine-src-path',
-    help: 'Path to your engine src directory, if you are building Flutter '
-          'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at '
-          'the location based on the value of the --flutter-root option.',
   );
 
 bool _listsEqual(List<dynamic> a, List<dynamic> b) {

dev/devicelab/lib/framework/manifest.dart

@@ -3,10 +3,14 @@
 // found in the LICENSE file.
 
 import 'package:meta/meta.dart';
+import 'package:platform/platform.dart';
 import 'package:yaml/yaml.dart';
 
 import 'utils.dart';
 
+Platform get platform => _platform ??= const LocalPlatform();
+Platform _platform;
+
 /// Loads manifest data from `manifest.yaml` file or from [yaml], if present.
 Manifest loadTaskManifest([ String yaml ]) {
   final dynamic manifestYaml = yaml == null
@@ -52,7 +56,7 @@ class ManifestTask {
   final String stage;
 
   /// Capabilities required of the build agent to be able to perform this task.
-  final List<dynamic> requiredAgentCapabilities;
+  final List<String> requiredAgentCapabilities;
 
   /// Whether this test is flaky.
   ///
@@ -61,6 +65,20 @@ class ManifestTask {
 
   /// An optional custom timeout specified in minutes.
   final int timeoutInMinutes;
+
+  /// Whether the task is supported by the current host platform
+  bool isSupportedByHost() {
+    final Set<String> supportedHosts = Set<String>.from(
+      requiredAgentCapabilities.map<String>(
+        (String str) => str.split('/')[0]
+      )
+    );
+    String hostPlatform = platform.operatingSystem;
+    if (hostPlatform == 'macos') {
+      hostPlatform = 'mac'; // package:platform uses 'macos' while manifest.yaml uses 'mac'
+    }
+    return supportedHosts.contains(hostPlatform);
+  }
 }
 
 /// Thrown when the manifest YAML is not valid.

dev/devicelab/lib/tasks/sample_catalog_generator.dart

@@ -39,7 +39,7 @@ Future<TaskResult> samplePageCatalogGenerator(String authorizationToken) async {
     await saveCatalogScreenshots(
       directory: dir('${flutterDirectory.path}/examples/catalog/.generated'),
       commit: commit,
-      token: authorizationToken,
+      token: authorizationToken, // TODO(fujino): workaround auth token for local runs
       prefix: isIosDevice ? 'ios_' : '',
     );
   });