Pipe through test-randomize-ordering-seed (#47243)

packages/flutter_tools/bin/fuchsia_tester.dart

@@ -22,6 +22,7 @@ import 'package:flutter_tools/src/project.dart';
 import 'package:flutter_tools/src/reporting/reporting.dart';
 import 'package:flutter_tools/src/test/coverage_collector.dart';
 import 'package:flutter_tools/src/test/runner.dart';
+import 'package:flutter_tools/src/test/test_wrapper.dart';
 
 // This was largely inspired by lib/src/commands/test.dart.
 
@@ -142,6 +143,7 @@ Future<void> run(List<String> args) async {
     }
 
     exitCode = await runTests(
+      const TestWrapper(),
       tests.keys.toList(),
       workDir: testDirectory,
       watcher: collector,

packages/flutter_tools/lib/src/commands/test.dart

@@ -21,10 +21,14 @@ import '../runner/flutter_command.dart';
 import '../test/coverage_collector.dart';
 import '../test/event_printer.dart';
 import '../test/runner.dart';
+import '../test/test_wrapper.dart';
 import '../test/watcher.dart';
 
 class TestCommand extends FastFlutterCommand {
-  TestCommand({ bool verboseHelp = false }) {
+  TestCommand({
+    bool verboseHelp = false,
+    this.testWrapper = const TestWrapper(),
+  }) : assert(testWrapper != null) {
     requiresPubspecYaml();
     usesPubOption();
     argParser
@@ -100,10 +104,20 @@ class TestCommand extends FastFlutterCommand {
         allowed: const <String>['tester', 'chrome'],
         defaultsTo: 'tester',
         help: 'The platform to run the unit tests on. Defaults to "tester".',
+      )
+      ..addOption('test-randomize-ordering-seed',
+        defaultsTo: '0',
+        help: 'If positive, use this as a seed to randomize the execution of '
+              'test cases (must be a 32bit unsigned integer).\n'
+              'If "random", pick a random seed to use.\n'
+              'If 0 or not set, do not randomize test case execution order.',
       );
     usesTrackWidgetCreation(verboseHelp: verboseHelp);
   }
 
+  /// The interface for starting and configuring the tester.
+  final TestWrapper testWrapper;
+
   @override
   Future<Set<DevelopmentArtifact>> get requiredArtifacts async {
     final Set<DevelopmentArtifact> results = <DevelopmentArtifact>{};
@@ -223,6 +237,7 @@ class TestCommand extends FastFlutterCommand {
       boolArg('disable-service-auth-codes');
 
     final int result = await runTests(
+      testWrapper,
       files,
       workDir: workDir,
       names: names,
@@ -240,6 +255,7 @@ class TestCommand extends FastFlutterCommand {
       buildTestAssets: buildTestAssets,
       flutterProject: flutterProject,
       web: stringArg('platform') == 'chrome',
+      randomSeed: stringArg('test-randomize-ordering-seed'),
     );
 
     if (collector != null) {

packages/flutter_tools/lib/src/test/flutter_platform.dart

@@ -7,10 +7,7 @@ import 'dart:async';
 import 'package:meta/meta.dart';
 import 'package:stream_channel/stream_channel.dart';
 
-import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/platform.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/hack_register_platform.dart' as hack; // ignore: implementation_imports
 import 'package:test_core/src/runner/runner_suite.dart'; // ignore: implementation_imports
 import 'package:test_core/src/runner/suite.dart'; // ignore: implementation_imports
 import 'package:test_core/src/runner/plugin/platform_helpers.dart'; // ignore: implementation_imports
@@ -27,6 +24,7 @@ import '../convert.dart';
 import '../dart/package_map.dart';
 import '../globals.dart';
 import '../project.dart';
+import '../test/test_wrapper.dart';
 import '../vmservice.dart';
 import 'test_compiler.dart';
 import 'test_config.dart';
@@ -71,6 +69,7 @@ typedef PlatformPluginRegistration = void Function(FlutterPlatform platform);
 /// (that is, one Dart file with a `*_test.dart` file name and a single `void
 /// main()`), you can set an observatory port explicitly.
 FlutterPlatform installHook({
+  TestWrapper testWrapper = const TestWrapper(),
   @required String shellPath,
   TestWatcher watcher,
   bool enableObservatory = false,
@@ -91,11 +90,12 @@ FlutterPlatform installHook({
   String icudtlPath,
   PlatformPluginRegistration platformPluginRegistration,
 }) {
+  assert(testWrapper != null);
   assert(enableObservatory || (!startPaused && observatoryPort == null));
 
   // registerPlatformPlugin can be injected for testing since it's not very mock-friendly.
   platformPluginRegistration ??= (FlutterPlatform platform) {
-    hack.registerPlatformPlugin(
+    testWrapper.registerPlatformPlugin(
       <Runtime>[Runtime.vm],
       () {
         return platform;

packages/flutter_tools/lib/src/test/flutter_web_platform.dart

@@ -17,7 +17,6 @@ import 'package:shelf_packages_handler/shelf_packages_handler.dart';
 import 'package:shelf_static/shelf_static.dart';
 import 'package:shelf_web_socket/shelf_web_socket.dart';
 import 'package:stream_channel/stream_channel.dart';
-import 'package:test_api/backend.dart'; // ignore: deprecated_member_use
 import 'package:test_api/src/backend/runtime.dart';
 import 'package:test_api/src/backend/suite_platform.dart';
 import 'package:test_api/src/util/stack_trace_mapper.dart';

packages/flutter_tools/lib/src/test/runner.dart

@@ -5,9 +5,6 @@
 import 'dart:async';
 
 import 'package:meta/meta.dart';
-import 'package:test_api/backend.dart'; // ignore: deprecated_member_use
-import 'package:test_core/src/executable.dart' as test; // ignore: implementation_imports
-import 'package:test_core/src/runner/hack_register_platform.dart' as hack; // ignore: implementation_imports
 
 import '../artifacts.dart';
 import '../base/common.dart';
@@ -22,10 +19,12 @@ import '../project.dart';
 import '../web/compile.dart';
 import 'flutter_platform.dart' as loader;
 import 'flutter_web_platform.dart';
+import 'test_wrapper.dart';
 import 'watcher.dart';
 
 /// Runs tests using package:test and the Flutter engine.
 Future<int> runTests(
+  TestWrapper testWrapper,
   List<String> testFiles, {
   Directory workDir,
   List<String> names = const <String>[],
@@ -47,6 +46,7 @@ Future<int> runTests(
   String icudtlPath,
   Directory coverageDirectory,
   bool web = false,
+  String randomSeed = '0',
 }) async {
   // Configure package:test to use the Flutter engine for child processes.
   final String shellPath = artifacts.getArtifactPath(Artifact.flutterTester);
@@ -67,6 +67,7 @@ Future<int> runTests(
       ...<String>['--name', name],
     for (String plainName in plainNames)
       ...<String>['--plain-name', plainName],
+    '--test-randomize-ordering-seed=$randomSeed',
   ];
   if (web) {
     final String tempBuildDir = fs.systemTempDirectory
@@ -89,7 +90,7 @@ Future<int> runTests(
       ..add('--precompiled=$tempBuildDir')
       ..add('--')
       ..addAll(testFiles);
-    hack.registerPlatformPlugin(
+    testWrapper.registerPlatformPlugin(
       <Runtime>[Runtime.chrome],
       () {
         return FlutterWebPlatform.start(
@@ -100,7 +101,7 @@ Future<int> runTests(
         );
       },
     );
-    await test.main(testArgs);
+    await testWrapper.main(testArgs);
     return exitCode;
   }
 
@@ -112,6 +113,7 @@ Future<int> runTests(
       ipv6 ? InternetAddressType.IPv6 : InternetAddressType.IPv4;
 
   final loader.FlutterPlatform platform = loader.installHook(
+    testWrapper: testWrapper,
     shellPath: shellPath,
     watcher: watcher,
     enableObservatory: enableObservatory,
@@ -144,7 +146,7 @@ Future<int> runTests(
     }
 
     printTrace('running test package with arguments: $testArgs');
-    await test.main(testArgs);
+    await testWrapper.main(testArgs);
 
     // test.main() sets dart:io's exitCode global.
     printTrace('test package returned with exit code $exitCode');

packages/flutter_tools/lib/src/test/test_wrapper.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:test_api/backend.dart'; // ignore: deprecated_member_use
+import 'package:test_core/src/runner/platform.dart'; // ignore: implementation_imports
+import 'package:test_core/src/executable.dart' as test; // ignore: implementation_imports
+import 'package:test_core/src/runner/hack_register_platform.dart' as hack; // ignore: implementation_imports
+
+export 'package:test_api/backend.dart' show Runtime; // ignore: deprecated_member_use
+export 'package:test_core/src/runner/platform.dart' show PlatformPlugin; // ignore: implementation_imports
+
+abstract class TestWrapper {
+  const factory TestWrapper() = _DefaultTestWrapper;
+
+  Future<void> main(List<String> args);
+  void registerPlatformPlugin(Iterable<Runtime> runtimes, FutureOr<PlatformPlugin> Function() platforms);
+}
+
+class _DefaultTestWrapper implements TestWrapper {
+  const _DefaultTestWrapper();
+
+  @override
+  Future<void> main(List<String> args) async {
+    await test.main(args);
+  }
+
+  @override
+  void registerPlatformPlugin(Iterable<Runtime> runtimes, FutureOr<PlatformPlugin> Function() platforms) {
+    hack.registerPlatformPlugin(runtimes, platforms);
+  }
+}

packages/flutter_tools/test/commands.shard/hermetic/test_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:args/command_runner.dart';
+import 'package:file/memory.dart';
+import 'package:flutter_tools/src/base/file_system.dart';
+import 'package:flutter_tools/src/cache.dart';
+import 'package:flutter_tools/src/commands/test.dart';
+import 'package:flutter_tools/src/test/test_wrapper.dart';
+import 'package:process/process.dart';
+
+import '../../src/common.dart';
+import '../../src/context.dart';
+import '../../src/testbed.dart';
+
+void main() {
+  Cache.disableLocking();
+  MemoryFileSystem fs;
+
+  setUp(() {
+    fs = MemoryFileSystem();
+    fs.file('pubspec.yaml').createSync();
+    fs.directory('test').childFile('some_test.dart').createSync(recursive: true);
+  });
+
+  testUsingContext('Pipes test-randomize-ordering-seed to package:test',
+      () async {
+    final FakePackageTest fakePackageTest = FakePackageTest();
+
+    final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest);
+    final CommandRunner<void> commandRunner =
+        createTestCommandRunner(testCommand);
+
+    await commandRunner.run(const <String>[
+      'test',
+      '--test-randomize-ordering-seed=random',
+      '--no-pub',
+    ]);
+    expect(
+      fakePackageTest.lastArgs,
+      contains('--test-randomize-ordering-seed=random'),
+    );
+  }, overrides: <Type, Generator>{
+    FileSystem: () => fs,
+    ProcessManager: () => FakeProcessManager.any(),
+    Cache: () => FakeCache(),
+  });
+}
+
+class FakePackageTest implements TestWrapper {
+  List<String> lastArgs;
+
+  @override
+  Future<void> main(List<String> args) async {
+    lastArgs = args;
+  }
+
+  @override
+  void registerPlatformPlugin(
+    Iterable<Runtime> runtimes,
+    FutureOr<PlatformPlugin> Function() platforms,
+  ) {}
+}

packages/flutter_tools/test/general.shard/forbidden_imports_test.dart

@@ -66,7 +66,7 @@ void main() {
       fs.path.join(flutterTools, 'lib', 'src', 'build_runner', 'build_script.dart'),
       fs.path.join(flutterTools, 'lib', 'src', 'test', 'flutter_platform.dart'),
       fs.path.join(flutterTools, 'lib', 'src', 'test', 'flutter_web_platform.dart'),
-      fs.path.join(flutterTools, 'lib', 'src', 'test', 'runner.dart'),
+      fs.path.join(flutterTools, 'lib', 'src', 'test', 'test_wrapper.dart'),
     ];
     bool _isNotWhitelisted(FileSystemEntity entity) => whitelistedPaths.every((String path) => path != entity.path);
 

packages/flutter_tools/tool/tool_coverage.dart

@@ -11,15 +11,12 @@ import 'package:async/async.dart';
 import 'package:coverage/coverage.dart';
 import 'package:flutter_tools/src/base/common.dart';
 import 'package:flutter_tools/src/context_runner.dart';
+import 'package:flutter_tools/src/test/test_wrapper.dart';
 import 'package:path/path.dart' as path;
 import 'package:stream_channel/isolate_channel.dart';
 import 'package:stream_channel/stream_channel.dart';
-import 'package:test_core/src/runner/hack_register_platform.dart' as hack; // ignore: implementation_imports
-import 'package:test_core/src/executable.dart' as test; // ignore: implementation_imports
 import 'package:vm_service_client/vm_service_client.dart'; // ignore: deprecated_member_use
-import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/platform.dart'; // ignore: implementation_imports
 import 'package:test_core/src/runner/runner_suite.dart'; // ignore: implementation_imports
 import 'package:test_core/src/runner/suite.dart'; // ignore: implementation_imports
 import 'package:test_core/src/runner/plugin/platform_helpers.dart'; // ignore: implementation_imports
@@ -35,7 +32,8 @@ import 'package:flutter_tools/src/test/coverage_collector.dart';
 Future<void> main(List<String> arguments) async {
   return runInContext(() async {
     final VMPlatform vmPlatform = VMPlatform();
-    hack.registerPlatformPlugin(
+    const TestWrapper test = TestWrapper();
+    test.registerPlatformPlugin(
       <Runtime>[Runtime.vm],
       () => vmPlatform,
     );