Run flutter tests in preview-dart-2 mode on travis flutter builds. (#14728)

* Run flutter tests in preview-dart-2 mode on travis flutter builds.

* Run dart2 tests on osx. Run dart tests in dart2.

* Fix name camelCase

* Default options to empty array, rather than null

* Troubleshoot failures

* More logging

* Troubleshoot: run single test

* Troubleshoot: run 15 tests

* Troubleshoot: run 15 tests with fix

* Try limit concurrency to 1

* Limit concurrency for preview-dart-2 tests to 4

* Move dart2 tests to allow_failures section

* Reinstate tests_dart_2 shard

* Raise concurrency to 8

* Reuse compiler across multiple test runs

* Allow to switch entry points when recompiling.

Actually use single compiler to incrementally recompile all tests executed by 'flutter test'.

* Remove leftover commented code

* Fix comment

* Lints

.travis.yml

@@ -6,6 +6,7 @@ osx_image: xcode9.2
 env:
   - SHARD=analyze
   - SHARD=tests
+  - SHARD=tests_dart2
   - SHARD=docs
   - SHARD=build_and_deploy_gallery
 matrix:
@@ -14,6 +15,8 @@ matrix:
     env: SHARD=analyze
   - os: osx
     env: SHARD=docs
+  allow_failures:
+  - env: SHARD=tests_dart2
 sudo: false
 filter_secrets: false
 

dev/bots/test.dart

@@ -29,6 +29,7 @@ const Map<String, ShardRunner> _kShards = const <String, ShardRunner>{
   'docs': _generateDocs,
   'analyze': _analyzeRepo,
   'tests': _runTests,
+  'tests_dart2': _runTestsDart2,
   'coverage': _runCoverage,
 };
 
@@ -129,35 +130,50 @@ Future<Null> _analyzeRepo() async {
   print('${bold}DONE: Analysis successful.$reset');
 }
 
-Future<Null> _runTests() async {
+Future<Null> _runTestsDart2() async {
+  if (Platform.isWindows) {
+    // AppVeyor platform is overloaded, won't be able to handle additional
+    // load of dart2 testing.
+    return;
+  }
+  _runTests(options: <String>['--preview-dart-2']);
+}
+
+Future<Null> _runTests({List<String> options: const <String>[]}) async {
   // Verify that the tests actually return failure on failure and success on success.
   final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
   await _runFlutterTest(automatedTests,
     script: path.join('test_smoke_test', 'fail_test.dart'),
+    options: options,
     expectFailure: true,
     printOutput: false,
   );
   await _runFlutterTest(automatedTests,
     script: path.join('test_smoke_test', 'pass_test.dart'),
+    options: options,
     printOutput: false,
   );
   await _runFlutterTest(automatedTests,
     script: path.join('test_smoke_test', 'crash1_test.dart'),
+    options: options,
     expectFailure: true,
     printOutput: false,
   );
   await _runFlutterTest(automatedTests,
     script: path.join('test_smoke_test', 'crash2_test.dart'),
+    options: options,
     expectFailure: true,
     printOutput: false,
   );
   await _runFlutterTest(automatedTests,
     script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'),
+    options: options,
     expectFailure: true,
     printOutput: false,
   );
   await _runFlutterTest(automatedTests,
     script: path.join('test_smoke_test', 'missing_import_test.broken_dart'),
+    options: options,
     expectFailure: true,
     printOutput: false,
   );
@@ -171,21 +187,21 @@ Future<Null> _runTests() async {
   await _verifyVersion(path.join(flutterRoot, 'version'));
 
   // Run tests.
-  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'));
-  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'));
-  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'));
-  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'));
+  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'), options: options);
+  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), options: options);
+  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), options: options);
+  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), options: options);
   await _pubRunTest(path.join(flutterRoot, 'packages', 'flutter_tools'));
   await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'));
 
-  await _runAllDartTests(path.join(flutterRoot, 'dev', 'devicelab'));
-  await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
-  await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'));
-  await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'));
-  await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'));
-  await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'));
-  await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'));
-  await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'));
+  await _runAllDartTests(path.join(flutterRoot, 'dev', 'devicelab'), options: options);
+  await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'), options: options);
+  await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'), options: options);
+  await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'), options: options);
+  await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'), options: options);
+  await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'), options: options);
+  await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: options);
+  await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'), options: options);
 
   print('${bold}DONE: All tests successful.$reset');
 }
@@ -356,8 +372,13 @@ Future<Null> _runFlutterTest(String workingDirectory, {
 
 Future<Null> _runAllDartTests(String workingDirectory, {
   Map<String, String> environment,
+  List<String> options,
 }) {
-  final List<String> args = <String>['--checked', path.join('test', 'all.dart')];
+  final List<String> args = <String>['--checked'];
+  if (options != null) {
+    args.addAll(options);
+  }
+  args.add(path.join('test', 'all.dart'));
   return _runCommand(dart, args,
     workingDirectory: workingDirectory,
     environment: environment,

packages/flutter_tools/lib/src/compile.dart

@@ -125,23 +125,27 @@ Future<String> compile(
 /// The wrapper is intended to stay resident in memory as user changes, reloads,
 /// restarts the Flutter app.
 class ResidentCompiler {
-  ResidentCompiler(this._sdkRoot, {bool trackWidgetCreation: false})
+  ResidentCompiler(this._sdkRoot, {bool trackWidgetCreation: false,
+      String packagesPath})
     : assert(_sdkRoot != null),
-      _trackWidgetCreation = trackWidgetCreation {
+      _trackWidgetCreation = trackWidgetCreation,
+      _packagesPath = packagesPath {
     // This is a URI, not a file path, so the forward slash is correct even on Windows.
     if (!_sdkRoot.endsWith('/'))
       _sdkRoot = '$_sdkRoot/';
   }
 
   final bool _trackWidgetCreation;
+  final String _packagesPath;
   String _sdkRoot;
   Process _server;
   final _StdoutHandler stdoutHandler = new _StdoutHandler();
 
   /// If invoked for the first time, it compiles Dart script identified by
   /// [mainPath], [invalidatedFiles] list is ignored.
-  /// Otherwise, [mainPath] is ignored, but [invalidatedFiles] is recompiled
-  /// into new binary.
+  /// On successive runs [invalidatedFiles] indicates which files need to be
+  /// recompiled. If [mainPath] is [null], previously used [mainPath] entry
+  /// point that is used for recompilation.
   /// Binary file name is returned if compilation was successful, otherwise
   /// null is returned.
   Future<String> recompile(String mainPath, List<String> invalidatedFiles,
@@ -154,7 +158,7 @@ class ResidentCompiler {
       return _compile(mainPath, outputPath);
 
     final String inputKey = new Uuid().generateV4();
-    _server.stdin.writeln('recompile $inputKey');
+    _server.stdin.writeln('recompile ${mainPath != null ? mainPath + " ": ""}$inputKey');
     invalidatedFiles.forEach(_server.stdin.writeln);
     _server.stdin.writeln(inputKey);
 
@@ -179,6 +183,9 @@ class ResidentCompiler {
     if (_trackWidgetCreation) {
       args.add('--track-widget-creation');
     }
+    if (_packagesPath != null) {
+      args.addAll(<String>['--packages', _packagesPath]);
+    }
     _server = await processManager.start(args);
     _server.stdout
       .transform(UTF8.decoder)

packages/flutter_tools/lib/src/test/flutter_platform.dart

@@ -83,6 +83,13 @@ enum _InitialResult { crashed, timedOut, connected }
 enum _TestResult { crashed, harnessBailed, testBailed }
 typedef Future<Null> _Finalizer();
 
+class CompilationRequest {
+  String path;
+  Completer<String> result;
+
+  CompilationRequest(this.path, this.result);
+}
+
 class _FlutterPlatform extends PlatformPlugin {
   _FlutterPlatform({
     @required this.shellPath,
@@ -93,7 +100,30 @@ class _FlutterPlatform extends PlatformPlugin {
     this.explicitObservatoryPort,
     this.host,
     this.previewDart2,
-  }) : assert(shellPath != null);
+  }) : assert(shellPath != null) {
+    compilerController.stream.listen((CompilationRequest request) async {
+      final bool isEmpty = compilationQueue.isEmpty;
+      compilationQueue.add(request);
+      // Only trigger processing if queue was empty - i.e. no other requests
+      // are currently being processed. This effectively enforces "one
+      // compilation request at a time".
+      if (isEmpty) {
+        while (compilationQueue.isNotEmpty) {
+          final CompilationRequest request = compilationQueue.first;
+          printTrace('Compiling ${request.path}');
+          final String outputPath = await compiler.recompile(request.path,
+            <String>[request.path]
+          );
+          print('Finished compilation of ${request.path} into $outputPath');
+          compiler.accept();
+          compiler.reset();
+          request.result.complete(outputPath);
+          // Only remove now when we finished processing the element
+          compilationQueue.removeAt(0);
+        }
+      }
+    });
+  }
 
   final String shellPath;
   final TestWatcher watcher;
@@ -103,6 +133,12 @@ class _FlutterPlatform extends PlatformPlugin {
   final int explicitObservatoryPort;
   final InternetAddress host;
   final bool previewDart2;
+  final StreamController<CompilationRequest> compilerController =
+      new StreamController<CompilationRequest>();
+  ResidentCompiler compiler =
+      new ResidentCompiler(artifacts.getArtifactPath(Artifact.flutterPatchedSdkPath),
+          packagesPath: PackageMap.globalPackagesPath);
+  final List<CompilationRequest> compilationQueue = <CompilationRequest>[];
 
   // Each time loadChannel() is called, we spin up a local WebSocket server,
   // then spin up the engine in a subprocess. We pass the engine a Dart file
@@ -203,12 +239,9 @@ class _FlutterPlatform extends PlatformPlugin {
       String bundlePath;
 
       if (previewDart2) {
-        mainDart = await compile(
-          sdkRoot: artifacts.getArtifactPath(Artifact.flutterPatchedSdkPath),
-          mainPath: listenerFile.path,
-          packagesPath: PackageMap.globalPackagesPath,
-          linkPlatformKernelIn: true
-        );
+        final Completer<String> completer = new Completer<String>();
+        compilerController.add(new CompilationRequest(listenerFile.path, completer));
+        mainDart = await completer.future;
 
         if (mainDart == null) {
           controller.sink.addError(_getErrorMessage('Compilation failed', testPath, shellPath));