Add support for flutter_test_config.dart (#17141)

This enables support for a `flutter_test_config.dart` configuration file,
which will be discovered and handed the responsibility of running the
test file (thus allowing it to run pre-test setup on a project level).

https://github.com/flutter/flutter/issues/16859

packages/flutter_test/lib/flutter_test.dart

@@ -3,6 +3,45 @@
 // found in the LICENSE file.
 
 /// Testing library for flutter, built on top of `package:test`.
+///
+/// ## Test Configuration
+///
+/// The testing library exposes a few constructs by which projects may configure
+/// their tests.
+///
+/// ### Per test or per file
+///
+/// Due to its use of `package:test` as a foundation, the testing library
+/// allows for tests to be initialized using the existing constructs found in
+/// `package:test`. These include the [setUp] and [setUpAll] methods.
+///
+/// ### Per directory hierarchy
+///
+/// In addition to the constructs provided by `package:test`, this library
+/// supports the configuration of tests at the directory level.
+///
+/// Before a test file is executed, the Flutter test framework will scan up the
+/// directory hierarchy, starting from the directory in which the test file
+/// resides, looking for a file named `flutter_test_config.dart`. If it finds
+/// such a configuration file, the file will be assumed to have a `main` method
+/// with the following signature:
+///
+/// ```dart
+/// void main(FutureOr<void> testMain());
+/// ```
+///
+/// The test framework will execute that method and pass it the `main()` method
+/// of the test. It is then the responsibility of the configuration file's
+/// `main()` method to invoke the test's `main()` method.
+///
+/// After the test framework finds a configuration file, it will stop scanning
+/// the directory hierarchy. In other words, the test configuration file that
+/// lives closest to the test file will be selected, and all other test
+/// configuration files will be ignored. Likewise, it will stop scanning the
+/// directory hierarchy when it finds a `pubspec.yaml`, since that signals the
+/// root of the project.
+///
+/// If no configuration file is located, the test will be executed like normal.
 library flutter_test;
 
 export 'dart:async' show Future;

packages/flutter_test/lib/src/goldens.dart

@@ -56,10 +56,16 @@ abstract class GoldenFileComparator {
 /// encoded PNGs, returning true only if there's an exact match.
 ///
 /// Callers may choose to override the default comparator by setting this to a
-/// custom comparator during test set-up. For example, some projects may wish to
-/// install a more intelligent comparator that knows how to decode the PNG
-/// images to raw pixels and compare pixel vales, reporting specific differences
-/// between the images.
+/// custom comparator during test set-up (or using directory-level test
+/// configuration). For example, some projects may wish to install a more
+/// intelligent comparator that knows how to decode the PNG images to raw
+/// pixels and compare pixel vales, reporting specific differences between the
+/// images.
+///
+/// See also:
+///
+///  * [flutter_test] for more information about how to configure tests at the
+///    directory-level.
 GoldenFileComparator goldenFileComparator = const _UninitializedComparator();
 
 /// Whether golden files should be automatically updated during tests rather

packages/flutter_test/test/test_config/child_directory/config_test.dart

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import '../config_test_utils.dart';
+
+void main() {
+  testConfig('flutter_test_config initializes tests in child folder', '/test_config');
+}

packages/flutter_test/test/test_config/child_directory/grandchild_directory/config_test.dart

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import '../../config_test_utils.dart';
+
+void main() {
+  testConfig('flutter_test_config initializes tests in grandchild folder', '/test_config');
+}

packages/flutter_test/test/test_config/config_test.dart

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'config_test_utils.dart';
+
+void main() {
+  testConfig('flutter_test_config initializes tests in same folder', '/test_config');
+}

packages/flutter_test/test/test_config/config_test_utils.dart

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_test/flutter_test.dart';
+
+void testConfig(
+  String description,
+  String expectedStringValue, {
+  Map<Type, dynamic> otherExpectedValues: const <Type, dynamic>{int: isNull},
+}) {
+  final String actualStringValue = Zone.current[String];
+  final Map<Type, dynamic> otherActualValues = otherExpectedValues.map<Type, dynamic>(
+    (Type key, dynamic value) {
+      return new MapEntry<Type, dynamic>(key, Zone.current[key]);
+    },
+  );
+
+  test(description, () {
+    expect(actualStringValue, expectedStringValue);
+    for (Type key in otherExpectedValues.keys) {
+      expect(otherActualValues[key], otherExpectedValues[key]);
+    }
+  });
+}

packages/flutter_test/test/test_config/flutter_test_config.dart

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+void main(FutureOr<void> testMain()) async {
+  await runZoned<dynamic>(testMain, zoneValues: <Type, String>{
+    String: '/test_config',
+  });
+}

packages/flutter_test/test/test_config/nested_config/config_test.dart

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import '../config_test_utils.dart';
+
+void main() {
+  testConfig(
+    'cwd config takes precedence over parent config',
+    '/test_config/nested_config',
+    otherExpectedValues: <Type, dynamic>{int: 123},
+  );
+}

packages/flutter_test/test/test_config/nested_config/flutter_test_config.dart

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+void main(FutureOr<void> testMain()) async {
+  await runZoned<dynamic>(testMain, zoneValues: <Type, dynamic>{
+    String: '/test_config/nested_config',
+    int: 123,
+  });
+}

packages/flutter_test/test/test_config/project_root/config_test.dart

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import '../config_test_utils.dart';
+
+void main() {
+  testConfig('pubspec.yaml causes config scanning to stop', null);
+}

packages/flutter_test/test/test_config/project_root/pubspec.yaml

+# This file exists to simulate a project root. The testing library should stop
+# scanning for a `flutter_test_config.dart` file when it reaches this directory
+name: dummy
+
+# PUBSPEC CHECKSUM: 0

packages/flutter_tools/lib/src/test/flutter_platform.dart

@@ -41,6 +41,14 @@ const Duration _kTestProcessTimeout = const Duration(minutes: 5);
 /// hold that against the test.
 const String _kStartTimeoutTimerMessage = 'sky_shell test process has entered main method';
 
+/// The name of the test configuration file that will be discovered by the
+/// test harness if it exists in the project directory hierarchy.
+const String _kTestConfigFileName = 'flutter_test_config.dart';
+
+/// The name of the file that signals the root of the project and that will
+/// cause the test harness to stop scanning for configuration files.
+const String _kProjectRootSentinel = 'pubspec.yaml';
+
 /// The address at which our WebSocket server resides and at which the sky_shell
 /// processes will host the Observatory server.
 final Map<InternetAddressType, InternetAddress> _kHosts = <InternetAddressType, InternetAddress>{
@@ -623,7 +631,25 @@ class _FlutterPlatform extends PlatformPlugin {
     Uri testUrl,
     String encodedWebsocketUrl,
   }) {
-    return '''
+    assert(testUrl.scheme == 'file');
+    File testConfigFile;
+    Directory directory = fs.file(testUrl).parent;
+    while (directory.path != directory.parent.path) {
+      final File configFile = directory.childFile(_kTestConfigFileName);
+      if (configFile.existsSync()) {
+        printTrace('Discovered $_kTestConfigFileName in ${directory.path}');
+        testConfigFile = configFile;
+        break;
+      }
+      if (directory.childFile(_kProjectRootSentinel).existsSync()) {
+        printTrace('Stopping scan for $_kTestConfigFileName; '
+                   'found project root at ${directory.path}');
+        break;
+      }
+      directory = directory.parent;
+    }
+    final StringBuffer buffer = new StringBuffer();
+    buffer.write('''
 import 'dart:convert';
 import 'dart:io';  // ignore: dart_io_import
 
@@ -637,6 +663,15 @@ import 'package:stream_channel/stream_channel.dart';
 import 'package:test/src/runner/vm/catch_isolate_errors.dart';
 
 import '$testUrl' as test;
+'''
+    );
+    if (testConfigFile != null) {
+      buffer.write('''
+import '${new Uri.file(testConfigFile.path)}' as test_config;
+'''
+      );
+    }
+    buffer.write('''
 
 void main() {
   print('$_kStartTimeoutTimerMessage');
@@ -646,7 +681,20 @@ void main() {
     catchIsolateErrors();
     goldenFileComparator = new LocalFileComparator(Uri.parse('$testUrl'));
     autoUpdateGoldenFiles = $updateGoldens;
+'''
+    );
+    if (testConfigFile != null) {
+      buffer.write('''
+    return () {
+      test_config.main(test.main);
+    };
+''');
+    } else {
+      buffer.write('''
     return test.main;
+''');
+    }
+    buffer.write('''
   });
   WebSocket.connect(server).then((WebSocket socket) {
     socket.map((dynamic x) {
@@ -656,7 +704,9 @@ void main() {
     socket.addStream(channel.stream.map(json.encode));
   });
 }
-''';
+'''
+    );
+    return buffer.toString();
   }
 
   File _cachedFontConfig;