// Copyright 2014 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:async'; import 'dart:developer' as developer; import 'dart:ui'; import 'package:flutter/foundation.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter/services.dart'; import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; // ignore: implementation_imports import 'package:test_core/src/direct_run.dart'; // ignore: implementation_imports import 'package:test_core/src/runner/engine.dart'; import 'package:vm_service/vm_service.dart' as vm; import 'package:vm_service/vm_service_io.dart' as vm_io; import '_callback_io.dart' if (dart.library.html) '_callback_web.dart' as driver_actions; import '_extension_io.dart' if (dart.library.html) '_extension_web.dart'; import 'common.dart'; import 'src/constants.dart'; import 'src/reporter.dart'; /// Toggles the legacy reporting mechansim where results are only collected /// for [testWidgets]. /// /// If [run] is called, this will be disabled. bool _isUsingLegacyReporting = true; /// Executes a block that contains tests. /// /// Example Usage: /// ``` /// import 'package:flutter_test/flutter_test.dart'; /// import 'package:integration_test/integration_test.dart'; /// /// void main() => run(_testMain); /// /// void _testMain() { /// test('A test', () { /// expect(true, true); /// }); /// } /// ``` /// /// If not explicitly passed, the default [reporter] will send results over the /// platform channel to native. Future<void> run( FutureOr<void> Function() testMain, { Reporter reporter = const _ReporterImpl(), }) async { _isUsingLegacyReporting = false; final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized() as IntegrationTestWidgetsFlutterBinding; // Pipe detailed exceptions within [testWidgets] to `package:test`. reportTestException = (FlutterErrorDetails details, String testDescription) { registerException('Test $testDescription failed: $details'); }; final Completer<List<TestResult>> resultsCompleter = Completer<List<TestResult>>(); await directRunTests( testMain, reporterFactory: (Engine engine) => ResultReporter(engine, resultsCompleter), ); final List<TestResult> results = await resultsCompleter.future; binding._updateTestResultState(<String, TestResult>{ for (final TestResult result in results) result.methodName: result, }); await reporter.report(results); } /// Abstract interface for a result reporter. abstract class Reporter { /// Reports test results. /// /// This method will be called at the end of [run] with the [results] of /// running the test suite. Future<void> report(List<TestResult> results); } /// Default implementation of the reporter that sends results over to the /// platform side. class _ReporterImpl implements Reporter { const _ReporterImpl(); @override Future<void> report( List<TestResult> results, ) async { try { await IntegrationTestWidgetsFlutterBinding._channel.invokeMethod<void>( 'allTestsFinished', <String, dynamic>{ 'results': <String, String>{ for (final TestResult result in results) result.methodName: result is Failure ? _formatFailureForPlatform(result) : success } }, ); } on MissingPluginException { print('Warning: integration_test test plugin was not detected.'); } } } String _formatFailureForPlatform(Failure failure) => '${failure.error} ${failure.details}'; /// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results /// on a channel to adapt them to native instrumentation test format. class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding implements IntegrationTestResults { /// If [run] is not used, sets up a listener to report that the tests are /// finished when everything is torn down. /// /// This functionality is deprecated – clients are expected to use [run] to /// execute their tests instead. IntegrationTestWidgetsFlutterBinding() { if (!_isUsingLegacyReporting) { // TODO(jiahaog): Point users to use the CLI https://github.com/flutter/flutter/issues/66264. print('Using the legacy test result reporter, which will not catch all ' 'errors thrown in declared tests. Consider wrapping tests with ' 'https://api.flutter.dev/flutter/integration_test/run.html instead.'); return; } tearDownAll(() async { _updateTestResultState(results); await const _ReporterImpl().report(results.values.toList()); }); final TestExceptionReporter oldTestExceptionReporter = reportTestException; reportTestException = (FlutterErrorDetails details, String testDescription) { results[testDescription] = Failure( testDescription, details.toString(), error: details.exception, ); oldTestExceptionReporter(details, testDescription); }; } void _updateTestResultState(Map<String, TestResult> results) { this.results = results; print('Test execution completed: $results'); _allTestsPassed.complete(!results.values.any((TestResult result) => result is Failure)); callbackManager.cleanup(); } @override bool get overrideHttpClient => false; @override bool get registerTestTextInput => false; Size _surfaceSize; // This flag is used to print warning messages when tracking performance // under debug mode. static bool _firstRun = false; /// Artificially changes the surface size to `size` on the Widget binding, /// then flushes microtasks. /// /// Set to null to use the default surface size. @override Future<void> setSurfaceSize(Size size) { return TestAsyncUtils.guard<void>(() async { assert(inTest); if (_surfaceSize == size) { return; } _surfaceSize = size; handleMetricsChanged(); }); } @override ViewConfiguration createViewConfiguration() { final double devicePixelRatio = window.devicePixelRatio; final Size size = _surfaceSize ?? window.physicalSize / devicePixelRatio; return TestViewConfiguration( size: size, window: window, ); } @override Completer<bool> get allTestsPassed => _allTestsPassed; final Completer<bool> _allTestsPassed = Completer<bool>(); @override List<Failure> get failureMethodsDetails => _failures; /// Similar to [WidgetsFlutterBinding.ensureInitialized]. /// /// Returns an instance of the [IntegrationTestWidgetsFlutterBinding], creating and /// initializing it if necessary. static WidgetsBinding ensureInitialized() { if (WidgetsBinding.instance == null) { IntegrationTestWidgetsFlutterBinding(); } assert(WidgetsBinding.instance is IntegrationTestWidgetsFlutterBinding); return WidgetsBinding.instance; } static const MethodChannel _channel = MethodChannel('plugins.flutter.io/integration_test'); /// Test results that will be populated after the tests have completed. @visibleForTesting Map<String, TestResult> results = <String, TestResult>{}; List<Failure> get _failures => results.values.whereType<Failure>().toList(); /// The extra data for the reported result. /// /// The values in `reportData` must be json-serializable objects or `null`. /// If it's `null`, no extra data is attached to the result. /// /// The default value is `null`. @override Map<String, dynamic> reportData; /// Manages callbacks received from driver side and commands send to driver /// side. final CallbackManager callbackManager = driver_actions.callbackManager; /// Taking a screenshot. /// /// Called by test methods. Implementation differs for each platform. Future<void> takeScreenshot(String screenshotName) async { await callbackManager.takeScreenshot(screenshotName); } /// The callback function to response the driver side input. @visibleForTesting Future<Map<String, dynamic>> callback(Map<String, String> params) async { return await callbackManager.callback( params, this /* as IntegrationTestResults */); } // Emulates the Flutter driver extension, returning 'pass' or 'fail'. @override void initServiceExtensions() { super.initServiceExtensions(); if (kIsWeb) { registerWebServiceExtension(callback); } registerServiceExtension(name: 'driver', callback: callback); } @override Future<void> runTest( Future<void> testBody(), VoidCallback invariantTester, { String description = '', Duration timeout, }) async { await super.runTest( testBody, invariantTester, description: description, timeout: timeout, ); results[description] ??= Success(description); } vm.VmService _vmService; /// Initialize the [vm.VmService] settings for the timeline. @visibleForTesting Future<void> enableTimeline({ List<String> streams = const <String>['all'], @visibleForTesting vm.VmService vmService, }) async { assert(streams != null); assert(streams.isNotEmpty); if (vmService != null) { _vmService = vmService; } if (_vmService == null) { final developer.ServiceProtocolInfo info = await developer.Service.getInfo(); assert(info.serverUri != null); _vmService = await vm_io.vmServiceConnectUri( 'ws://localhost:${info.serverUri.port}${info.serverUri.path}ws', ); } await _vmService.setVMTimelineFlags(streams); } /// Runs [action] and returns a [vm.Timeline] trace for it. /// /// Waits for the `Future` returned by [action] to complete prior to stopping /// the trace. /// /// The `streams` parameter limits the recorded timeline event streams to only /// the ones listed. By default, all streams are recorded. /// See `timeline_streams` in /// [Dart-SDK/runtime/vm/timeline.cc](https://github.com/dart-lang/sdk/blob/master/runtime/vm/timeline.cc) /// /// If [retainPriorEvents] is true, retains events recorded prior to calling /// [action]. Otherwise, prior events are cleared before calling [action]. By /// default, prior events are cleared. Future<vm.Timeline> traceTimeline( Future<dynamic> action(), { List<String> streams = const <String>['all'], bool retainPriorEvents = false, }) async { await enableTimeline(streams: streams); if (retainPriorEvents) { await action(); return await _vmService.getVMTimeline(); } await _vmService.clearVMTimeline(); final vm.Timestamp startTime = await _vmService.getVMTimelineMicros(); await action(); final vm.Timestamp endTime = await _vmService.getVMTimelineMicros(); return await _vmService.getVMTimeline( timeOriginMicros: startTime.timestamp, timeExtentMicros: endTime.timestamp, ); } /// This is a convenience wrap of [traceTimeline] and send the result back to /// the host for the [flutter_driver] style tests. /// /// This records the timeline during `action` and adds the result to /// [reportData] with `reportKey`. The [reportData] contains extra information /// from the test other than test success/fail. It will be passed back to the /// host and be processed by the [ResponseDataCallback] defined in /// [integration_test_driver.integrationDriver]. By default it will be written /// to `build/integration_response_data.json` with the key `timeline`. /// /// For tests with multiple calls of this method, `reportKey` needs to be a /// unique key, otherwise the later result will override earlier one. /// /// The `streams` and `retainPriorEvents` parameters are passed as-is to /// [traceTimeline]. Future<void> traceAction( Future<dynamic> action(), { List<String> streams = const <String>['all'], bool retainPriorEvents = false, String reportKey = 'timeline', }) async { final vm.Timeline timeline = await traceTimeline( action, streams: streams, retainPriorEvents: retainPriorEvents, ); reportData ??= <String, dynamic>{}; reportData[reportKey] = timeline.toJson(); } /// Watches the [FrameTiming] during `action` and report it to the binding /// with key `reportKey`. /// /// This can be used to implement performance tests previously using /// [traceAction] and [TimelineSummary] from [flutter_driver] Future<void> watchPerformance( Future<void> action(), { String reportKey = 'performance', }) async { assert(() { if (_firstRun) { debugPrint(kDebugWarning); _firstRun = false; } return true; }()); // The engine could batch FrameTimings and send them only once per second. // Delay for a sufficient time so either old FrameTimings are flushed and not // interfering our measurements here, or new FrameTimings are all reported. // TODO(CareF): remove this when flush FrameTiming is readly in engine. // See https://github.com/flutter/flutter/issues/64808 // and https://github.com/flutter/flutter/issues/67593 Future<void> delayForFrameTimings() => Future<void>.delayed(const Duration(seconds: 2)); await delayForFrameTimings(); // flush old FrameTimings final List<FrameTiming> frameTimings = <FrameTiming>[]; final TimingsCallback watcher = frameTimings.addAll; addTimingsCallback(watcher); await action(); await delayForFrameTimings(); // make sure all FrameTimings are reported removeTimingsCallback(watcher); final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings); reportData ??= <String, dynamic>{}; reportData[reportKey] = frameTimes.summary; } @override Timeout get defaultTestTimeout => _defaultTestTimeout ?? super.defaultTestTimeout; /// Configures the default timeout for [testWidgets]. /// /// See [TestWidgetsFlutterBinding.defaultTestTimeout] for more details. set defaultTestTimeout(Timeout timeout) => _defaultTestTimeout = timeout; Timeout _defaultTestTimeout; }