integration_test.dart 12.7 KB
Newer Older
1 2 3 4 5 6 7 8
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:developer' as developer;
import 'dart:ui';

9
import 'package:flutter/rendering.dart';
Dan Field's avatar
Dan Field committed
10 11
import 'package:flutter_test/flutter_test.dart';
import 'package:flutter/foundation.dart';
12 13 14 15 16
import 'package:flutter/services.dart';
import 'package:flutter/widgets.dart';
import 'package:vm_service/vm_service.dart' as vm;
import 'package:vm_service/vm_service_io.dart' as vm_io;

Dan Field's avatar
Dan Field committed
17
import '_callback_io.dart' if (dart.library.html) '_callback_web.dart' as driver_actions;
18 19
import '_extension_io.dart' if (dart.library.html) '_extension_web.dart';
import 'common.dart';
20

Dan Field's avatar
Dan Field committed
21
const String _success = 'success';
22

23 24 25 26 27 28 29 30 31 32 33 34
/// Whether results should be reported to the native side over the method
/// channel.
///
/// This is enabled by default for use by native test frameworks like Android
/// instrumentation or XCTest. When running with the Flutter Tool through
/// `flutter test integration_test` though, it will be disabled as the Flutter
/// tool will be responsible for collection of test results.
const bool _shouldReportResultsToNative = bool.fromEnvironment(
  'INTEGRATION_TEST_SHOULD_REPORT_RESULTS_TO_NATIVE',
  defaultValue: true,
);

35 36
/// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results
/// on a channel to adapt them to native instrumentation test format.
Dan Field's avatar
Dan Field committed
37 38 39
class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding implements IntegrationTestResults {
  /// Sets up a listener to report that the tests are finished when everything is
  /// torn down.
40 41
  IntegrationTestWidgetsFlutterBinding() {
    tearDownAll(() async {
42 43 44 45 46 47 48 49 50 51 52
      if (!_allTestsPassed.isCompleted) {
        _allTestsPassed.complete(true);
      }
      callbackManager.cleanup();

      // TODO(jiahaog): Print the message directing users to run with
      // `flutter test` when Web is supported.
      if (!_shouldReportResultsToNative || kIsWeb) {
        return;
      }

Dan Field's avatar
Dan Field committed
53 54 55 56
      try {
        await _channel.invokeMethod<void>(
          'allTestsFinished',
          <String, dynamic>{
57
            'results': results.map<String, dynamic>((String name, Object result) {
Dan Field's avatar
Dan Field committed
58 59 60 61 62 63 64 65
              if (result is Failure) {
                return MapEntry<String, dynamic>(name, result.details);
              }
              return MapEntry<String, Object>(name, result);
            })
          },
        );
      } on MissingPluginException {
66 67 68 69 70 71 72 73 74 75 76 77 78
        print(r'''
Warning: integration_test plugin was not detected.

If you're running the tests with `flutter drive`, please make sure your tests
are in the `integration_test/` directory of your package and use
`flutter test $path_to_test` to run it instead.

If you're running the tests with Android instrumentation or XCTest, this means
that you are not capturing test results properly! See the following link for
how to set up the integration_test plugin:

https://flutter.dev/docs/testing/integration-tests#testing-on-firebase-test-lab
''');
Dan Field's avatar
Dan Field committed
79
      }
80 81 82
    });

    final TestExceptionReporter oldTestExceptionReporter = reportTestException;
Dan Field's avatar
Dan Field committed
83 84 85 86 87 88
    reportTestException =
        (FlutterErrorDetails details, String testDescription) {
      results[testDescription] = Failure(testDescription, details.toString());
      if (!_allTestsPassed.isCompleted) {
        _allTestsPassed.complete(false);
      }
89 90 91 92 93 94 95 96 97 98
      oldTestExceptionReporter(details, testDescription);
    };
  }

  @override
  bool get overrideHttpClient => false;

  @override
  bool get registerTestTextInput => false;

99
  Size? _surfaceSize;
100 101 102 103 104 105 106 107 108 109

  // This flag is used to print warning messages when tracking performance
  // under debug mode.
  static bool _firstRun = false;

  /// Artificially changes the surface size to `size` on the Widget binding,
  /// then flushes microtasks.
  ///
  /// Set to null to use the default surface size.
  @override
110
  Future<void> setSurfaceSize(Size? size) {
111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
    return TestAsyncUtils.guard<void>(() async {
      assert(inTest);
      if (_surfaceSize == size) {
        return;
      }
      _surfaceSize = size;
      handleMetricsChanged();
    });
  }

  @override
  ViewConfiguration createViewConfiguration() {
    final double devicePixelRatio = window.devicePixelRatio;
    final Size size = _surfaceSize ?? window.physicalSize / devicePixelRatio;
    return TestViewConfiguration(
      size: size,
      window: window,
    );
  }

  @override
  Completer<bool> get allTestsPassed => _allTestsPassed;
  final Completer<bool> _allTestsPassed = Completer<bool>();

  @override
  List<Failure> get failureMethodsDetails => _failures;

  /// Similar to [WidgetsFlutterBinding.ensureInitialized].
  ///
  /// Returns an instance of the [IntegrationTestWidgetsFlutterBinding], creating and
  /// initializing it if necessary.
  static WidgetsBinding ensureInitialized() {
    if (WidgetsBinding.instance == null) {
      IntegrationTestWidgetsFlutterBinding();
    }
    assert(WidgetsBinding.instance is IntegrationTestWidgetsFlutterBinding);
147
    return WidgetsBinding.instance!;
148 149 150 151 152 153
  }

  static const MethodChannel _channel =
      MethodChannel('plugins.flutter.io/integration_test');

  /// Test results that will be populated after the tests have completed.
Dan Field's avatar
Dan Field committed
154 155 156
  ///
  /// Keys are the test descriptions, and values are either [_success] or
  /// a [Failure].
157
  @visibleForTesting
Dan Field's avatar
Dan Field committed
158
  Map<String, Object> results = <String, Object>{};
159 160 161 162 163 164 165 166 167 168

  List<Failure> get _failures => results.values.whereType<Failure>().toList();

  /// The extra data for the reported result.
  ///
  /// The values in `reportData` must be json-serializable objects or `null`.
  /// If it's `null`, no extra data is attached to the result.
  ///
  /// The default value is `null`.
  @override
169
  Map<String, dynamic>? reportData;
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184

  /// Manages callbacks received from driver side and commands send to driver
  /// side.
  final CallbackManager callbackManager = driver_actions.callbackManager;

  /// Taking a screenshot.
  ///
  /// Called by test methods. Implementation differs for each platform.
  Future<void> takeScreenshot(String screenshotName) async {
    await callbackManager.takeScreenshot(screenshotName);
  }

  /// The callback function to response the driver side input.
  @visibleForTesting
  Future<Map<String, dynamic>> callback(Map<String, String> params) async {
185
    return callbackManager.callback(
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
        params, this /* as IntegrationTestResults */);
  }

  // Emulates the Flutter driver extension, returning 'pass' or 'fail'.
  @override
  void initServiceExtensions() {
    super.initServiceExtensions();

    if (kIsWeb) {
      registerWebServiceExtension(callback);
    }

    registerServiceExtension(name: 'driver', callback: callback);
  }

  @override
  Future<void> runTest(
203
    Future<void> Function() testBody,
204 205
    VoidCallback invariantTester, {
    String description = '',
206
    Duration? timeout,
207 208 209 210 211 212 213
  }) async {
    await super.runTest(
      testBody,
      invariantTester,
      description: description,
      timeout: timeout,
    );
Dan Field's avatar
Dan Field committed
214
    results[description] ??= _success;
215 216
  }

217
  vm.VmService? _vmService;
218 219 220 221 222

  /// Initialize the [vm.VmService] settings for the timeline.
  @visibleForTesting
  Future<void> enableTimeline({
    List<String> streams = const <String>['all'],
223
    @visibleForTesting vm.VmService? vmService,
224
  }) async {
225
    assert(streams != null);
226 227 228 229 230 231 232 233 234
    assert(streams.isNotEmpty);
    if (vmService != null) {
      _vmService = vmService;
    }
    if (_vmService == null) {
      final developer.ServiceProtocolInfo info =
          await developer.Service.getInfo();
      assert(info.serverUri != null);
      _vmService = await vm_io.vmServiceConnectUri(
235
        'ws://localhost:${info.serverUri!.port}${info.serverUri!.path}ws',
236 237
      );
    }
238
    await _vmService!.setVMTimelineFlags(streams);
239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
  }

  /// Runs [action] and returns a [vm.Timeline] trace for it.
  ///
  /// Waits for the `Future` returned by [action] to complete prior to stopping
  /// the trace.
  ///
  /// The `streams` parameter limits the recorded timeline event streams to only
  /// the ones listed. By default, all streams are recorded.
  /// See `timeline_streams` in
  /// [Dart-SDK/runtime/vm/timeline.cc](https://github.com/dart-lang/sdk/blob/master/runtime/vm/timeline.cc)
  ///
  /// If [retainPriorEvents] is true, retains events recorded prior to calling
  /// [action]. Otherwise, prior events are cleared before calling [action]. By
  /// default, prior events are cleared.
  Future<vm.Timeline> traceTimeline(
255
    Future<dynamic> Function() action, {
256 257 258 259 260 261
    List<String> streams = const <String>['all'],
    bool retainPriorEvents = false,
  }) async {
    await enableTimeline(streams: streams);
    if (retainPriorEvents) {
      await action();
262
      return _vmService!.getVMTimeline();
263 264
    }

265 266
    await _vmService!.clearVMTimeline();
    final vm.Timestamp startTime = await _vmService!.getVMTimelineMicros();
267
    await action();
268
    final vm.Timestamp endTime = await _vmService!.getVMTimelineMicros();
269
    return _vmService!.getVMTimeline(
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
      timeOriginMicros: startTime.timestamp,
      timeExtentMicros: endTime.timestamp,
    );
  }

  /// This is a convenience wrap of [traceTimeline] and send the result back to
  /// the host for the [flutter_driver] style tests.
  ///
  /// This records the timeline during `action` and adds the result to
  /// [reportData] with `reportKey`. The [reportData] contains extra information
  /// from the test other than test success/fail. It will be passed back to the
  /// host and be processed by the [ResponseDataCallback] defined in
  /// [integration_test_driver.integrationDriver]. By default it will be written
  /// to `build/integration_response_data.json` with the key `timeline`.
  ///
  /// For tests with multiple calls of this method, `reportKey` needs to be a
  /// unique key, otherwise the later result will override earlier one.
  ///
  /// The `streams` and `retainPriorEvents` parameters are passed as-is to
  /// [traceTimeline].
  Future<void> traceAction(
291
    Future<dynamic> Function() action, {
292 293 294 295 296 297 298 299 300 301
    List<String> streams = const <String>['all'],
    bool retainPriorEvents = false,
    String reportKey = 'timeline',
  }) async {
    final vm.Timeline timeline = await traceTimeline(
      action,
      streams: streams,
      retainPriorEvents: retainPriorEvents,
    );
    reportData ??= <String, dynamic>{};
302
    reportData![reportKey] = timeline.toJson();
303 304 305 306 307 308 309 310
  }

  /// Watches the [FrameTiming] during `action` and report it to the binding
  /// with key `reportKey`.
  ///
  /// This can be used to implement performance tests previously using
  /// [traceAction] and [TimelineSummary] from [flutter_driver]
  Future<void> watchPerformance(
311
    Future<void> Function() action, {
312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
    String reportKey = 'performance',
  }) async {
    assert(() {
      if (_firstRun) {
        debugPrint(kDebugWarning);
        _firstRun = false;
      }
      return true;
    }());

    // The engine could batch FrameTimings and send them only once per second.
    // Delay for a sufficient time so either old FrameTimings are flushed and not
    // interfering our measurements here, or new FrameTimings are all reported.
    // TODO(CareF): remove this when flush FrameTiming is readly in engine.
    //              See https://github.com/flutter/flutter/issues/64808
    //              and https://github.com/flutter/flutter/issues/67593
    final List<FrameTiming> frameTimings = <FrameTiming>[];
329 330 331 332 333 334 335 336 337 338 339 340
    Future<void> delayForFrameTimings() async {
      int count = 0;
      while (frameTimings.isEmpty) {
        count++;
        await Future<void>.delayed(const Duration(seconds: 2));
        if (count > 20) {
          print('delayForFrameTimings is taking longer than expected...');
        }
      }
    }

    await Future<void>.delayed(const Duration(seconds: 2)); // flush old FrameTimings
341 342 343 344 345 346 347 348
    final TimingsCallback watcher = frameTimings.addAll;
    addTimingsCallback(watcher);
    await action();
    await delayForFrameTimings(); // make sure all FrameTimings are reported
    removeTimingsCallback(watcher);
    final FrameTimingSummarizer frameTimes =
        FrameTimingSummarizer(frameTimings);
    reportData ??= <String, dynamic>{};
349
    reportData![reportKey] = frameTimes.summary;
350
  }
351 352 353 354 355 356 357 358

  @override
  Timeout get defaultTestTimeout => _defaultTestTimeout ?? super.defaultTestTimeout;

  /// Configures the default timeout for [testWidgets].
  ///
  /// See [TestWidgetsFlutterBinding.defaultTestTimeout] for more details.
  set defaultTestTimeout(Timeout timeout) => _defaultTestTimeout = timeout;
359
  Timeout? _defaultTestTimeout;
360 361 362 363 364 365 366 367

  @override
  void attachRootWidget(Widget rootWidget) {
    // This is a workaround where screenshots of root widgets have incorrect
    // bounds.
    // TODO(jiahaog): Remove when https://github.com/flutter/flutter/issues/66006 is fixed.
    super.attachRootWidget(RepaintBoundary(child: rootWidget));
  }
368
}