integration_test.dart 17.6 KB
Newer Older
1 2 3 4 5 6
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:developer' as developer;
7
import 'dart:io' show HttpClient, SocketException, WebSocket;
8 9
import 'dart:ui';

Dan Field's avatar
Dan Field committed
10
import 'package:flutter/foundation.dart';
11
import 'package:flutter/rendering.dart';
12 13
import 'package:flutter/services.dart';
import 'package:flutter/widgets.dart';
14
import 'package:flutter_test/flutter_test.dart';
15 16
import 'package:vm_service/vm_service.dart' as vm;

Dan Field's avatar
Dan Field committed
17
import '_callback_io.dart' if (dart.library.html) '_callback_web.dart' as driver_actions;
18 19
import '_extension_io.dart' if (dart.library.html) '_extension_web.dart';
import 'common.dart';
20
import 'src/channel.dart';
21

Dan Field's avatar
Dan Field committed
22
const String _success = 'success';
23

24 25 26 27 28 29 30 31 32 33 34 35
/// Whether results should be reported to the native side over the method
/// channel.
///
/// This is enabled by default for use by native test frameworks like Android
/// instrumentation or XCTest. When running with the Flutter Tool through
/// `flutter test integration_test` though, it will be disabled as the Flutter
/// tool will be responsible for collection of test results.
const bool _shouldReportResultsToNative = bool.fromEnvironment(
  'INTEGRATION_TEST_SHOULD_REPORT_RESULTS_TO_NATIVE',
  defaultValue: true,
);

36 37
/// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results
/// on a channel to adapt them to native instrumentation test format.
Dan Field's avatar
Dan Field committed
38 39 40
class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding implements IntegrationTestResults {
  /// Sets up a listener to report that the tests are finished when everything is
  /// torn down.
41 42
  IntegrationTestWidgetsFlutterBinding() {
    tearDownAll(() async {
43
      if (!_allTestsPassed.isCompleted) {
44
        _allTestsPassed.complete(failureMethodsDetails.isEmpty);
45 46 47 48 49 50 51 52 53
      }
      callbackManager.cleanup();

      // TODO(jiahaog): Print the message directing users to run with
      // `flutter test` when Web is supported.
      if (!_shouldReportResultsToNative || kIsWeb) {
        return;
      }

Dan Field's avatar
Dan Field committed
54
      try {
55
        await integrationTestChannel.invokeMethod<void>(
Dan Field's avatar
Dan Field committed
56 57
          'allTestsFinished',
          <String, dynamic>{
58
            'results': results.map<String, dynamic>((String name, Object result) {
Dan Field's avatar
Dan Field committed
59 60 61 62
              if (result is Failure) {
                return MapEntry<String, dynamic>(name, result.details);
              }
              return MapEntry<String, Object>(name, result);
63
            }),
Dan Field's avatar
Dan Field committed
64 65 66
          },
        );
      } on MissingPluginException {
67
        debugPrint(r'''
68 69 70 71 72 73 74 75 76 77 78 79
Warning: integration_test plugin was not detected.

If you're running the tests with `flutter drive`, please make sure your tests
are in the `integration_test/` directory of your package and use
`flutter test $path_to_test` to run it instead.

If you're running the tests with Android instrumentation or XCTest, this means
that you are not capturing test results properly! See the following link for
how to set up the integration_test plugin:

https://flutter.dev/docs/testing/integration-tests#testing-on-firebase-test-lab
''');
Dan Field's avatar
Dan Field committed
80
      }
81 82 83
    });

    final TestExceptionReporter oldTestExceptionReporter = reportTestException;
Dan Field's avatar
Dan Field committed
84 85 86
    reportTestException =
        (FlutterErrorDetails details, String testDescription) {
      results[testDescription] = Failure(testDescription, details.toString());
87 88 89 90 91 92 93 94 95 96
      oldTestExceptionReporter(details, testDescription);
    };
  }

  @override
  bool get overrideHttpClient => false;

  @override
  bool get registerTestTextInput => false;

97
  Size? _surfaceSize;
98 99 100 101 102 103 104 105 106 107

  // This flag is used to print warning messages when tracking performance
  // under debug mode.
  static bool _firstRun = false;

  /// Artificially changes the surface size to `size` on the Widget binding,
  /// then flushes microtasks.
  ///
  /// Set to null to use the default surface size.
  @override
108
  Future<void> setSurfaceSize(Size? size) {
109 110 111 112 113 114 115 116 117 118 119 120
    return TestAsyncUtils.guard<void>(() async {
      assert(inTest);
      if (_surfaceSize == size) {
        return;
      }
      _surfaceSize = size;
      handleMetricsChanged();
    });
  }

  @override
  ViewConfiguration createViewConfiguration() {
121 122 123
    final FlutterView view = platformDispatcher.implicitView!;
    final double devicePixelRatio = view.devicePixelRatio;
    final Size size = _surfaceSize ?? view.physicalSize / devicePixelRatio;
124
    return TestViewConfiguration.fromView(
125
      size: size,
126
      view: view,
127 128 129 130 131 132 133 134
    );
  }

  @override
  Completer<bool> get allTestsPassed => _allTestsPassed;
  final Completer<bool> _allTestsPassed = Completer<bool>();

  @override
135
  List<Failure> get failureMethodsDetails => results.values.whereType<Failure>().toList();
136

137 138 139 140 141 142 143
  @override
  void initInstances() {
    super.initInstances();
    _instance = this;
  }

  /// The singleton instance of this object.
144
  ///
145 146 147 148 149 150
  /// Provides access to the features exposed by this class. The binding must
  /// be initialized before using this getter; this is typically done by calling
  /// [IntegrationTestWidgetsFlutterBinding.ensureInitialized].
  static IntegrationTestWidgetsFlutterBinding get instance => BindingBase.checkInstance(_instance);
  static IntegrationTestWidgetsFlutterBinding? _instance;

151 152
  /// Returns an instance of the [IntegrationTestWidgetsFlutterBinding], creating and
  /// initializing it if necessary.
153 154 155 156 157
  ///
  /// See also:
  ///
  ///  * [WidgetsFlutterBinding.ensureInitialized], the equivalent in the widgets framework.
  static IntegrationTestWidgetsFlutterBinding ensureInitialized() {
158
    if (_instance == null) {
159
      IntegrationTestWidgetsFlutterBinding();
160
    }
161
    return _instance!;
162 163 164
  }

  /// Test results that will be populated after the tests have completed.
Dan Field's avatar
Dan Field committed
165 166 167
  ///
  /// Keys are the test descriptions, and values are either [_success] or
  /// a [Failure].
168
  @visibleForTesting
Dan Field's avatar
Dan Field committed
169
  Map<String, Object> results = <String, Object>{};
170 171 172 173 174 175 176 177

  /// The extra data for the reported result.
  ///
  /// The values in `reportData` must be json-serializable objects or `null`.
  /// If it's `null`, no extra data is attached to the result.
  ///
  /// The default value is `null`.
  @override
178
  Map<String, dynamic>? reportData;
179 180 181 182 183

  /// Manages callbacks received from driver side and commands send to driver
  /// side.
  final CallbackManager callbackManager = driver_actions.callbackManager;

184 185 186 187
  /// Takes a screenshot.
  ///
  /// On Android, you need to call `convertFlutterSurfaceToImage()`, and
  /// pump a frame before taking a screenshot.
188
  Future<List<int>> takeScreenshot(String screenshotName, [Map<String, Object?>? args]) async {
189 190
    reportData ??= <String, dynamic>{};
    reportData!['screenshots'] ??= <dynamic>[];
191
    final Map<String, dynamic> data = await callbackManager.takeScreenshot(screenshotName, args);
192 193 194 195 196 197 198 199 200 201
    assert(data.containsKey('bytes'));

    (reportData!['screenshots']! as List<dynamic>).add(data);
    return data['bytes']! as List<int>;
  }

  /// Android only. Converts the Flutter surface to an image view.
  /// Be aware that if you are conducting a perf test, you may not want to call
  /// this method since the this is an expensive operation that affects the
  /// rendering of a Flutter app.
202
  ///
203 204 205 206
  /// Once the screenshot is taken, call `revertFlutterImage()` to restore
  /// the original Flutter surface.
  Future<void> convertFlutterSurfaceToImage() async {
    await callbackManager.convertFlutterSurfaceToImage();
207 208 209 210 211
  }

  /// The callback function to response the driver side input.
  @visibleForTesting
  Future<Map<String, dynamic>> callback(Map<String, String> params) async {
212
    return callbackManager.callback(
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
        params, this /* as IntegrationTestResults */);
  }

  // Emulates the Flutter driver extension, returning 'pass' or 'fail'.
  @override
  void initServiceExtensions() {
    super.initServiceExtensions();

    if (kIsWeb) {
      registerWebServiceExtension(callback);
    }

    registerServiceExtension(name: 'driver', callback: callback);
  }

  @override
  Future<void> runTest(
230
    Future<void> Function() testBody,
231 232
    VoidCallback invariantTester, {
    String description = '',
233 234 235 236
    @Deprecated(
      'This parameter has no effect. Use the `timeout` parameter on `testWidgets` instead. '
      'This feature was deprecated after v2.6.0-1.0.pre.'
    )
237
    Duration? timeout,
238 239 240 241 242 243
  }) async {
    await super.runTest(
      testBody,
      invariantTester,
      description: description,
    );
Dan Field's avatar
Dan Field committed
244
    results[description] ??= _success;
245 246
  }

247
  vm.VmService? _vmService;
248 249 250 251 252

  /// Initialize the [vm.VmService] settings for the timeline.
  @visibleForTesting
  Future<void> enableTimeline({
    List<String> streams = const <String>['all'],
253
    @visibleForTesting vm.VmService? vmService,
254
    @visibleForTesting HttpClient? httpClient,
255 256 257 258 259 260
  }) async {
    assert(streams.isNotEmpty);
    if (vmService != null) {
      _vmService = vmService;
    }
    if (_vmService == null) {
261
      final developer.ServiceProtocolInfo info = await developer.Service.getInfo();
262
      assert(info.serverUri != null);
263 264 265 266 267 268 269 270 271 272 273 274
      final String address = 'ws://localhost:${info.serverUri!.port}${info.serverUri!.path}ws';
      try {
        _vmService = await _vmServiceConnectUri(address, httpClient: httpClient);
      } on SocketException catch(e, s) {
        throw StateError(
          'Failed to connect to VM Service at $address.\n'
          'This may happen if DDS is enabled. If this test was launched via '
          '`flutter drive`, try adding `--no-dds`.\n'
          'The original exception was:\n'
          '$e\n$s',
        );
      }
275
    }
276
    await _vmService!.setVMTimelineFlags(streams);
277 278 279 280 281 282 283 284 285 286
  }

  /// Runs [action] and returns a [vm.Timeline] trace for it.
  ///
  /// Waits for the `Future` returned by [action] to complete prior to stopping
  /// the trace.
  ///
  /// The `streams` parameter limits the recorded timeline event streams to only
  /// the ones listed. By default, all streams are recorded.
  /// See `timeline_streams` in
287
  /// [Dart-SDK/runtime/vm/timeline.cc](https://github.com/dart-lang/sdk/blob/main/runtime/vm/timeline.cc)
288 289 290 291 292
  ///
  /// If [retainPriorEvents] is true, retains events recorded prior to calling
  /// [action]. Otherwise, prior events are cleared before calling [action]. By
  /// default, prior events are cleared.
  Future<vm.Timeline> traceTimeline(
293
    Future<dynamic> Function() action, {
294 295 296 297 298 299
    List<String> streams = const <String>['all'],
    bool retainPriorEvents = false,
  }) async {
    await enableTimeline(streams: streams);
    if (retainPriorEvents) {
      await action();
300
      return _vmService!.getVMTimeline();
301 302
    }

303 304
    await _vmService!.clearVMTimeline();
    final vm.Timestamp startTime = await _vmService!.getVMTimelineMicros();
305
    await action();
306
    final vm.Timestamp endTime = await _vmService!.getVMTimelineMicros();
307
    return _vmService!.getVMTimeline(
308 309 310 311 312
      timeOriginMicros: startTime.timestamp,
      timeExtentMicros: endTime.timestamp,
    );
  }

313 314
  /// This is a convenience method that calls [traceTimeline] and sends the
  /// result back to the host for the [flutter_driver] style tests.
315 316 317 318 319 320 321 322 323
  ///
  /// This records the timeline during `action` and adds the result to
  /// [reportData] with `reportKey`. The [reportData] contains extra information
  /// from the test other than test success/fail. It will be passed back to the
  /// host and be processed by the [ResponseDataCallback] defined in
  /// [integration_test_driver.integrationDriver]. By default it will be written
  /// to `build/integration_response_data.json` with the key `timeline`.
  ///
  /// For tests with multiple calls of this method, `reportKey` needs to be a
324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
  /// unique key, otherwise the later result will override earlier one. Tests
  /// that call this multiple times must also provide a custom
  /// [ResponseDataCallback] to decide where and how to write the output
  /// timelines. For example,
  ///
  /// ```dart
  /// import 'package:integration_test/integration_test_driver.dart';
  ///
  /// Future<void> main() {
  ///   return integrationDriver(
  ///     responseDataCallback: (data) async {
  ///       if (data != null) {
  ///         for (var entry in data.entries) {
  ///           print('Writing ${entry.key} to the disk.');
  ///           await writeResponseData(
  ///             entry.value as Map<String, dynamic>,
  ///             testOutputFilename: entry.key,
  ///           );
  ///         }
  ///       }
  ///     },
  ///   );
  /// }
  /// ```
348 349 350 351
  ///
  /// The `streams` and `retainPriorEvents` parameters are passed as-is to
  /// [traceTimeline].
  Future<void> traceAction(
352
    Future<dynamic> Function() action, {
353 354 355 356 357 358 359 360 361 362
    List<String> streams = const <String>['all'],
    bool retainPriorEvents = false,
    String reportKey = 'timeline',
  }) async {
    final vm.Timeline timeline = await traceTimeline(
      action,
      streams: streams,
      retainPriorEvents: retainPriorEvents,
    );
    reportData ??= <String, dynamic>{};
363
    reportData![reportKey] = timeline.toJson();
364 365
  }

366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
  Future<_GarbageCollectionInfo> _runAndGetGCInfo(Future<void> Function() action) async {
    if (kIsWeb) {
      await action();
      return const _GarbageCollectionInfo();
    }

    final vm.Timeline timeline = await traceTimeline(
      action,
      streams: <String>['GC'],
    );

    final int oldGenGCCount = timeline.traceEvents!.where((vm.TimelineEvent event) {
      return event.json!['cat'] == 'GC' && event.json!['name'] == 'CollectOldGeneration';
    }).length;
    final int newGenGCCount = timeline.traceEvents!.where((vm.TimelineEvent event) {
      return event.json!['cat'] == 'GC' && event.json!['name'] == 'CollectNewGeneration';
    }).length;
    return _GarbageCollectionInfo(
      oldCount: oldGenGCCount,
      newCount: newGenGCCount,
    );
  }

389 390 391 392 393 394
  /// Watches the [FrameTiming] during `action` and report it to the binding
  /// with key `reportKey`.
  ///
  /// This can be used to implement performance tests previously using
  /// [traceAction] and [TimelineSummary] from [flutter_driver]
  Future<void> watchPerformance(
395
    Future<void> Function() action, {
396 397 398 399 400 401 402 403 404 405 406 407 408
    String reportKey = 'performance',
  }) async {
    assert(() {
      if (_firstRun) {
        debugPrint(kDebugWarning);
        _firstRun = false;
      }
      return true;
    }());

    // The engine could batch FrameTimings and send them only once per second.
    // Delay for a sufficient time so either old FrameTimings are flushed and not
    // interfering our measurements here, or new FrameTimings are all reported.
Lioness100's avatar
Lioness100 committed
409
    // TODO(CareF): remove this when flush FrameTiming is readily in engine.
410 411 412
    //              See https://github.com/flutter/flutter/issues/64808
    //              and https://github.com/flutter/flutter/issues/67593
    final List<FrameTiming> frameTimings = <FrameTiming>[];
413 414 415 416 417 418
    Future<void> delayForFrameTimings() async {
      int count = 0;
      while (frameTimings.isEmpty) {
        count++;
        await Future<void>.delayed(const Duration(seconds: 2));
        if (count > 20) {
419
          debugPrint('delayForFrameTimings is taking longer than expected...');
420 421 422 423 424
        }
      }
    }

    await Future<void>.delayed(const Duration(seconds: 2)); // flush old FrameTimings
425 426
    final TimingsCallback watcher = frameTimings.addAll;
    addTimingsCallback(watcher);
427 428
    final _GarbageCollectionInfo gcInfo = await _runAndGetGCInfo(action);

429 430
    await delayForFrameTimings(); // make sure all FrameTimings are reported
    removeTimingsCallback(watcher);
431 432 433 434 435 436

    final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(
      frameTimings,
      newGenGCCount: gcInfo.newCount,
      oldGenGCCount: gcInfo.oldCount,
    );
437
    reportData ??= <String, dynamic>{};
438
    reportData![reportKey] = frameTimes.summary;
439
  }
440 441

  @override
442
  Timeout defaultTestTimeout = Timeout.none;
443

444 445 446 447 448 449 450 451
  @override
  void attachRootWidget(Widget rootWidget) {
    // This is a workaround where screenshots of root widgets have incorrect
    // bounds.
    // TODO(jiahaog): Remove when https://github.com/flutter/flutter/issues/66006 is fixed.
    super.attachRootWidget(RepaintBoundary(child: rootWidget));
  }

452 453 454 455 456 457 458 459 460 461 462 463 464 465
  @override
  void reportExceptionNoticed(FlutterErrorDetails exception) {
    // This method is called to log errors as they happen, and they will also
    // be eventually logged again at the end of the tests. The superclass
    // behavior is specific to the "live" execution semantics of
    // [LiveTestWidgetsFlutterBinding] so users don't have to wait until tests
    // finish to see the stack traces.
    //
    // Disable this because Integration Tests follow the semantics of
    // [AutomatedTestWidgetsFlutterBinding] that does not log the stack traces
    // live, and avoids the doubly logged stack trace.
    // TODO(jiahaog): Integration test binding should not inherit from
    // `LiveTestWidgetsFlutterBinding` https://github.com/flutter/flutter/issues/81534
  }
466
}
467 468 469 470 471 472 473 474

@immutable
class _GarbageCollectionInfo {
  const _GarbageCollectionInfo({this.oldCount = -1, this.newCount = -1});

  final int oldCount;
  final int newCount;
}
475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500

// Connect to the given uri and return a new [VmService] instance.
//
// Copied from vm_service_io so that we can pass a custom [HttpClient] for
// testing. Currently, the WebSocket API reuses an HttpClient that
// is created before the test can change the HttpOverrides.
Future<vm.VmService> _vmServiceConnectUri(
  String wsUri, {
  HttpClient? httpClient,
}) async {
  final WebSocket socket = await WebSocket.connect(wsUri, customClient: httpClient);
  final StreamController<dynamic> controller = StreamController<dynamic>();
  final Completer<void> streamClosedCompleter = Completer<void>();

  socket.listen(
    (dynamic data) => controller.add(data),
    onDone: () => streamClosedCompleter.complete(),
  );

  return vm.VmService(
    controller.stream,
    (String message) => socket.add(message),
    disposeHandler: () => socket.close(),
    streamClosed: streamClosedCompleter.future,
  );
}