drive.dart 17.6 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
yjbanov's avatar
yjbanov committed
2 3 4 5
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
6

7
import 'package:args/args.dart';
8
import 'package:meta/meta.dart';
9
import 'package:package_config/package_config_types.dart';
yjbanov's avatar
yjbanov committed
10

11
import '../android/android_device.dart';
12
import '../application_package.dart';
13
import '../artifacts.dart';
Devon Carew's avatar
Devon Carew committed
14
import '../base/common.dart';
15
import '../base/file_system.dart';
16
import '../base/io.dart';
17
import '../base/logger.dart';
18
import '../base/platform.dart';
19
import '../base/signals.dart';
20
import '../base/utils.dart';
21
import '../build_info.dart';
22
import '../dart/package_map.dart';
23
import '../device.dart';
24
import '../drive/drive_service.dart';
25
import '../drive/web_driver_service.dart' show Browser;
26
import '../globals.dart' as globals;
27
import '../ios/devices.dart';
28
import '../resident_runner.dart';
29
import '../runner/flutter_command.dart' show FlutterCommandCategory, FlutterCommandResult, FlutterOptions;
30
import '../web/web_device.dart';
yjbanov's avatar
yjbanov committed
31 32 33 34 35 36 37 38 39 40 41
import 'run.dart';

/// Runs integration (a.k.a. end-to-end) tests.
///
/// An integration test is a program that runs in a separate process from your
/// Flutter application. It connects to the application and acts like a user,
/// performing taps, scrolls, reading out widget properties and verifying their
/// correctness.
///
/// This command takes a target Flutter application that you would like to test
/// as the `--target` option (defaults to `lib/main.dart`). It then looks for a
42 43
/// corresponding test file within the `test_driver` directory. The test file is
/// expected to have the same name but contain the `_test.dart` suffix. The
44
/// `_test.dart` file would generally be a Dart program that uses
45
/// `package:flutter_driver` and exercises your application. Most commonly it
yjbanov's avatar
yjbanov committed
46 47 48 49 50 51 52
/// is a test written using `package:test`, but you are free to use something
/// else.
///
/// The app and the test are launched simultaneously. Once the test completes
/// the application is stopped and the command exits. If all these steps are
/// successful the exit code will be `0`. Otherwise, you will see a non-zero
/// exit code.
53
class DriveCommand extends RunCommandBase {
54 55
  DriveCommand({
    bool verboseHelp = false,
56
    @visibleForTesting FlutterDriverFactory? flutterDriverFactory,
57
    @visibleForTesting this.signalsToHandle = const <ProcessSignal>{ProcessSignal.sigint, ProcessSignal.sigterm},
58
    required FileSystem fileSystem,
59
    required Logger logger,
60
    required Platform platform,
61
    required this.signals,
62 63
  }) : _flutterDriverFactory = flutterDriverFactory,
       _fileSystem = fileSystem,
64
       _logger = logger,
65
       _fsUtils = FileSystemUtils(fileSystem: fileSystem, platform: platform),
66
       super(verboseHelp: verboseHelp) {
67
    requiresPubspecYaml();
68
    addEnableExperimentation(hide: !verboseHelp);
69 70 71 72 73

    // By default, the drive app should not publish the VM service port over mDNS
    // to prevent a local network permission dialog on iOS 14+,
    // which cannot be accepted or dismissed in a CI environment.
    addPublishPort(enabledByDefault: false, verboseHelp: verboseHelp);
74
    addMultidexOption();
75 76 77
    argParser
      ..addFlag('keep-app-running',
        help: 'Will keep the Flutter application running when done testing.\n'
78
              'By default, "flutter drive" stops the application after tests are finished, '
79
              'and "--keep-app-running" overrides this. On the other hand, if "--use-existing-app" '
80
              'is specified, then "flutter drive" instead defaults to leaving the application '
81
              'running, and "--no-keep-app-running" overrides it.',
82 83
      )
      ..addOption('use-existing-app',
84
        help: 'Connect to an already running instance via the given Dart VM Service URL. '
85
              'If this option is given, the application will not be automatically started, '
86
              'and it will only be stopped if "--no-keep-app-running" is explicitly set.',
87 88 89
        valueHelp: 'url',
      )
      ..addOption('driver',
90 91 92 93 94 95
        help: 'The test file to run on the host (as opposed to the target file to run on '
              'the device).\n'
              'By default, this file has the same base name as the target file, but in the '
              '"test_driver/" directory instead, and with "_test" inserted just before the '
              'extension, so e.g. if the target is "lib/main.dart", the driver will be '
              '"test_driver/main_test.dart".',
96
        valueHelp: 'path',
97 98 99
      )
      ..addFlag('build',
        defaultsTo: true,
100
        help: '(deprecated) Build the app before running. To use an existing app, pass the "--${FlutterOptions.kUseApplicationBinary}" '
101
              'flag with an existing APK.',
102
      )
103 104 105 106
      ..addOption('screenshot',
        valueHelp: 'path/to/directory',
        help: 'Directory location to write screenshots on test failure.',
      )
107 108
      ..addOption('driver-port',
        defaultsTo: '4444',
109
        help: 'The port where Webdriver server is launched at.',
110 111 112 113
        valueHelp: '4444'
      )
      ..addFlag('headless',
        defaultsTo: true,
114
        help: 'Whether the driver browser is going to be launched in headless mode.',
115
      )
116 117 118
      ..addOption(
        'browser-name',
        defaultsTo: Browser.chrome.cliName,
119
        help: 'Name of the browser where tests will be executed.',
120 121
        allowed: Browser.values.map((Browser e) => e.cliName),
        allowedHelp: CliEnum.allowedHelp(Browser.values),
122 123 124
      )
      ..addOption('browser-dimension',
        defaultsTo: '1600,1024',
125 126 127
        help: 'The dimension of the browser when running a Flutter Web test. '
              'This will affect screenshot and all offset-related actions.',
        valueHelp: 'width,height',
128 129 130
      )
      ..addFlag('android-emulator',
        defaultsTo: true,
131 132
        help: 'Whether to perform Flutter Driver testing using an Android Emulator. '
              'Works only if "browser-name" is set to "android-chrome".')
133
      ..addOption('chrome-binary',
134 135
        help: 'Location of the Chrome binary. '
              'Works only if "browser-name" is set to "chrome".')
136
      ..addOption('write-sksl-on-exit',
137 138
        help: 'Attempts to write an SkSL file when the drive process is finished '
              'to the provided file, overwriting it if necessary.')
139
      ..addMultiOption('test-arguments', help: 'Additional arguments to pass to the '
140 141 142
          'Dart VM running The test script.')
      ..addOption('profile-memory', help: 'Launch devtools and profile application memory, writing '
          'The output data to the file path provided to this argument as JSON.',
143 144 145 146 147 148
          valueHelp: 'profile_memory.json')
      ..addOption('timeout',
        help: 'Timeout the test after the given number of seconds. If the '
              '"--screenshot" option is provided, a screenshot will be taken '
              'before exiting. Defaults to no timeout.',
        valueHelp: '360');
yjbanov's avatar
yjbanov committed
149 150
  }

151 152 153 154 155
  final Signals signals;

  /// The [ProcessSignal]s that will lead to a screenshot being taken (if the option is provided).
  final Set<ProcessSignal> signalsToHandle;

156
  // `pub` must always be run due to the test script running from source,
157 158
  // even if an application binary is used. Default to true unless the user explicitly
  // specified not to.
159
  @override
160
  bool get shouldRunPub {
161
    if (argResults!.wasParsed('pub') && !boolArg('pub')) {
162 163 164 165
      return false;
    }
    return true;
  }
166

167
  FlutterDriverFactory? _flutterDriverFactory;
168
  final FileSystem _fileSystem;
169
  final Logger _logger;
170
  final FileSystemUtils _fsUtils;
171 172
  Timer? timeoutTimer;
  Map<ProcessSignal, Object>? screenshotTokens;
173

174
  @override
175
  final String name = 'drive';
176 177

  @override
178
  final String description = 'Run integration tests for the project on an attached device or emulator.';
179

180 181 182
  @override
  String get category => FlutterCommandCategory.project;

183
  @override
184 185
  final List<String> aliases = <String>['driver'];

186
  String? get userIdentifier => stringArg(FlutterOptions.kDeviceUser);
187

188
  String? get screenshot => stringArg('screenshot');
189

190 191
  @override
  bool get startPausedDefault => true;
192

193 194 195
  @override
  bool get cachePubGet => false;

196
  String? get applicationBinaryPath => stringArg(FlutterOptions.kUseApplicationBinary);
197 198

  Future<Device?> get targetedDevice async {
199 200 201
    return findTargetDevice(
      includeDevicesUnsupportedByProject: applicationBinaryPath == null,
    );
202 203
  }

204 205
  // Wireless iOS devices need `publish-port` to be enabled because it requires mDNS.
  // If the flag wasn't provided as an actual argument and it's a wireless device,
206 207 208 209 210
  // change it to be enabled.
  @override
  Future<bool> get disablePortPublication async {
    final ArgResults? localArgResults = argResults;
    final Device? device = await targetedDevice;
211 212 213
    final bool isWirelessIOSDevice = device is IOSDevice && device.isWirelesslyConnected;
    if (isWirelessIOSDevice && localArgResults != null && !localArgResults.wasParsed('publish-port')) {
      _logger.printTrace('A wireless iOS device is being used. Changing `publish-port` to be enabled.');
214 215
      return false;
    }
216
    return !boolArg('publish-port');
217 218
  }

219 220 221
  @override
  Future<void> validateCommand() async {
    if (userIdentifier != null) {
222
      final Device? device = await findTargetDevice();
223 224 225 226 227 228 229
      if (device is! AndroidDevice) {
        throwToolExit('--${FlutterOptions.kDeviceUser} is only supported for Android');
      }
    }
    return super.validateCommand();
  }

230
  @override
231
  Future<FlutterCommandResult> runCommand() async {
232
    final String? testFile = _getTestFile();
233
    if (testFile == null) {
234
      throwToolExit(null);
235
    }
236 237 238
    if (await _fileSystem.type(testFile) != FileSystemEntityType.file) {
      throwToolExit('Test file not found: $testFile');
    }
239
    final Device? device = await targetedDevice;
240
    if (device == null) {
241
      throwToolExit(null);
242
    }
243
    if (screenshot != null && !device.supportsScreenshot) {
244
      _logger.printError('Screenshot not supported for ${device.name}.');
245
    }
246

247 248
    final bool web = device is WebServerDevice || device is ChromiumDevice;
    _flutterDriverFactory ??= FlutterDriverFactory(
249
      applicationPackageFactory: ApplicationPackageFactory.instance!,
250
      logger: _logger,
251
      processUtils: globals.processUtils,
252
      dartSdkPath: globals.artifacts!.getArtifactPath(Artifact.engineDartBinary),
253
      devtoolsLauncher: DevtoolsLauncher.instance!,
254 255
    );
    final PackageConfig packageConfig = await loadPackageConfigWithLogging(
256
      _fileSystem.file('.packages'),
257
      logger: _logger,
258
      throwOnError: false,
259 260
    );
    final DriverService driverService = _flutterDriverFactory!.createDriverService(web);
261
    final BuildInfo buildInfo = await getBuildInfo();
262
    final DebuggingOptions debuggingOptions = await createDebuggingOptions(web);
263
    final File? applicationBinary = applicationBinaryPath == null
264
      ? null
265
      : _fileSystem.file(applicationBinaryPath);
266

267
    bool screenshotTaken = false;
268
    try {
269
      if (stringArg('use-existing-app') == null) {
270 271 272 273
        await driverService.start(
          buildInfo,
          device,
          debuggingOptions,
274
          ipv6 ?? false,
275 276 277 278 279 280 281 282 283
          applicationBinary: applicationBinary,
          route: route,
          userIdentifier: userIdentifier,
          mainPath: targetFile,
          platformArgs: <String, Object>{
            if (traceStartup)
              'trace-startup': traceStartup,
            if (web)
              '--no-launch-chrome': true,
284
            if (boolArg('multidex'))
285 286 287 288
              'multidex': true,
          }
        );
      } else {
289
        final Uri? uri = Uri.tryParse(stringArg('use-existing-app')!);
290
        if (uri == null) {
291
          throwToolExit('Invalid VM Service URI: ${stringArg('use-existing-app')}');
292
        }
293 294 295 296
        await driverService.reuseApplication(
          uri,
          device,
          debuggingOptions,
297
          ipv6 ?? false,
298
        );
299
      }
300

301
      final Future<int> testResultFuture = driverService.startTest(
302 303 304 305
        testFile,
        stringsArg('test-arguments'),
        <String, String>{},
        packageConfig,
306 307
        chromeBinary: stringArg('chrome-binary'),
        headless: boolArg('headless'),
308
        webBrowserFlags: stringsArg(FlutterOptions.kWebBrowserFlag),
309 310 311 312
        browserDimension: stringArg('browser-dimension')!.split(','),
        browserName: stringArg('browser-name'),
        driverPort: stringArg('driver-port') != null
          ? int.tryParse(stringArg('driver-port')!)
313
          : null,
314 315
        androidEmulator: boolArg('android-emulator'),
        profileMemory: stringArg('profile-memory'),
316
      );
317

318 319 320 321
      if (screenshot != null) {
        // If the test is sent a signal or times out, take a screenshot
        _registerScreenshotCallbacks(device, _fileSystem.directory(screenshot));
      }
322

323
      final int testResult = await testResultFuture;
324 325 326 327 328 329

      if (timeoutTimer != null) {
        timeoutTimer!.cancel();
      }
      _unregisterScreenshotCallbacks();

330 331
      if (testResult != 0 && screenshot != null) {
        // Take a screenshot while the app is still running.
332
        await _takeScreenshot(device, _fileSystem.directory(screenshot));
333 334
        screenshotTaken = true;
      }
335

336
      if (boolArg('keep-app-running')) {
337
        _logger.printStatus('Leaving the application running.');
338
      } else {
339 340
        final File? skslFile = stringArg('write-sksl-on-exit') != null
          ? _fileSystem.file(stringArg('write-sksl-on-exit'))
341 342 343 344 345 346
          : null;
        await driverService.stop(userIdentifier: userIdentifier, writeSkslOnExit: skslFile);
      }
      if (testResult != 0) {
        throwToolExit(null);
      }
347
    } on Exception catch (_) {
348 349 350
      // On exceptions, including ToolExit, take a screenshot on the device
      // unless a screenshot was already taken on test failure.
      if (!screenshotTaken && screenshot != null) {
351
        await _takeScreenshot(device, _fileSystem.directory(screenshot));
352 353
      }
      rethrow;
354
    }
355

356
    return FlutterCommandResult.success();
yjbanov's avatar
yjbanov committed
357 358
  }

359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
  int? get _timeoutSeconds {
    final String? timeoutString = stringArg('timeout');
    if (timeoutString == null) {
      return null;
    }
    final int? timeoutSeconds = int.tryParse(timeoutString);
    if (timeoutSeconds == null || timeoutSeconds <= 0) {
      throwToolExit(
        'Invalid value "$timeoutString" provided to the option --timeout: '
        'expected a positive integer representing seconds.',
      );
    }
    return timeoutSeconds;
  }

374
  void _registerScreenshotCallbacks(Device device, Directory screenshotDir) {
375
    _logger.printTrace('Registering signal handlers...');
376 377
    final Map<ProcessSignal, Object> tokens = <ProcessSignal, Object>{};
    for (final ProcessSignal signal in signalsToHandle) {
378 379 380 381 382
      tokens[signal] = signals.addHandler(
        signal,
        (ProcessSignal signal) {
          _unregisterScreenshotCallbacks();
          _logger.printError('Caught $signal');
383
          return _takeScreenshot(device, screenshotDir);
384 385 386 387 388 389 390 391 392 393 394
        },
      );
    }
    screenshotTokens = tokens;

    final int? timeoutSeconds = _timeoutSeconds;
    if (timeoutSeconds != null) {
      timeoutTimer = Timer(
        Duration(seconds: timeoutSeconds),
        () {
          _unregisterScreenshotCallbacks();
395
          _takeScreenshot(device, screenshotDir);
396 397 398
          throwToolExit('Timed out after $timeoutSeconds seconds');
        }
      );
399 400 401
    }
  }

402 403 404 405 406 407
  void _unregisterScreenshotCallbacks() {
    if (screenshotTokens != null) {
      _logger.printTrace('Unregistering signal handlers...');
      for (final MapEntry<ProcessSignal, Object> entry in screenshotTokens!.entries) {
        signals.removeHandler(entry.key, entry.value);
      }
408
    }
409
    timeoutTimer?.cancel();
410
  }
411

412 413
  String? _getTestFile() {
    if (argResults!['driver'] != null) {
414
      return stringArg('driver');
415
    }
416 417 418

    // If the --driver argument wasn't provided, then derive the value from
    // the target file.
419
    String appFile = _fileSystem.path.normalize(targetFile);
420

421
    // This command extends `flutter run` and therefore CWD == package dir
422
    final String packageDir = _fileSystem.currentDirectory.path;
423 424 425

    // Make appFile path relative to package directory because we are looking
    // for the corresponding test file relative to it.
426 427
    if (!_fileSystem.path.isRelative(appFile)) {
      if (!_fileSystem.path.isWithin(packageDir, appFile)) {
428
        _logger.printError(
429 430 431 432 433
          'Application file $appFile is outside the package directory $packageDir'
        );
        return null;
      }

434
      appFile = _fileSystem.path.relative(appFile, from: packageDir);
435 436
    }

437
    final List<String> parts = _fileSystem.path.split(appFile);
438 439

    if (parts.length < 2) {
440
      _logger.printError(
441 442 443 444 445 446 447 448 449
        'Application file $appFile must reside in one of the sub-directories '
        'of the package structure, not in the root directory.'
      );
      return null;
    }

    // Look for the test file inside `test_driver/` matching the sub-path, e.g.
    // if the application is `lib/foo/bar.dart`, the test file is expected to
    // be `test_driver/foo/bar_test.dart`.
450
    final String pathWithNoExtension = _fileSystem.path.withoutExtension(_fileSystem.path.joinAll(
451
      <String>[packageDir, 'test_driver', ...parts.skip(1)]));
452
    return '${pathWithNoExtension}_test${_fileSystem.path.extension(appFile)}';
453
  }
454

455
  Future<void> _takeScreenshot(Device device, Directory outputDirectory) async {
456 457 458
    if (!device.supportsScreenshot) {
      return;
    }
459
    try {
460
      outputDirectory.createSync(recursive: true);
461 462 463 464 465 466
      final File outputFile = _fsUtils.getUniqueFile(
        outputDirectory,
        'drive',
        'png',
      );
      await device.takeScreenshot(outputFile);
467
      _logger.printStatus('Screenshot written to ${outputFile.path}');
468
    } on Exception catch (error) {
469
      _logger.printError('Error taking screenshot: $error');
470
    }
471 472
  }
}