drive.dart 21.9 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
yjbanov's avatar
yjbanov committed
2 3 4 5
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
6 7 8
import 'dart:math' as math;

import 'package:meta/meta.dart';
9
import 'package:webdriver/async_io.dart' as async_io;
yjbanov's avatar
yjbanov committed
10

11
import '../application_package.dart';
Devon Carew's avatar
Devon Carew committed
12
import '../base/common.dart';
13
import '../base/file_system.dart';
14
import '../base/process.dart';
15
import '../build_info.dart';
16
import '../cache.dart';
17
import '../dart/package_map.dart';
18
import '../dart/sdk.dart';
19
import '../device.dart';
20
import '../globals.dart' as globals;
21
import '../project.dart';
22
import '../resident_runner.dart';
23
import '../runner/flutter_command.dart' show FlutterCommandResult;
24
import '../web/web_runner.dart';
yjbanov's avatar
yjbanov committed
25 26 27 28 29 30 31 32 33 34 35
import 'run.dart';

/// Runs integration (a.k.a. end-to-end) tests.
///
/// An integration test is a program that runs in a separate process from your
/// Flutter application. It connects to the application and acts like a user,
/// performing taps, scrolls, reading out widget properties and verifying their
/// correctness.
///
/// This command takes a target Flutter application that you would like to test
/// as the `--target` option (defaults to `lib/main.dart`). It then looks for a
36 37
/// corresponding test file within the `test_driver` directory. The test file is
/// expected to have the same name but contain the `_test.dart` suffix. The
38
/// `_test.dart` file would generally be a Dart program that uses
39
/// `package:flutter_driver` and exercises your application. Most commonly it
yjbanov's avatar
yjbanov committed
40 41 42 43 44 45 46
/// is a test written using `package:test`, but you are free to use something
/// else.
///
/// The app and the test are launched simultaneously. Once the test completes
/// the application is stopped and the command exits. If all these steps are
/// successful the exit code will be `0`. Otherwise, you will see a non-zero
/// exit code.
47 48
class DriveCommand extends RunCommandBase {
  DriveCommand() {
49 50
    requiresPubspecYaml();

51 52 53 54
    argParser
      ..addFlag('keep-app-running',
        defaultsTo: null,
        help: 'Will keep the Flutter application running when done testing.\n'
55 56 57
              'By default, "flutter drive" stops the application after tests are finished, '
              'and --keep-app-running overrides this. On the other hand, if --use-existing-app '
              'is specified, then "flutter drive" instead defaults to leaving the application '
58 59 60
              'running, and --no-keep-app-running overrides it.',
      )
      ..addOption('use-existing-app',
61 62
        help: 'Connect to an already running instance via the given observatory URL. '
              'If this option is given, the application will not be automatically started, '
63 64 65 66
              'and it will only be stopped if --no-keep-app-running is explicitly set.',
        valueHelp: 'url',
      )
      ..addOption('driver',
67 68 69 70 71 72
        help: 'The test file to run on the host (as opposed to the target file to run on '
              'the device).\n'
              'By default, this file has the same base name as the target file, but in the '
              '"test_driver/" directory instead, and with "_test" inserted just before the '
              'extension, so e.g. if the target is "lib/main.dart", the driver will be '
              '"test_driver/main_test.dart".',
73
        valueHelp: 'path',
74 75 76 77
      )
      ..addFlag('build',
        defaultsTo: true,
        help: 'Build the app before running.',
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
      )
      ..addOption('driver-port',
        defaultsTo: '4444',
        help: 'The port where Webdriver server is launched at. Defaults to 4444.',
        valueHelp: '4444'
      )
      ..addFlag('headless',
        defaultsTo: true,
        help: 'Whether the driver browser is going to be launched in headless mode. Defaults to true.',
      )
      ..addOption('browser-name',
        defaultsTo: 'chrome',
        help: 'Name of browser where tests will be executed. \n'
              'Following browsers are supported: \n'
              'Chrome, Firefox, Safari (macOS and iOS) and Edge. Defaults to Chrome.',
        allowed: <String>[
94
          'android-chrome',
95 96 97 98 99 100 101 102 103 104 105 106
          'chrome',
          'edge',
          'firefox',
          'ios-safari',
          'safari',
        ]
      )
      ..addOption('browser-dimension',
        defaultsTo: '1600,1024',
        help: 'The dimension of browser when running Flutter Web test. \n'
              'This will affect screenshot and all offset-related actions. \n'
              'By default. it is set to 1600,1024 (1600 by 1024).',
107 108 109 110
      )
      ..addFlag('android-emulator',
        defaultsTo: true,
        help: 'Whether to perform Flutter Driver testing on Android Emulator.'
111 112 113 114
          'Works only if \'browser-name\' is set to \'android-chrome\'')
      ..addOption('chrome-binary',
        help: 'Location of Chrome binary. '
          'Works only if \'browser-name\' is set to \'chrome\'');
yjbanov's avatar
yjbanov committed
115 116
  }

117
  @override
118
  final String name = 'drive';
119 120

  @override
121
  final String description = 'Runs Flutter Driver tests for the current project.';
122 123

  @override
124 125 126 127
  final List<String> aliases = <String>['driver'];

  Device _device;
  Device get device => _device;
128
  bool get shouldBuild => boolArg('build');
yjbanov's avatar
yjbanov committed
129

130
  bool get verboseSystemLogs => boolArg('verbose-system-logs');
131

132
  /// Subscription to log messages printed on the device or simulator.
133
  // ignore: cancel_subscriptions
134 135
  StreamSubscription<String> _deviceLogSubscription;

136
  @override
137
  Future<FlutterCommandResult> runCommand() async {
138
    final String testFile = _getTestFile();
139
    if (testFile == null) {
140
      throwToolExit(null);
141
    }
yjbanov's avatar
yjbanov committed
142

143
    _device = await findTargetDevice();
144
    if (device == null) {
145
      throwToolExit(null);
146
    }
147

148
    if (await globals.fs.type(testFile) != FileSystemEntityType.file) {
149
      throwToolExit('Test file not found: $testFile');
150
    }
yjbanov's avatar
yjbanov committed
151

152
    String observatoryUri;
153
    ResidentRunner residentRunner;
154
    final BuildInfo buildInfo = getBuildInfo();
155
    final bool isWebPlatform = await device.targetPlatform == TargetPlatform.web_javascript;
156
    if (argResults['use-existing-app'] == null) {
157
      globals.printStatus('Starting application: $targetFile');
158

159
      if (buildInfo.isRelease && !isWebPlatform) {
160
        // This is because we need VM service to be able to drive the app.
161
        // For Flutter Web, testing in release mode is allowed.
162
        throwToolExit(
163
          'Flutter Driver (non-web) does not support running in release mode.\n'
164 165 166 167 168 169
          '\n'
          'Use --profile mode for testing application performance.\n'
          'Use --debug (default) mode for testing correctness (with assertions).'
        );
      }

170
      if (isWebPlatform && buildInfo.isDebug) {
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
        // TODO(angjieli): remove this once running against
        // target under test_driver in debug mode is supported
        throwToolExit(
          'Flutter Driver web does not support running in debug mode.\n'
          '\n'
          'Use --profile mode for testing application performance.\n'
          'Use --release mode for testing correctness (with assertions).'
        );
      }

      Uri webUri;

      if (isWebPlatform) {
        // Start Flutter web application for current test
        final FlutterProject flutterProject = FlutterProject.current();
        final FlutterDevice flutterDevice = await FlutterDevice.create(
          device,
          flutterProject: flutterProject,
          target: targetFile,
190
          buildInfo: buildInfo
191 192 193 194 195 196
        );
        residentRunner = webRunnerFactory.createWebRunner(
          flutterDevice,
          target: targetFile,
          flutterProject: flutterProject,
          ipv6: ipv6,
197 198 199 200 201 202 203 204 205
          debuggingOptions: getBuildInfo().isRelease ?
            DebuggingOptions.disabled(
              getBuildInfo(),
              port: stringArg('web-port')
            )
            : DebuggingOptions.enabled(
              getBuildInfo(),
              port: stringArg('web-port')
            ),
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
          stayResident: false,
          urlTunneller: null,
        );
        final Completer<void> appStartedCompleter = Completer<void>.sync();
        final int result = await residentRunner.run(
          appStartedCompleter: appStartedCompleter,
          route: route,
        );
        if (result != 0) {
          throwToolExit(null, exitCode: result);
        }
        // Wait until the app is started.
        await appStartedCompleter.future;
        webUri = residentRunner.uri;
      }

      final LaunchResult result = await appStarter(this, webUri);
223
      if (result == null) {
224
        throwToolExit('Application failed to start. Will not run test. Quitting.', exitCode: 1);
225
      }
226
      observatoryUri = result.observatoryUri.toString();
227
    } else {
228
      globals.printStatus('Will connect to already running application instance.');
229
      observatoryUri = stringArg('use-existing-app');
yjbanov's avatar
yjbanov committed
230 231
    }

232 233
    Cache.releaseLockEarly();

234 235 236 237
    final Map<String, String> environment = <String, String>{
      'VM_SERVICE_URL': observatoryUri,
    };

238
    async_io.WebDriver driver;
239 240 241
    // For web device, WebDriver session will be launched beforehand
    // so that FlutterDriver can reuse it.
    if (isWebPlatform) {
242 243 244
      final Browser browser = _browserNameToEnum(
          argResults['browser-name'].toString());
      final String driverPort = argResults['driver-port'].toString();
245
      // start WebDriver
246 247 248 249 250
      try {
        driver = await _createDriver(
          driverPort,
          browser,
          argResults['headless'].toString() == 'true',
251
          stringArg('chrome-binary'),
252 253 254 255 256 257 258 259 260
        );
      } on Exception catch (ex) {
        throwToolExit(
          'Unable to start WebDriver Session for Flutter for Web testing. \n'
          'Make sure you have the correct WebDriver Server running at $driverPort. \n'
          'Make sure the WebDriver Server matches option --browser-name. \n'
          '$ex'
        );
      }
261

262 263
      final bool isAndroidChrome = browser == Browser.androidChrome;
      final bool useEmulator = argResults['android-emulator'] as bool;
264
      // set window size
265 266 267 268 269 270 271 272 273 274 275
      // for android chrome, skip such action
      if (!isAndroidChrome) {
        final List<String> dimensions = argResults['browser-dimension'].split(
            ',') as List<String>;
        assert(dimensions.length == 2);
        int x, y;
        try {
          x = int.parse(dimensions[0]);
          y = int.parse(dimensions[1]);
        } on FormatException catch (ex) {
          throwToolExit('''
276 277 278
Dimension provided to --browser-dimension is invalid:
$ex
        ''');
279 280
        }
        final async_io.Window window = await driver.window;
281 282
        await window.setLocation(const math.Point<int>(0, 0));
        await window.setSize(math.Rectangle<int>(0, 0, x, y));
283 284 285 286 287 288 289 290 291
      }

      // add driver info to environment variables
      environment.addAll(<String, String> {
        'DRIVER_SESSION_ID': driver.id,
        'DRIVER_SESSION_URI': driver.uri.toString(),
        'DRIVER_SESSION_SPEC': driver.spec.toString(),
        'SUPPORT_TIMELINE_ACTION': (browser == Browser.chrome).toString(),
        'FLUTTER_WEB_TEST': 'true',
292
        'ANDROID_CHROME_ON_EMULATOR': (isAndroidChrome && useEmulator).toString(),
293 294 295
      });
    }

yjbanov's avatar
yjbanov committed
296
    try {
297
      await testRunner(<String>[testFile], environment);
298
    } on Exception catch (error, stackTrace) {
299
      if (error is ToolExit) {
300
        rethrow;
301
      }
302
      throw Exception('Unable to run test: $error\n$stackTrace');
yjbanov's avatar
yjbanov committed
303
    } finally {
304
      await residentRunner?.exit();
305
      await driver?.quit();
306
      if (boolArg('keep-app-running') ?? (argResults['use-existing-app'] != null)) {
307
        globals.printStatus('Leaving the application running.');
308
      } else {
309
        globals.printStatus('Stopping application instance.');
310
        await appStopper(this);
311
      }
yjbanov's avatar
yjbanov committed
312
    }
313

314
    return FlutterCommandResult.success();
yjbanov's avatar
yjbanov committed
315 316 317
  }

  String _getTestFile() {
318
    if (argResults['driver'] != null) {
319
      return stringArg('driver');
320
    }
321 322 323

    // If the --driver argument wasn't provided, then derive the value from
    // the target file.
324
    String appFile = globals.fs.path.normalize(targetFile);
325

326
    // This command extends `flutter run` and therefore CWD == package dir
327
    final String packageDir = globals.fs.currentDirectory.path;
328 329 330

    // Make appFile path relative to package directory because we are looking
    // for the corresponding test file relative to it.
331 332 333
    if (!globals.fs.path.isRelative(appFile)) {
      if (!globals.fs.path.isWithin(packageDir, appFile)) {
        globals.printError(
334 335 336 337 338
          'Application file $appFile is outside the package directory $packageDir'
        );
        return null;
      }

339
      appFile = globals.fs.path.relative(appFile, from: packageDir);
340 341
    }

342
    final List<String> parts = globals.fs.path.split(appFile);
343 344

    if (parts.length < 2) {
345
      globals.printError(
346 347 348 349 350 351 352 353 354
        'Application file $appFile must reside in one of the sub-directories '
        'of the package structure, not in the root directory.'
      );
      return null;
    }

    // Look for the test file inside `test_driver/` matching the sub-path, e.g.
    // if the application is `lib/foo/bar.dart`, the test file is expected to
    // be `test_driver/foo/bar_test.dart`.
355
    final String pathWithNoExtension = globals.fs.path.withoutExtension(globals.fs.path.joinAll(
356
      <String>[packageDir, 'test_driver', ...parts.skip(1)]));
357
    return '${pathWithNoExtension}_test${globals.fs.path.extension(appFile)}';
yjbanov's avatar
yjbanov committed
358 359
  }
}
360 361

Future<Device> findTargetDevice() async {
362
  final List<Device> devices = await deviceManager.findTargetDevices(FlutterProject.current());
363

364
  if (deviceManager.hasSpecifiedDeviceId) {
365
    if (devices.isEmpty) {
366
      globals.printStatus("No devices found with name or id matching '${deviceManager.specifiedDeviceId}'");
367 368 369
      return null;
    }
    if (devices.length > 1) {
370
      globals.printStatus("Found ${devices.length} devices with name or id matching '${deviceManager.specifiedDeviceId}':");
371
      await Device.printDevices(devices);
372 373 374
      return null;
    }
    return devices.first;
375 376
  }

377
  if (devices.isEmpty) {
378
    globals.printError('No devices found.');
379
    return null;
380
  } else if (devices.length > 1) {
381
    globals.printStatus('Found multiple connected devices:');
382
    await Device.printDevices(devices);
383
  }
384
  globals.printStatus('Using device ${devices.first.name}.');
385
  return devices.first;
386 387 388
}

/// Starts the application on the device given command configuration.
389
typedef AppStarter = Future<LaunchResult> Function(DriveCommand command, Uri webUri);
390

391
AppStarter appStarter = _startApp; // (mutable for testing)
392
void restoreAppStarter() {
393
  appStarter = _startApp;
394 395
}

396
Future<LaunchResult> _startApp(DriveCommand command, Uri webUri) async {
397
  final String mainPath = findMainDartFile(command.targetFile);
398 399
  if (await globals.fs.type(mainPath) != FileSystemEntityType.file) {
    globals.printError('Tried to run $mainPath, but that file does not exist.');
400
    return null;
401 402
  }

403
  globals.printTrace('Stopping previously running application, if any.');
404 405
  await appStopper(command);

406
  final ApplicationPackage package = await command.applicationPackages
407
      .getPackageForPlatform(await command.device.targetPlatform);
408 409

  if (command.shouldBuild) {
410
    globals.printTrace('Installing application package.');
411
    if (await command.device.isAppInstalled(package)) {
412
      await command.device.uninstallApp(package);
413
    }
414 415
    await command.device.installApp(package);
  }
416

417
  final Map<String, dynamic> platformArgs = <String, dynamic>{};
418
  if (command.traceStartup) {
419
    platformArgs['trace-startup'] = command.traceStartup;
420
  }
421

422 423 424 425 426 427 428 429 430
  if (webUri != null) {
    platformArgs['uri'] = webUri.toString();
    if (!command.getBuildInfo().isDebug) {
      // For web device, startApp will be triggered twice
      // and it will error out for chrome the second time.
      platformArgs['no-launch-chrome'] = true;
    }
  }

431
  globals.printTrace('Starting application.');
432 433

  // Forward device log messages to the terminal window running the "drive" command.
434 435 436 437
  final DeviceLogReader logReader = await command.device.getLogReader(app: package);
  command._deviceLogSubscription = logReader
    .logLines
    .listen(globals.printStatus);
438

439
  final LaunchResult result = await command.device.startApp(
440 441 442
    package,
    mainPath: mainPath,
    route: command.route,
443
    debuggingOptions: DebuggingOptions.enabled(
444
      command.getBuildInfo(),
Devon Carew's avatar
Devon Carew committed
445
      startPaused: true,
446
      hostVmServicePort: command.hostVmservicePort,
447
      verboseSystemLogs: command.verboseSystemLogs,
448
      cacheSkSL: command.cacheSkSL,
449
      dumpSkpOnShaderCompilation: command.dumpSkpOnShaderCompilation,
Devon Carew's avatar
Devon Carew committed
450
    ),
451
    platformArgs: platformArgs,
452
    prebuiltApplication: !command.shouldBuild,
453 454
  );

455 456
  if (!result.started) {
    await command._deviceLogSubscription.cancel();
457
    return null;
458 459
  }

460
  return result;
461 462 463
}

/// Runs driver tests.
464
typedef TestRunner = Future<void> Function(List<String> testArgs, Map<String, String> environment);
465
TestRunner testRunner = _runTests;
466
void restoreTestRunner() {
467
  testRunner = _runTests;
468 469
}

470
Future<void> _runTests(List<String> testArgs, Map<String, String> environment) async {
471
  globals.printTrace('Running driver tests.');
472

473
  globalPackagesPath = globals.fs.path.normalize(globals.fs.path.absolute(globalPackagesPath));
474
  final String dartVmPath = globals.fs.path.join(dartSdkPath, 'bin', 'dart');
475
  final int result = await processUtils.stream(
476 477 478 479
    <String>[
      dartVmPath,
      ...dartVmFlags,
      ...testArgs,
480
      '--packages=$globalPackagesPath',
481 482
      '-rexpanded',
    ],
483
    environment: environment,
484
  );
485
  if (result != 0) {
486
    throwToolExit('Driver tests failed: $result', exitCode: result);
487
  }
488 489 490 491
}


/// Stops the application.
492
typedef AppStopper = Future<bool> Function(DriveCommand command);
493
AppStopper appStopper = _stopApp;
494
void restoreAppStopper() {
495
  appStopper = _stopApp;
496 497
}

498
Future<bool> _stopApp(DriveCommand command) async {
499
  globals.printTrace('Stopping application.');
500
  final ApplicationPackage package = await command.applicationPackages.getPackageForPlatform(await command.device.targetPlatform);
501
  final bool stopped = await command.device.stopApp(package);
502
  await command._deviceLogSubscription?.cancel();
503
  return stopped;
504
}
505 506 507 508

/// A list of supported browsers
@visibleForTesting
enum Browser {
509 510
  /// Chrome on Android: https://developer.chrome.com/multidevice/android/overview
  androidChrome,
511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
  /// Chrome: https://www.google.com/chrome/
  chrome,
  /// Edge: https://www.microsoft.com/en-us/windows/microsoft-edge
  edge,
  /// Firefox: https://www.mozilla.org/en-US/firefox/
  firefox,
  /// Safari in iOS: https://www.apple.com/safari/
  iosSafari,
  /// Safari in macOS: https://www.apple.com/safari/
  safari,
}

/// Converts [browserName] string to [Browser]
Browser _browserNameToEnum(String browserName){
  switch (browserName) {
526
    case 'android-chrome': return Browser.androidChrome;
527 528 529 530 531 532 533 534 535
    case 'chrome': return Browser.chrome;
    case 'edge': return Browser.edge;
    case 'firefox': return Browser.firefox;
    case 'ios-safari': return Browser.iosSafari;
    case 'safari': return Browser.safari;
  }
  throw UnsupportedError('Browser $browserName not supported');
}

536
Future<async_io.WebDriver> _createDriver(String driverPort, Browser browser, bool headless, String chromeBinary) async {
537 538
  return async_io.createDriver(
      uri: Uri.parse('http://localhost:$driverPort/'),
539
      desired: getDesiredCapabilities(browser, headless, chromeBinary),
540
      spec: async_io.WebDriverSpec.Auto
541 542 543
  );
}

544 545
/// Returns desired capabilities for given [browser], [headless] and
/// [chromeBinary].
546
@visibleForTesting
547
Map<String, dynamic> getDesiredCapabilities(Browser browser, bool headless, [String chromeBinary]) {
548 549 550 551 552
  switch (browser) {
    case Browser.chrome:
      return <String, dynamic>{
        'acceptInsecureCerts': true,
        'browserName': 'chrome',
553
        'goog:loggingPrefs': <String, String>{ async_io.LogType.performance: 'ALL'},
554
        'chromeOptions': <String, dynamic>{
555 556
          if (chromeBinary != null)
            'binary': chromeBinary,
557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575
          'w3c': false,
          'args': <String>[
            '--bwsi',
            '--disable-background-timer-throttling',
            '--disable-default-apps',
            '--disable-extensions',
            '--disable-popup-blocking',
            '--disable-translate',
            '--no-default-browser-check',
            '--no-sandbox',
            '--no-first-run',
            if (headless) '--headless'
          ],
          'perfLoggingPrefs': <String, String>{
            'traceCategories':
            'devtools.timeline,'
                'v8,blink.console,benchmark,blink,'
                'blink.user_timing'
          }
576
        },
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
      };
      break;
    case Browser.firefox:
      return <String, dynamic>{
        'acceptInsecureCerts': true,
        'browserName': 'firefox',
        'moz:firefoxOptions' : <String, dynamic>{
          'args': <String>[
            if (headless) '-headless'
          ],
          'prefs': <String, dynamic>{
            'dom.file.createInChild': true,
            'dom.timeout.background_throttling_max_budget': -1,
            'media.autoplay.default': 0,
            'media.gmp-manager.url': '',
            'media.gmp-provider.enabled': false,
            'network.captive-portal-service.enabled': false,
            'security.insecure_field_warning.contextual.enabled': false,
            'test.currentTimeOffsetSeconds': 11491200
          },
          'log': <String, String>{'level': 'trace'}
        }
      };
      break;
    case Browser.edge:
      return <String, dynamic>{
        'acceptInsecureCerts': true,
        'browserName': 'edge',
      };
      break;
    case Browser.safari:
      return <String, dynamic>{
        'browserName': 'safari',
      };
      break;
    case Browser.iosSafari:
      return <String, dynamic>{
        'platformName': 'ios',
        'browserName': 'safari',
        'safari:useSimulator': true
      };
618 619 620 621 622 623 624 625 626
    case Browser.androidChrome:
      return <String, dynamic>{
        'browserName': 'chrome',
        'platformName': 'android',
        'goog:chromeOptions': <String, dynamic>{
          'androidPackage': 'com.android.chrome',
          'args': <String>['--disable-fullscreen']
        },
      };
627 628 629 630
    default:
      throw UnsupportedError('Browser $browser not supported.');
  }
}