// Copyright 2014 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:async'; import 'package:args/args.dart'; import 'package:meta/meta.dart'; import 'package:package_config/package_config_types.dart'; import '../android/android_device.dart'; import '../application_package.dart'; import '../artifacts.dart'; import '../base/common.dart'; import '../base/file_system.dart'; import '../base/io.dart'; import '../base/logger.dart'; import '../base/platform.dart'; import '../base/signals.dart'; import '../build_info.dart'; import '../dart/package_map.dart'; import '../device.dart'; import '../drive/drive_service.dart'; import '../globals.dart' as globals; import '../ios/devices.dart'; import '../ios/iproxy.dart'; import '../resident_runner.dart'; import '../runner/flutter_command.dart' show FlutterCommandCategory, FlutterCommandResult, FlutterOptions; import '../web/web_device.dart'; import 'run.dart'; /// Runs integration (a.k.a. end-to-end) tests. /// /// An integration test is a program that runs in a separate process from your /// Flutter application. It connects to the application and acts like a user, /// performing taps, scrolls, reading out widget properties and verifying their /// correctness. /// /// This command takes a target Flutter application that you would like to test /// as the `--target` option (defaults to `lib/main.dart`). It then looks for a /// corresponding test file within the `test_driver` directory. The test file is /// expected to have the same name but contain the `_test.dart` suffix. The /// `_test.dart` file would generally be a Dart program that uses /// `package:flutter_driver` and exercises your application. Most commonly it /// is a test written using `package:test`, but you are free to use something /// else. /// /// The app and the test are launched simultaneously. Once the test completes /// the application is stopped and the command exits. If all these steps are /// successful the exit code will be `0`. Otherwise, you will see a non-zero /// exit code. class DriveCommand extends RunCommandBase { DriveCommand({ bool verboseHelp = false, @visibleForTesting FlutterDriverFactory? flutterDriverFactory, @visibleForTesting this.signalsToHandle = const <ProcessSignal>{ProcessSignal.sigint, ProcessSignal.sigterm}, required FileSystem fileSystem, required Logger logger, required Platform platform, required this.signals, }) : _flutterDriverFactory = flutterDriverFactory, _fileSystem = fileSystem, _logger = logger, _fsUtils = FileSystemUtils(fileSystem: fileSystem, platform: platform), super(verboseHelp: verboseHelp) { requiresPubspecYaml(); addEnableExperimentation(hide: !verboseHelp); // By default, the drive app should not publish the VM service port over mDNS // to prevent a local network permission dialog on iOS 14+, // which cannot be accepted or dismissed in a CI environment. addPublishPort(enabledByDefault: false, verboseHelp: verboseHelp); addMultidexOption(); argParser ..addFlag('keep-app-running', defaultsTo: null, help: 'Will keep the Flutter application running when done testing.\n' 'By default, "flutter drive" stops the application after tests are finished, ' 'and "--keep-app-running" overrides this. On the other hand, if "--use-existing-app" ' 'is specified, then "flutter drive" instead defaults to leaving the application ' 'running, and "--no-keep-app-running" overrides it.', ) ..addOption('use-existing-app', help: 'Connect to an already running instance via the given Dart VM Service URL. ' 'If this option is given, the application will not be automatically started, ' 'and it will only be stopped if "--no-keep-app-running" is explicitly set.', valueHelp: 'url', ) ..addOption('driver', help: 'The test file to run on the host (as opposed to the target file to run on ' 'the device).\n' 'By default, this file has the same base name as the target file, but in the ' '"test_driver/" directory instead, and with "_test" inserted just before the ' 'extension, so e.g. if the target is "lib/main.dart", the driver will be ' '"test_driver/main_test.dart".', valueHelp: 'path', ) ..addFlag('build', defaultsTo: true, help: '(deprecated) Build the app before running. To use an existing app, pass the "--${FlutterOptions.kUseApplicationBinary}" ' 'flag with an existing APK.', ) ..addOption('screenshot', valueHelp: 'path/to/directory', help: 'Directory location to write screenshots on test failure.', ) ..addOption('driver-port', defaultsTo: '4444', help: 'The port where Webdriver server is launched at.', valueHelp: '4444' ) ..addFlag('headless', defaultsTo: true, help: 'Whether the driver browser is going to be launched in headless mode.', ) ..addOption('browser-name', defaultsTo: 'chrome', help: 'Name of the browser where tests will be executed.', allowed: <String>[ 'android-chrome', 'chrome', 'edge', 'firefox', 'ios-safari', 'safari', ], allowedHelp: <String, String>{ 'android-chrome': 'Chrome on Android (see also "--android-emulator").', 'chrome': 'Google Chrome on this computer (see also "--chrome-binary").', 'edge': 'Microsoft Edge on this computer (Windows only).', 'firefox': 'Mozilla Firefox on this computer.', 'ios-safari': 'Apple Safari on an iOS device.', 'safari': 'Apple Safari on this computer (macOS only).', }, ) ..addOption('browser-dimension', defaultsTo: '1600,1024', help: 'The dimension of the browser when running a Flutter Web test. ' 'This will affect screenshot and all offset-related actions.', valueHelp: 'width,height', ) ..addFlag('android-emulator', defaultsTo: true, help: 'Whether to perform Flutter Driver testing using an Android Emulator. ' 'Works only if "browser-name" is set to "android-chrome".') ..addOption('chrome-binary', help: 'Location of the Chrome binary. ' 'Works only if "browser-name" is set to "chrome".') ..addOption('write-sksl-on-exit', help: 'Attempts to write an SkSL file when the drive process is finished ' 'to the provided file, overwriting it if necessary.') ..addMultiOption('test-arguments', help: 'Additional arguments to pass to the ' 'Dart VM running The test script.') ..addOption('profile-memory', help: 'Launch devtools and profile application memory, writing ' 'The output data to the file path provided to this argument as JSON.', valueHelp: 'profile_memory.json') ..addOption('timeout', help: 'Timeout the test after the given number of seconds. If the ' '"--screenshot" option is provided, a screenshot will be taken ' 'before exiting. Defaults to no timeout.', valueHelp: '360'); } final Signals signals; /// The [ProcessSignal]s that will lead to a screenshot being taken (if the option is provided). final Set<ProcessSignal> signalsToHandle; // `pub` must always be run due to the test script running from source, // even if an application binary is used. Default to true unless the user explicitly // specified not to. @override bool get shouldRunPub { if (argResults!.wasParsed('pub') && !boolArgDeprecated('pub')) { return false; } return true; } FlutterDriverFactory? _flutterDriverFactory; final FileSystem _fileSystem; final Logger _logger; final FileSystemUtils _fsUtils; Timer? timeoutTimer; Map<ProcessSignal, Object>? screenshotTokens; @override final String name = 'drive'; @override final String description = 'Run integration tests for the project on an attached device or emulator.'; @override String get category => FlutterCommandCategory.project; @override final List<String> aliases = <String>['driver']; String? get userIdentifier => stringArgDeprecated(FlutterOptions.kDeviceUser); String? get screenshot => stringArgDeprecated('screenshot'); @override bool get startPausedDefault => true; @override bool get cachePubGet => false; String? get applicationBinaryPath => stringArgDeprecated(FlutterOptions.kUseApplicationBinary); Future<Device?> get targetedDevice async { return findTargetDevice(includeUnsupportedDevices: applicationBinaryPath == null); } // Network devices need `publish-port` to be enabled because it requires mDNS. // If the flag wasn't provided as an actual argument and it's a network device, // change it to be enabled. @override Future<bool> get disablePortPublication async { final ArgResults? localArgResults = argResults; final Device? device = await targetedDevice; final bool isNetworkDevice = device is IOSDevice && device.interfaceType == IOSDeviceConnectionInterface.network; if (isNetworkDevice && localArgResults != null && !localArgResults.wasParsed('publish-port')) { _logger.printTrace('Network device is being used. Changing `publish-port` to be enabled.'); return false; } return !boolArgDeprecated('publish-port'); } @override Future<void> validateCommand() async { if (userIdentifier != null) { final Device? device = await findTargetDevice(); if (device is! AndroidDevice) { throwToolExit('--${FlutterOptions.kDeviceUser} is only supported for Android'); } } return super.validateCommand(); } @override Future<FlutterCommandResult> runCommand() async { final String? testFile = _getTestFile(); if (testFile == null) { throwToolExit(null); } if (await _fileSystem.type(testFile) != FileSystemEntityType.file) { throwToolExit('Test file not found: $testFile'); } final Device? device = await targetedDevice; if (device == null) { throwToolExit(null); } if (screenshot != null && !device.supportsScreenshot) { _logger.printError('Screenshot not supported for ${device.name}.'); } final bool web = device is WebServerDevice || device is ChromiumDevice; _flutterDriverFactory ??= FlutterDriverFactory( applicationPackageFactory: ApplicationPackageFactory.instance!, logger: _logger, processUtils: globals.processUtils, dartSdkPath: globals.artifacts!.getArtifactPath(Artifact.engineDartBinary), devtoolsLauncher: DevtoolsLauncher.instance!, ); final PackageConfig packageConfig = await loadPackageConfigWithLogging( _fileSystem.file('.packages'), logger: _logger, throwOnError: false, ); final DriverService driverService = _flutterDriverFactory!.createDriverService(web); final BuildInfo buildInfo = await getBuildInfo(); final DebuggingOptions debuggingOptions = await createDebuggingOptions(web); final File? applicationBinary = applicationBinaryPath == null ? null : _fileSystem.file(applicationBinaryPath); bool screenshotTaken = false; try { if (stringArgDeprecated('use-existing-app') == null) { await driverService.start( buildInfo, device, debuggingOptions, ipv6 ?? false, applicationBinary: applicationBinary, route: route, userIdentifier: userIdentifier, mainPath: targetFile, platformArgs: <String, Object>{ if (traceStartup) 'trace-startup': traceStartup, if (web) '--no-launch-chrome': true, if (boolArgDeprecated('multidex')) 'multidex': true, } ); } else { final Uri? uri = Uri.tryParse(stringArgDeprecated('use-existing-app')!); if (uri == null) { throwToolExit('Invalid VM Service URI: ${stringArgDeprecated('use-existing-app')}'); } await driverService.reuseApplication( uri, device, debuggingOptions, ipv6 ?? false, ); } final Future<int> testResultFuture = driverService.startTest( testFile, stringsArg('test-arguments'), <String, String>{}, packageConfig, chromeBinary: stringArgDeprecated('chrome-binary'), headless: boolArgDeprecated('headless'), webBrowserFlags: stringsArg(FlutterOptions.kWebBrowserFlag), browserDimension: stringArgDeprecated('browser-dimension')!.split(','), browserName: stringArgDeprecated('browser-name'), driverPort: stringArgDeprecated('driver-port') != null ? int.tryParse(stringArgDeprecated('driver-port')!) : null, androidEmulator: boolArgDeprecated('android-emulator'), profileMemory: stringArgDeprecated('profile-memory'), ); // If the test is sent a signal or times out, take a screenshot _registerScreenshotCallbacks(device); final int testResult = await testResultFuture; if (timeoutTimer != null) { timeoutTimer!.cancel(); } _unregisterScreenshotCallbacks(); if (testResult != 0 && screenshot != null) { // Take a screenshot while the app is still running. await _takeScreenshot(device); screenshotTaken = true; } if (boolArgDeprecated('keep-app-running')) { _logger.printStatus('Leaving the application running.'); } else { final File? skslFile = stringArgDeprecated('write-sksl-on-exit') != null ? _fileSystem.file(stringArgDeprecated('write-sksl-on-exit')) : null; await driverService.stop(userIdentifier: userIdentifier, writeSkslOnExit: skslFile); } if (testResult != 0) { throwToolExit(null); } } on Exception catch(_) { // On exceptions, including ToolExit, take a screenshot on the device // unless a screenshot was already taken on test failure. if (!screenshotTaken && screenshot != null) { await _takeScreenshot(device); } rethrow; } return FlutterCommandResult.success(); } int? get _timeoutSeconds { final String? timeoutString = stringArg('timeout'); if (timeoutString == null) { return null; } final int? timeoutSeconds = int.tryParse(timeoutString); if (timeoutSeconds == null || timeoutSeconds <= 0) { throwToolExit( 'Invalid value "$timeoutString" provided to the option --timeout: ' 'expected a positive integer representing seconds.', ); } return timeoutSeconds; } void _registerScreenshotCallbacks(Device device) { _logger.printTrace('Registering signal handlers...'); final Map<ProcessSignal, Object> tokens = <ProcessSignal, Object>{}; for (final ProcessSignal signal in signalsToHandle) { tokens[signal] = signals.addHandler( signal, (ProcessSignal signal) { _unregisterScreenshotCallbacks(); _logger.printError('Caught $signal'); return _takeScreenshot(device); }, ); } screenshotTokens = tokens; final int? timeoutSeconds = _timeoutSeconds; if (timeoutSeconds != null) { timeoutTimer = Timer( Duration(seconds: timeoutSeconds), () { _unregisterScreenshotCallbacks(); _takeScreenshot(device); throwToolExit('Timed out after $timeoutSeconds seconds'); } ); } } void _unregisterScreenshotCallbacks() { if (screenshotTokens != null) { _logger.printTrace('Unregistering signal handlers...'); for (final MapEntry<ProcessSignal, Object> entry in screenshotTokens!.entries) { signals.removeHandler(entry.key, entry.value); } } timeoutTimer?.cancel(); } String? _getTestFile() { if (argResults!['driver'] != null) { return stringArgDeprecated('driver'); } // If the --driver argument wasn't provided, then derive the value from // the target file. String appFile = _fileSystem.path.normalize(targetFile); // This command extends `flutter run` and therefore CWD == package dir final String packageDir = _fileSystem.currentDirectory.path; // Make appFile path relative to package directory because we are looking // for the corresponding test file relative to it. if (!_fileSystem.path.isRelative(appFile)) { if (!_fileSystem.path.isWithin(packageDir, appFile)) { _logger.printError( 'Application file $appFile is outside the package directory $packageDir' ); return null; } appFile = _fileSystem.path.relative(appFile, from: packageDir); } final List<String> parts = _fileSystem.path.split(appFile); if (parts.length < 2) { _logger.printError( 'Application file $appFile must reside in one of the sub-directories ' 'of the package structure, not in the root directory.' ); return null; } // Look for the test file inside `test_driver/` matching the sub-path, e.g. // if the application is `lib/foo/bar.dart`, the test file is expected to // be `test_driver/foo/bar_test.dart`. final String pathWithNoExtension = _fileSystem.path.withoutExtension(_fileSystem.path.joinAll( <String>[packageDir, 'test_driver', ...parts.skip(1)])); return '${pathWithNoExtension}_test${_fileSystem.path.extension(appFile)}'; } Future<void> _takeScreenshot(Device device) async { if (!device.supportsScreenshot) { return; } try { final Directory outputDirectory = _fileSystem.directory(screenshot) ..createSync(recursive: true); final File outputFile = _fsUtils.getUniqueFile( outputDirectory, 'drive', 'png', ); await device.takeScreenshot(outputFile); _logger.printStatus('Screenshot written to ${outputFile.path}'); } on Exception catch (error) { _logger.printError('Error taking screenshot: $error'); } } }