test_test.dart 19.6 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5 6 7 8 9
// TODO(gspencergoog): Remove this tag once this test's state leaks/test
// dependencies have been fixed.
// https://github.com/flutter/flutter/issues/85160
// Fails with "flutter test --test-randomize-ordering-seed=1000"
@Tags(<String>['no-shuffle'])
10
library;
11

12
import 'dart:async';
13
import 'dart:convert';
14

15
import 'package:flutter_tools/src/base/file_system.dart';
16
import 'package:flutter_tools/src/base/io.dart';
17

18 19
import '../src/common.dart';
import 'test_utils.dart';
20 21 22

// This test depends on some files in ///dev/automated_tests/flutter_test/*

23 24 25
final String automatedTestsDirectory = fileSystem.path.join('..', '..', 'dev', 'automated_tests');
final String missingDependencyDirectory = fileSystem.path.join('..', '..', 'dev', 'missing_dependency_tests');
final String flutterTestDirectory = fileSystem.path.join(automatedTestsDirectory, 'flutter_test');
26
final String integrationTestDirectory = fileSystem.path.join(automatedTestsDirectory, 'integration_test');
27
final String flutterBin = fileSystem.path.join(getFlutterRoot(), 'bin', platform.isWindows ? 'flutter.bat' : 'flutter');
28

29 30 31 32
// Running Integration Tests in the Flutter Tester will still exercise the same
// flows specific to Integration Tests.
final List<String> integrationTestExtraArgs = <String>['-d', 'flutter-tester'];

33
void main() {
34
  setUpAll(() async {
35 36 37 38 39 40 41 42 43 44
    expect(
      await processManager.run(
        <String>[
          flutterBin,
          'pub',
          'get',
        ],
        workingDirectory: flutterTestDirectory
      ),
      const ProcessResultMatcher(),
45
    );
46 47 48 49 50 51 52 53 54 55
    expect(
      await processManager.run(
        <String>[
          flutterBin,
          'pub',
          'get',
        ],
        workingDirectory: missingDependencyDirectory
      ),
      const ProcessResultMatcher(),
56 57
    );
  });
58

59
  testWithoutContext('flutter test should not have extraneous error messages', () async {
60 61
    return _testFile('trivial_widget', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
  });
62

63 64 65 66
  testWithoutContext('integration test should not have extraneous error messages', () async {
    return _testFile('trivial_widget', automatedTestsDirectory, integrationTestDirectory, exitCode: isZero, extraArguments: integrationTestExtraArgs);
  });

67 68 69 70
  testWithoutContext('flutter test set the working directory correctly', () async {
    return _testFile('working_directory', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
  });

71
  testWithoutContext('flutter test should report nice errors for exceptions thrown within testWidgets()', () async {
72 73
    return _testFile('exception_handling', automatedTestsDirectory, flutterTestDirectory);
  });
74

75 76 77 78
  testWithoutContext('integration test should report nice errors for exceptions thrown within testWidgets()', () async {
    return _testFile('exception_handling', automatedTestsDirectory, integrationTestDirectory, extraArguments: integrationTestExtraArgs);
  });

79
  testWithoutContext('flutter test should report a nice error when a guarded function was called without await', () async {
80 81
    return _testFile('test_async_utils_guarded', automatedTestsDirectory, flutterTestDirectory);
  });
82

83
  testWithoutContext('flutter test should report a nice error when an async function was called without await', () async {
84 85
    return _testFile('test_async_utils_unguarded', automatedTestsDirectory, flutterTestDirectory);
  });
86

87
  testWithoutContext('flutter test should report a nice error when a Ticker is left running', () async {
88 89
    return _testFile('ticker', automatedTestsDirectory, flutterTestDirectory);
  });
90

91 92
  testWithoutContext('flutter test should report a nice error when a pubspec.yaml is missing a flutter_test dependency', () async {
    final String missingDependencyTests = fileSystem.path.join('..', '..', 'dev', 'missing_dependency_tests');
93 94
    return _testFile('trivial', missingDependencyTests, missingDependencyTests);
  });
95

96
  testWithoutContext('flutter test should report which user-created widget caused the error', () async {
97
    return _testFile('print_user_created_ancestor', automatedTestsDirectory, flutterTestDirectory,
98
        extraArguments: const <String>['--track-widget-creation']);
99
  });
100

101
  testWithoutContext('flutter test should report which user-created widget caused the error - no flag', () async {
102 103 104
    return _testFile('print_user_created_ancestor_no_flag', automatedTestsDirectory, flutterTestDirectory,
       extraArguments: const <String>['--no-track-widget-creation']);
  });
105

106
  testWithoutContext('flutter test should report the correct user-created widget that caused the error', () async {
107 108 109
    return _testFile('print_correct_local_widget', automatedTestsDirectory, flutterTestDirectory,
      extraArguments: const <String>['--track-widget-creation']);
  });
110

111
  testWithoutContext('flutter test should can load assets within its own package', () async {
112 113
    return _testFile('package_assets', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
  });
114

115 116 117 118 119
  testWithoutContext('flutter test should support dart defines', () async {
    return _testFile('dart_defines', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero,
      extraArguments: <String>['--dart-define=flutter.test.foo=bar']);
  });

120
  testWithoutContext('flutter test should run a test when its name matches a regexp', () async {
121 122 123 124 125 126 127 128 129 130
    final ProcessResult result = await _runFlutterTest(
      'filtering',
      automatedTestsDirectory,
      flutterTestDirectory,
      extraArguments: const <String>['--name', 'inc.*de'],
    );
    expect(
      result,
      ProcessResultMatcher(stdoutPattern: RegExp(r'\+\d+: All tests passed!')),
    );
131
  });
132

133
  testWithoutContext('flutter test should run a test when its name contains a string', () async {
134 135 136 137 138 139 140 141 142 143
    final ProcessResult result = await _runFlutterTest(
      'filtering',
      automatedTestsDirectory,
      flutterTestDirectory,
      extraArguments: const <String>['--plain-name', 'include'],
    );
    expect(
      result,
      ProcessResultMatcher(stdoutPattern: RegExp(r'\+\d+: All tests passed!')),
    );
144
  });
145

146
  testWithoutContext('flutter test should run a test with a given tag', () async {
147 148 149 150 151 152 153 154 155 156
    final ProcessResult result = await _runFlutterTest(
      'filtering_tag',
      automatedTestsDirectory,
      flutterTestDirectory,
      extraArguments: const <String>['--tags', 'include-tag'],
    );
    expect(
      result,
      ProcessResultMatcher(stdoutPattern: RegExp(r'\+\d+: All tests passed!')),
    );
157 158
  });

159
  testWithoutContext('flutter test should not run a test with excluded tag', () async {
160 161
    final ProcessResult result = await _runFlutterTest('filtering_tag', automatedTestsDirectory, flutterTestDirectory,
        extraArguments: const <String>['--exclude-tags', 'exclude-tag']);
162 163 164 165
    expect(
      result,
      ProcessResultMatcher(stdoutPattern: RegExp(r'\+\d+: All tests passed!')),
    );
166 167
  });

168
  testWithoutContext('flutter test should run all tests when tags are unspecified', () async {
169
    final ProcessResult result = await _runFlutterTest('filtering_tag', automatedTestsDirectory, flutterTestDirectory);
170 171 172 173 174 175 176
    expect(
      result,
      ProcessResultMatcher(
        exitCode: 1,
        stdoutPattern: RegExp(r'\+\d+ -1: Some tests failed\.'),
      ),
    );
177 178
  });

179
  testWithoutContext('flutter test should run a widgetTest with a given tag', () async {
180 181
    final ProcessResult result = await _runFlutterTest('filtering_tag_widget', automatedTestsDirectory, flutterTestDirectory,
        extraArguments: const <String>['--tags', 'include-tag']);
182 183 184 185
    expect(
      result,
      ProcessResultMatcher(stdoutPattern: RegExp(r'\+\d+: All tests passed!')),
    );
186 187
  });

188
  testWithoutContext('flutter test should not run a widgetTest with excluded tag', () async {
189 190
    final ProcessResult result = await _runFlutterTest('filtering_tag_widget', automatedTestsDirectory, flutterTestDirectory,
        extraArguments: const <String>['--exclude-tags', 'exclude-tag']);
191 192 193 194
    expect(
      result,
      ProcessResultMatcher(stdoutPattern: RegExp(r'\+\d+: All tests passed!')),
    );
195 196
  });

197
  testWithoutContext('flutter test should run all widgetTest when tags are unspecified', () async {
198
    final ProcessResult result = await _runFlutterTest('filtering_tag_widget', automatedTestsDirectory, flutterTestDirectory);
199 200 201 202 203 204 205
    expect(
      result,
      ProcessResultMatcher(
        exitCode: 1,
        stdoutPattern: RegExp(r'\+\d+ -1: Some tests failed\.'),
      ),
    );
206 207
  });

208 209 210
  testWithoutContext('flutter test should run a test with an exact name in URI format', () async {
    final ProcessResult result = await _runFlutterTest('uri_format', automatedTestsDirectory, flutterTestDirectory,
      query: 'full-name=exactTestName');
211 212 213 214
    expect(
      result,
      ProcessResultMatcher(stdoutPattern: RegExp(r'\+\d+: All tests passed!')),
    );
215 216 217 218 219
  });

  testWithoutContext('flutter test should run a test by line number in URI format', () async {
    final ProcessResult result = await _runFlutterTest('uri_format', automatedTestsDirectory, flutterTestDirectory,
      query: 'line=11');
220 221 222 223
    expect(
      result,
      ProcessResultMatcher(stdoutPattern: RegExp(r'\+\d+: All tests passed!')),
    );
224 225
  });

226
  testWithoutContext('flutter test should test runs to completion', () async {
227 228
    final ProcessResult result = await _runFlutterTest('trivial', automatedTestsDirectory, flutterTestDirectory,
      extraArguments: const <String>['--verbose']);
229 230 231 232 233 234
    final String stdout = (result.stdout as String).replaceAll('\r', '\n');
    expect(stdout, contains(RegExp(r'\+\d+: All tests passed\!')));
    expect(stdout, contains('test 0: Starting flutter_tester process with command'));
    expect(stdout, contains('test 0: deleting temporary directory'));
    expect(stdout, contains('test 0: finished'));
    expect(stdout, contains('test package returned with exit code 0'));
235 236 237
    if ((result.stderr as String).isNotEmpty) {
      fail('unexpected error output from test:\n\n${result.stderr}\n-- end stderr --\n\n');
    }
238
    expect(result, const ProcessResultMatcher());
239 240
  });

241
  testWithoutContext('flutter test should run all tests inside of a directory with no trailing slash', () async {
242
    final ProcessResult result = await _runFlutterTest(null, automatedTestsDirectory, '$flutterTestDirectory/child_directory',
243
      extraArguments: const <String>['--verbose']);
244 245 246 247 248 249
    final String stdout = (result.stdout as String).replaceAll('\r', '\n');
    expect(result.stdout, contains(RegExp(r'\+\d+: All tests passed\!')));
    expect(stdout, contains('test 0: Starting flutter_tester process with command'));
    expect(stdout, contains('test 0: deleting temporary directory'));
    expect(stdout, contains('test 0: finished'));
    expect(stdout, contains('test package returned with exit code 0'));
250 251 252
    if ((result.stderr as String).isNotEmpty) {
      fail('unexpected error output from test:\n\n${result.stderr}\n-- end stderr --\n\n');
    }
253
    expect(result, const ProcessResultMatcher());
254
  });
255 256 257 258

  testWithoutContext('flutter gold skips tests where the expectations are missing', () async {
    return _testFile('flutter_gold', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
  });
259 260

  testWithoutContext('flutter test should respect --serve-observatory', () async {
261 262
    Process? process;
    StreamSubscription<String>? sub;
263 264 265 266 267 268
    try {
      process = await _runFlutterTestConcurrent('trivial', automatedTestsDirectory, flutterTestDirectory,
        extraArguments: const <String>['--start-paused', '--serve-observatory']);
      final Completer<Uri> completer = Completer<Uri>();
      final RegExp vmServiceUriRegExp = RegExp(r'((http)?:\/\/)[^\s]+');
      sub = process.stdout.transform(utf8.decoder).listen((String e) {
269
        if (!completer.isCompleted && vmServiceUriRegExp.hasMatch(e)) {
270 271 272 273 274 275 276 277
          completer.complete(Uri.parse(vmServiceUriRegExp.firstMatch(e)!.group(0)!));
        }
      });
      final Uri vmServiceUri = await completer.future;
      final HttpClient client = HttpClient();
      final HttpClientRequest request = await client.getUrl(vmServiceUri);
      final HttpClientResponse response = await request.close();
      final String content = await response.transform(utf8.decoder).join();
278
      expect(content, contains('Dart VM Observatory'));
279
    } finally {
280 281
      await sub?.cancel();
      process?.kill();
282 283
    }
  });
284 285

  testWithoutContext('flutter test should serve DevTools', () async {
286 287
    Process? process;
    StreamSubscription<String>? sub;
288 289 290 291 292 293 294 295 296 297 298 299 300 301 302
    try {
      process = await _runFlutterTestConcurrent('trivial', automatedTestsDirectory, flutterTestDirectory,
        extraArguments: const <String>['--start-paused']);
      final Completer<Uri> completer = Completer<Uri>();
      final RegExp devToolsUriRegExp = RegExp(r'The Flutter DevTools debugger and profiler is available at: (http://[^\s]+)');
      sub = process.stdout.transform(utf8.decoder).listen((String e) {
        if (!completer.isCompleted && devToolsUriRegExp.hasMatch(e)) {
          completer.complete(Uri.parse(devToolsUriRegExp.firstMatch(e)!.group(1)!));
        }
      });
      final Uri devToolsUri = await completer.future;
      final HttpClient client = HttpClient();
      final HttpClientRequest request = await client.getUrl(devToolsUri);
      final HttpClientResponse response = await request.close();
      final String content = await response.transform(utf8.decoder).join();
303
      expect(content, contains('DevTools'));
304
    } finally {
305 306
      await sub?.cancel();
      process?.kill();
307 308
    }
  });
309 310
}

311
Future<void> _testFile(
312 313 314
  String testName,
  String workingDirectory,
  String testDirectory, {
315
  Matcher? exitCode,
316 317
  List<String> extraArguments = const <String>[],
}) async {
318
  exitCode ??= isNonZero;
319 320
  final String fullTestExpectation = fileSystem.path.join(testDirectory, '${testName}_expectation.txt');
  final File expectationFile = fileSystem.file(fullTestExpectation);
321
  if (!expectationFile.existsSync()) {
322
    fail('missing expectation file: $expectationFile');
323
  }
324

325 326 327 328 329 330
  final ProcessResult exec = await _runFlutterTest(
    testName,
    workingDirectory,
    testDirectory,
    extraArguments: extraArguments,
  );
331

332 333 334 335 336 337
  expect(
    exec.exitCode,
    exitCode,
    reason: '"$testName" returned code ${exec.exitCode}\n\nstdout:\n'
            '${exec.stdout}\nstderr:\n${exec.stderr}',
  );
338 339 340 341
  List<String> output = (exec.stdout as String).split('\n');

  output = _removeMacFontServerWarning(output);

342
  if (output.first.startsWith('Waiting for another flutter command to release the startup lock...')) {
343
    output.removeAt(0);
344 345
  }
  if (output.first.startsWith('Running "flutter pub get" in')) {
346
    output.removeAt(0);
347
  }
348 349 350
  // Whether cached artifacts need to be downloaded is dependent on what
  // previous tests have run. Disregard these messages.
  output.removeWhere(RegExp(r'Downloading .*\.\.\.').hasMatch);
351
  output.add('<<stderr>>');
352
  output.addAll((exec.stderr as String).split('\n'));
353
  final List<String> expectations = fileSystem.file(fullTestExpectation).readAsLinesSync();
354 355 356
  bool allowSkip = false;
  int expectationLineNumber = 0;
  int outputLineNumber = 0;
357
  bool haveSeenStdErrMarker = false;
358
  while (expectationLineNumber < expectations.length) {
359 360 361 362 363
    expect(
      output,
      hasLength(greaterThan(outputLineNumber)),
      reason: 'Failure in $testName to compare to $fullTestExpectation',
    );
364
    final String expectationLine = expectations[expectationLineNumber];
365
    String outputLine = output[outputLineNumber];
366 367 368 369 370 371
    if (expectationLine == '<<skip until matching line>>') {
      allowSkip = true;
      expectationLineNumber += 1;
      continue;
    }
    if (allowSkip) {
372
      if (!RegExp(expectationLine).hasMatch(outputLine)) {
373 374 375 376 377
        outputLineNumber += 1;
        continue;
      }
      allowSkip = false;
    }
378 379 380 381
    if (expectationLine == '<<stderr>>') {
      expect(haveSeenStdErrMarker, isFalse);
      haveSeenStdErrMarker = true;
    }
382 383 384 385 386 387 388 389 390 391 392 393
    if (!RegExp(expectationLine).hasMatch(outputLine) && outputLineNumber + 1 < output.length) {
      // Check if the RegExp can match the next two lines in the output so
      // that it is possible to write expectations that still hold even if a
      // line is wrapped slightly differently due to for example a file name
      // being longer on one platform than another.
      final String mergedLines = '$outputLine\n${output[outputLineNumber+1]}';
      if (RegExp(expectationLine).hasMatch(mergedLines)) {
        outputLineNumber += 1;
        outputLine = mergedLines;
      }
    }

394
    expect(outputLine, matches(expectationLine), reason: 'Full output:\n- - - -----8<----- - - -\n${output.join("\n")}\n- - - -----8<----- - - -');
395 396 397 398
    expectationLineNumber += 1;
    outputLineNumber += 1;
  }
  expect(allowSkip, isFalse);
399
  if (!haveSeenStdErrMarker) {
400
    expect(exec.stderr, '');
401
  }
402
}
403

404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423
final RegExp _fontServerProtocolPattern = RegExp(r'flutter_tester.*Font server protocol version mismatch');
final RegExp _unableToConnectToFontDaemonPattern = RegExp(r'flutter_tester.*XType: unable to make a connection to the font daemon!');
final RegExp _xtFontStaticRegistryPattern = RegExp(r'flutter_tester.*XType: XTFontStaticRegistry is enabled as fontd is not available');

// https://github.com/flutter/flutter/issues/132990
List<String> _removeMacFontServerWarning(List<String> output) {
  return output.where((String line) {
    if (_fontServerProtocolPattern.hasMatch(line)) {
      return false;
    }
    if (_unableToConnectToFontDaemonPattern.hasMatch(line)) {
      return false;
    }
    if (_xtFontStaticRegistryPattern.hasMatch(line)) {
      return false;
    }
    return true;
  }).toList();
}

424
Future<ProcessResult> _runFlutterTest(
425
  String? testName,
426 427
  String workingDirectory,
  String testDirectory, {
428
  List<String> extraArguments = const <String>[],
429
  String? query,
430 431
}) async {

432 433 434 435
  String testPath;
  if (testName == null) {
    // Test everything in the directory.
    testPath = testDirectory;
436
    final Directory directoryToTest = fileSystem.directory(testPath);
437
    if (!directoryToTest.existsSync()) {
438
      fail('missing test directory: $directoryToTest');
439
    }
440 441
  } else {
    // Test just a specific test file.
442 443
    testPath = fileSystem.path.join(testDirectory, '${testName}_test.dart');
    final File testFile = fileSystem.file(testPath);
444
    if (!testFile.existsSync()) {
445
      fail('missing test file: $testFile');
446
    }
447
  }
448

449 450 451
  final List<String> args = <String>[
    'test',
    '--no-color',
452
    '--no-version-check',
453
    '--no-pub',
454 455
    '--reporter',
    'compact',
456
    ...extraArguments,
457 458
    if (query != null) Uri.file(testPath).replace(query: query).toString()
    else testPath,
459
  ];
460

461 462 463 464 465 466 467
  return Process.run(
    flutterBin, // Uses the precompiled flutter tool for faster tests,
    args,
    workingDirectory: workingDirectory,
    stdoutEncoding: utf8,
    stderrEncoding: utf8,
  );
468
}
469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510

Future<Process> _runFlutterTestConcurrent(
  String? testName,
  String workingDirectory,
  String testDirectory, {
  List<String> extraArguments = const <String>[],
}) async {

  String testPath;
  if (testName == null) {
    // Test everything in the directory.
    testPath = testDirectory;
    final Directory directoryToTest = fileSystem.directory(testPath);
    if (!directoryToTest.existsSync()) {
      fail('missing test directory: $directoryToTest');
    }
  } else {
    // Test just a specific test file.
    testPath = fileSystem.path.join(testDirectory, '${testName}_test.dart');
    final File testFile = fileSystem.file(testPath);
    if (!testFile.existsSync()) {
      fail('missing test file: $testFile');
    }
  }

  final List<String> args = <String>[
    'test',
    '--no-color',
    '--no-version-check',
    '--no-pub',
    '--reporter',
    'compact',
    ...extraArguments,
    testPath,
  ];

  return Process.start(
    flutterBin, // Uses the precompiled flutter tool for faster tests,
    args,
    workingDirectory: workingDirectory,
  );
}