test_test.dart 13.5 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5 6
// @dart = 2.8

7
import 'dart:async';
8
import 'dart:convert';
9

10
import 'package:flutter_tools/src/base/file_system.dart';
11
import 'package:flutter_tools/src/base/io.dart';
12

13 14
import '../src/common.dart';
import 'test_utils.dart';
15 16 17

// This test depends on some files in ///dev/automated_tests/flutter_test/*

18 19 20 21
final String automatedTestsDirectory = fileSystem.path.join('..', '..', 'dev', 'automated_tests');
final String missingDependencyDirectory = fileSystem.path.join('..', '..', 'dev', 'missing_dependency_tests');
final String flutterTestDirectory = fileSystem.path.join(automatedTestsDirectory, 'flutter_test');
final String flutterBin = fileSystem.path.join(getFlutterRoot(), 'bin', platform.isWindows ? 'flutter.bat' : 'flutter');
22

23
void main() {
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
  setUpAll(() async {
    await processManager.run(
      <String>[
        flutterBin,
        'pub',
        'get'
      ],
      workingDirectory: flutterTestDirectory
    );
    await processManager.run(
      <String>[
        flutterBin,
        'pub',
        'get'
      ],
      workingDirectory: missingDependencyDirectory
    );
  });
42

43
  testWithoutContext('flutter test should not have extraneous error messages', () async {
44 45
    return _testFile('trivial_widget', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
  });
46

47 48 49 50
  testWithoutContext('flutter test set the working directory correctly', () async {
    return _testFile('working_directory', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
  });

51
  testWithoutContext('flutter test should report nice errors for exceptions thrown within testWidgets()', () async {
52 53
    return _testFile('exception_handling', automatedTestsDirectory, flutterTestDirectory);
  });
54

55
  testWithoutContext('flutter test should report a nice error when a guarded function was called without await', () async {
56 57
    return _testFile('test_async_utils_guarded', automatedTestsDirectory, flutterTestDirectory);
  });
58

59
  testWithoutContext('flutter test should report a nice error when an async function was called without await', () async {
60 61
    return _testFile('test_async_utils_unguarded', automatedTestsDirectory, flutterTestDirectory);
  });
62

63
  testWithoutContext('flutter test should report a nice error when a Ticker is left running', () async {
64 65
    return _testFile('ticker', automatedTestsDirectory, flutterTestDirectory);
  });
66

67 68
  testWithoutContext('flutter test should report a nice error when a pubspec.yaml is missing a flutter_test dependency', () async {
    final String missingDependencyTests = fileSystem.path.join('..', '..', 'dev', 'missing_dependency_tests');
69 70
    return _testFile('trivial', missingDependencyTests, missingDependencyTests);
  });
71

72
  testWithoutContext('flutter test should report which user-created widget caused the error', () async {
73
    return _testFile('print_user_created_ancestor', automatedTestsDirectory, flutterTestDirectory,
74
        extraArguments: const <String>['--track-widget-creation']);
75
  });
76

77
  testWithoutContext('flutter test should report which user-created widget caused the error - no flag', () async {
78 79 80
    return _testFile('print_user_created_ancestor_no_flag', automatedTestsDirectory, flutterTestDirectory,
       extraArguments: const <String>['--no-track-widget-creation']);
  });
81

82
  testWithoutContext('flutter test should report the correct user-created widget that caused the error', () async {
83 84 85
    return _testFile('print_correct_local_widget', automatedTestsDirectory, flutterTestDirectory,
      extraArguments: const <String>['--track-widget-creation']);
  });
86

87
  testWithoutContext('flutter test should can load assets within its own package', () async {
88 89
    return _testFile('package_assets', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
  });
90

91 92 93 94 95
  testWithoutContext('flutter test should support dart defines', () async {
    return _testFile('dart_defines', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero,
      extraArguments: <String>['--dart-define=flutter.test.foo=bar']);
  });

96
  testWithoutContext('flutter test should run a test when its name matches a regexp', () async {
97 98 99 100 101 102 103
    final ProcessResult result = await _runFlutterTest('filtering', automatedTestsDirectory, flutterTestDirectory,
      extraArguments: const <String>['--name', 'inc.*de']);
    if (!(result.stdout as String).contains('+1: All tests passed')) {
      fail('unexpected output from test:\n\n${result.stdout}\n-- end stdout --\n\n');
    }
    expect(result.exitCode, 0);
  });
104

105
  testWithoutContext('flutter test should run a test when its name contains a string', () async {
106 107 108 109 110 111 112
    final ProcessResult result = await _runFlutterTest('filtering', automatedTestsDirectory, flutterTestDirectory,
      extraArguments: const <String>['--plain-name', 'include']);
    if (!(result.stdout as String).contains('+1: All tests passed')) {
      fail('unexpected output from test:\n\n${result.stdout}\n-- end stdout --\n\n');
    }
    expect(result.exitCode, 0);
  });
113

114
  testWithoutContext('flutter test should run a test with a given tag', () async {
115 116 117 118 119 120 121 122
    final ProcessResult result = await _runFlutterTest('filtering_tag', automatedTestsDirectory, flutterTestDirectory,
        extraArguments: const <String>['--tags', 'include-tag']);
    if (!(result.stdout as String).contains('+1: All tests passed')) {
      fail('unexpected output from test:\n\n${result.stdout}\n-- end stdout --\n\n');
    }
    expect(result.exitCode, 0);
  });

123
  testWithoutContext('flutter test should not run a test with excluded tag', () async {
124 125 126 127 128 129 130 131
    final ProcessResult result = await _runFlutterTest('filtering_tag', automatedTestsDirectory, flutterTestDirectory,
        extraArguments: const <String>['--exclude-tags', 'exclude-tag']);
    if (!(result.stdout as String).contains('+1: All tests passed')) {
      fail('unexpected output from test:\n\n${result.stdout}\n-- end stdout --\n\n');
    }
    expect(result.exitCode, 0);
  });

132
  testWithoutContext('flutter test should run all tests when tags are unspecified', () async {
133 134 135 136 137 138 139
    final ProcessResult result = await _runFlutterTest('filtering_tag', automatedTestsDirectory, flutterTestDirectory);
    if (!(result.stdout as String).contains('+1 -1: Some tests failed')) {
      fail('unexpected output from test:\n\n${result.stdout}\n-- end stdout --\n\n');
    }
    expect(result.exitCode, 1);
  });

140
  testWithoutContext('flutter test should run a widgetTest with a given tag', () async {
141 142 143 144 145 146 147 148
    final ProcessResult result = await _runFlutterTest('filtering_tag_widget', automatedTestsDirectory, flutterTestDirectory,
        extraArguments: const <String>['--tags', 'include-tag']);
    if (!(result.stdout as String).contains('+1: All tests passed')) {
      fail('unexpected output from test:\n\n${result.stdout}\n-- end stdout --\n\n');
    }
    expect(result.exitCode, 0);
  });

149
  testWithoutContext('flutter test should not run a widgetTest with excluded tag', () async {
150 151 152 153 154 155 156 157
    final ProcessResult result = await _runFlutterTest('filtering_tag_widget', automatedTestsDirectory, flutterTestDirectory,
        extraArguments: const <String>['--exclude-tags', 'exclude-tag']);
    if (!(result.stdout as String).contains('+1: All tests passed')) {
      fail('unexpected output from test:\n\n${result.stdout}\n-- end stdout --\n\n');
    }
    expect(result.exitCode, 0);
  });

158
  testWithoutContext('flutter test should run all widgetTest when tags are unspecified', () async {
159 160 161 162 163 164 165
    final ProcessResult result = await _runFlutterTest('filtering_tag_widget', automatedTestsDirectory, flutterTestDirectory);
    if (!(result.stdout as String).contains('+1 -1: Some tests failed')) {
      fail('unexpected output from test:\n\n${result.stdout}\n-- end stdout --\n\n');
    }
    expect(result.exitCode, 1);
  });

166
  testWithoutContext('flutter test should test runs to completion', () async {
167 168 169 170
    final ProcessResult result = await _runFlutterTest('trivial', automatedTestsDirectory, flutterTestDirectory,
      extraArguments: const <String>['--verbose']);
    final String stdout = result.stdout as String;
    if ((!stdout.contains('+1: All tests passed')) ||
171
        (!stdout.contains('test 0: Starting flutter_tester process with command')) ||
172 173 174 175 176 177 178 179 180 181 182
        (!stdout.contains('test 0: deleting temporary directory')) ||
        (!stdout.contains('test 0: finished')) ||
        (!stdout.contains('test package returned with exit code 0'))) {
      fail('unexpected output from test:\n\n${result.stdout}\n-- end stdout --\n\n');
    }
    if ((result.stderr as String).isNotEmpty) {
      fail('unexpected error output from test:\n\n${result.stderr}\n-- end stderr --\n\n');
    }
    expect(result.exitCode, 0);
  });

183
  testWithoutContext('flutter test should run all tests inside of a directory with no trailing slash', () async {
184 185 186 187
    final ProcessResult result = await _runFlutterTest(null, automatedTestsDirectory, flutterTestDirectory + '/child_directory',
      extraArguments: const <String>['--verbose']);
    final String stdout = result.stdout as String;
    if ((!stdout.contains('+2: All tests passed')) ||
188
        (!stdout.contains('test 0: Starting flutter_tester process with command')) ||
189 190 191 192 193 194 195 196 197
        (!stdout.contains('test 0: deleting temporary directory')) ||
        (!stdout.contains('test 0: finished')) ||
        (!stdout.contains('test package returned with exit code 0'))) {
      fail('unexpected output from test:\n\n${result.stdout}\n-- end stdout --\n\n');
    }
    if ((result.stderr as String).isNotEmpty) {
      fail('unexpected error output from test:\n\n${result.stderr}\n-- end stderr --\n\n');
    }
    expect(result.exitCode, 0);
198
  });
199 200
}

201
Future<void> _testFile(
202 203 204 205 206 207
  String testName,
  String workingDirectory,
  String testDirectory, {
  Matcher exitCode,
  List<String> extraArguments = const <String>[],
}) async {
208
  exitCode ??= isNonZero;
209 210
  final String fullTestExpectation = fileSystem.path.join(testDirectory, '${testName}_expectation.txt');
  final File expectationFile = fileSystem.file(fullTestExpectation);
211
  if (!expectationFile.existsSync()) {
212
    fail('missing expectation file: $expectationFile');
213
  }
214

215 216 217 218 219 220
  final ProcessResult exec = await _runFlutterTest(
    testName,
    workingDirectory,
    testDirectory,
    extraArguments: extraArguments,
  );
221

222
  expect(exec.exitCode, exitCode);
223
  final List<String> output = (exec.stdout as String).split('\n');
224
  if (output.first.startsWith('Waiting for another flutter command to release the startup lock...')) {
225
    output.removeAt(0);
226 227
  }
  if (output.first.startsWith('Running "flutter pub get" in')) {
228
    output.removeAt(0);
229
  }
230
  output.add('<<stderr>>');
231
  output.addAll((exec.stderr as String).split('\n'));
232
  final List<String> expectations = fileSystem.file(fullTestExpectation).readAsLinesSync();
233 234 235
  bool allowSkip = false;
  int expectationLineNumber = 0;
  int outputLineNumber = 0;
236
  bool haveSeenStdErrMarker = false;
237
  while (expectationLineNumber < expectations.length) {
238 239 240 241 242
    expect(
      output,
      hasLength(greaterThan(outputLineNumber)),
      reason: 'Failure in $testName to compare to $fullTestExpectation',
    );
243
    final String expectationLine = expectations[expectationLineNumber];
244
    String outputLine = output[outputLineNumber];
245 246 247 248 249 250
    if (expectationLine == '<<skip until matching line>>') {
      allowSkip = true;
      expectationLineNumber += 1;
      continue;
    }
    if (allowSkip) {
251
      if (!RegExp(expectationLine).hasMatch(outputLine)) {
252 253 254 255 256
        outputLineNumber += 1;
        continue;
      }
      allowSkip = false;
    }
257 258 259 260
    if (expectationLine == '<<stderr>>') {
      expect(haveSeenStdErrMarker, isFalse);
      haveSeenStdErrMarker = true;
    }
261 262 263 264 265 266 267 268 269 270 271 272
    if (!RegExp(expectationLine).hasMatch(outputLine) && outputLineNumber + 1 < output.length) {
      // Check if the RegExp can match the next two lines in the output so
      // that it is possible to write expectations that still hold even if a
      // line is wrapped slightly differently due to for example a file name
      // being longer on one platform than another.
      final String mergedLines = '$outputLine\n${output[outputLineNumber+1]}';
      if (RegExp(expectationLine).hasMatch(mergedLines)) {
        outputLineNumber += 1;
        outputLine = mergedLines;
      }
    }

273
    expect(outputLine, matches(expectationLine), reason: 'Full output:\n- - - -----8<----- - - -\n${output.join("\n")}\n- - - -----8<----- - - -');
274 275 276 277
    expectationLineNumber += 1;
    outputLineNumber += 1;
  }
  expect(allowSkip, isFalse);
278
  if (!haveSeenStdErrMarker) {
279
    expect(exec.stderr, '');
280
  }
281
}
282

283 284 285 286
Future<ProcessResult> _runFlutterTest(
  String testName,
  String workingDirectory,
  String testDirectory, {
287
  List<String> extraArguments = const <String>[],
288 289
}) async {

290 291 292 293
  String testPath;
  if (testName == null) {
    // Test everything in the directory.
    testPath = testDirectory;
294
    final Directory directoryToTest = fileSystem.directory(testPath);
295
    if (!directoryToTest.existsSync()) {
296
      fail('missing test directory: $directoryToTest');
297
    }
298 299
  } else {
    // Test just a specific test file.
300 301
    testPath = fileSystem.path.join(testDirectory, '${testName}_test.dart');
    final File testFile = fileSystem.file(testPath);
302
    if (!testFile.existsSync()) {
303
      fail('missing test file: $testFile');
304
    }
305
  }
306

307 308 309
  final List<String> args = <String>[
    'test',
    '--no-color',
310
    '--no-version-check',
311
    '--no-pub',
312
    ...extraArguments,
313
    testPath,
314
  ];
315

316 317 318 319 320 321 322
  return Process.run(
    flutterBin, // Uses the precompiled flutter tool for faster tests,
    args,
    workingDirectory: workingDirectory,
    stdoutEncoding: utf8,
    stderrEncoding: utf8,
  );
323
}