test_compat.dart 17.5 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';

7
import 'package:meta/meta.dart';
8 9 10
import 'package:test_api/src/backend/declarer.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/group_entry.dart'; // ignore: implementation_imports
11
import 'package:test_api/src/backend/invoker.dart'; // ignore: implementation_imports
12 13
import 'package:test_api/src/backend/live_test.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/message.dart'; // ignore: implementation_imports
14
import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
15
import 'package:test_api/src/backend/state.dart'; // ignore: implementation_imports
16 17 18
import 'package:test_api/src/backend/suite.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/test.dart'; // ignore: implementation_imports
19
// ignore: deprecated_member_use
20 21
import 'package:test_api/test_api.dart';

22 23 24
// ignore: deprecated_member_use
export 'package:test_api/fake.dart' show Fake;

25
Declarer? _localDeclarer;
26
Declarer get _declarer {
27
  final Declarer? declarer = Zone.current[#test.declarer] as Declarer?;
28 29 30 31 32 33 34 35 36
  if (declarer != null) {
    return declarer;
  }
  // If no declarer is defined, this test is being run via `flutter run -t test_file.dart`.
  if (_localDeclarer == null) {
    _localDeclarer = Declarer();
    Future<void>(() {
      Invoker.guard<Future<void>>(() async {
        final _Reporter reporter = _Reporter(color: false); // disable color when run directly.
37
        // ignore: recursive_getters, this self-call is safe since it will just fetch the declarer instance
38 39 40 41 42 43 44
        final Group group = _declarer.build();
        final Suite suite = Suite(group, SuitePlatform(Runtime.vm));
        await _runGroup(suite, group, <Group>[], reporter);
        reporter._onDone();
      });
    });
  }
45
  return _localDeclarer!;
46 47 48 49 50 51 52 53
}

Future<void> _runGroup(Suite suiteConfig, Group group, List<Group> parents, _Reporter reporter) async {
  parents.add(group);
  try {
    final bool skipGroup = group.metadata.skip;
    bool setUpAllSucceeded = true;
    if (!skipGroup && group.setUpAll != null) {
54
      final LiveTest liveTest = group.setUpAll!.load(suiteConfig, groups: parents);
55 56 57 58
      await _runLiveTest(suiteConfig, liveTest, reporter, countSuccess: false);
      setUpAllSucceeded = liveTest.state.result.isPassing;
    }
    if (setUpAllSucceeded) {
59
      for (final GroupEntry entry in group.entries) {
60 61 62
        if (entry is Group) {
          await _runGroup(suiteConfig, entry, parents, reporter);
        } else if (entry.metadata.skip) {
63
          await _runSkippedTest(suiteConfig, entry as Test, parents, reporter);
64
        } else {
65
          final Test test = entry as Test;
66 67 68 69 70 71 72
          await _runLiveTest(suiteConfig, test.load(suiteConfig, groups: parents), reporter);
        }
      }
    }
    // Even if we're closed or setUpAll failed, we want to run all the
    // teardowns to ensure that any state is properly cleaned up.
    if (!skipGroup && group.tearDownAll != null) {
73
      final LiveTest liveTest = group.tearDownAll!.load(suiteConfig, groups: parents);
74 75 76 77 78 79 80
      await _runLiveTest(suiteConfig, liveTest, reporter, countSuccess: false);
    }
  } finally {
    parents.remove(group);
  }
}

81
Future<void> _runLiveTest(Suite suiteConfig, LiveTest liveTest, _Reporter reporter, { bool countSuccess = true }) async {
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97
  reporter._onTestStarted(liveTest);
  // Schedule a microtask to ensure that [onTestStarted] fires before the
  // first [LiveTest.onStateChange] event.
  await Future<void>.microtask(liveTest.run);
  // Once the test finishes, use await null to do a coarse-grained event
  // loop pump to avoid starving non-microtask events.
  await null;
  final bool isSuccess = liveTest.state.result.isPassing;
  if (isSuccess) {
    reporter.passed.add(liveTest);
  } else {
    reporter.failed.add(liveTest);
  }
}

Future<void> _runSkippedTest(Suite suiteConfig, Test test, List<Group> parents, _Reporter reporter) async {
98
  final LocalTest skipped = LocalTest(test.name, test.metadata, () { }, trace: test.trace);
99
  if (skipped.metadata.skipReason != null) {
100
    reporter.log('Skip: ${skipped.metadata.skipReason}');
101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
  }
  final LiveTest liveTest = skipped.load(suiteConfig);
  reporter._onTestStarted(liveTest);
  reporter.skipped.add(skipped);
}

// TODO(nweiz): This and other top-level functions should throw exceptions if
// they're called after the declarer has finished declaring.
/// Creates a new test case with the given description (converted to a string)
/// and body.
///
/// The description will be added to the descriptions of any surrounding
/// [group]s. If [testOn] is passed, it's parsed as a [platform selector][]; the
/// test will only be run on matching platforms.
///
/// [platform selector]: https://github.com/dart-lang/test/tree/master/pkgs/test#platform-selectors
///
/// If [timeout] is passed, it's used to modify or replace the default timeout
/// of 30 seconds. Timeout modifications take precedence in suite-group-test
/// order, so [timeout] will also modify any timeouts set on the group or suite.
///
/// If [skip] is a String or `true`, the test is skipped. If it's a String, it
/// should explain why the test is skipped; this reason will be printed instead
/// of running the test.
///
/// If [tags] is passed, it declares user-defined tags that are applied to the
/// test. These tags can be used to select or skip the test on the command line,
/// or to do bulk test configuration. All tags should be declared in the
/// [package configuration file][configuring tags]. The parameter can be an
/// [Iterable] of tag names, or a [String] representing a single tag.
///
/// If [retry] is passed, the test will be retried the provided number of times
/// before being marked as a failure.
///
135
/// [configuring tags]: https://github.com/dart-lang/test/blob/44d6cb196f34a93a975ed5f3cb76afcc3a7b39b0/doc/package_config.md#configuring-tags
136 137 138 139 140 141 142 143 144 145
///
/// [onPlatform] allows tests to be configured on a platform-by-platform
/// basis. It's a map from strings that are parsed as [PlatformSelector]s to
/// annotation classes: [Timeout], [Skip], or lists of those. These
/// annotations apply only on the given platforms. For example:
///
///     test('potentially slow test', () {
///       // ...
///     }, onPlatform: {
///       // This test is especially slow on Windows.
Anas35's avatar
Anas35 committed
146
///       'windows': Timeout.factor(2),
147
///       'browser': [
148
///         Skip('add browser support'),
149
///         // This will be slow on browsers once it works on them.
Anas35's avatar
Anas35 committed
150
///         Timeout.factor(2)
151 152 153 154 155
///       ]
///     });
///
/// If multiple platforms match, the annotations apply in order as through
/// they were in nested groups.
156
@isTest
157 158
void test(
  Object description,
159
  dynamic Function() body, {
160 161
  String? testOn,
  Timeout? timeout,
162 163
  dynamic skip,
  dynamic tags,
164 165
  Map<String, dynamic>? onPlatform,
  int? retry,
166
}) {
167
  _declarer.test(
168 169
    description.toString(),
    body,
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184
    testOn: testOn,
    timeout: timeout,
    skip: skip,
    onPlatform: onPlatform,
    tags: tags,
    retry: retry,
  );
}

/// Creates a group of tests.
///
/// A group's description (converted to a string) is included in the descriptions
/// of any tests or sub-groups it contains. [setUp] and [tearDown] are also scoped
/// to the containing group.
///
185
/// If `skip` is a String or `true`, the group is skipped. If it's a String, it
186 187
/// should explain why the group is skipped; this reason will be printed instead
/// of running the group's tests.
188
@isTestGroup
189
void group(Object description, void Function() body, { dynamic skip }) {
190
  _declarer.group(description.toString(), body, skip: skip);
191 192 193 194
}

/// Registers a function to be run before tests.
///
195
/// This function will be called before each test is run. The `body` may be
196 197 198
/// asynchronous; if so, it must return a [Future].
///
/// If this is called within a test group, it applies only to tests in that
199
/// group. The `body` will be run after any set-up callbacks in parent groups or
200 201 202 203
/// at the top level.
///
/// Each callback at the top level or in a given group will be run in the order
/// they were declared.
204
void setUp(dynamic Function() body) {
205 206 207 208 209
  _declarer.setUp(body);
}

/// Registers a function to be run after tests.
///
210
/// This function will be called after each test is run. The `body` may be
211 212 213
/// asynchronous; if so, it must return a [Future].
///
/// If this is called within a test group, it applies only to tests in that
214
/// group. The `body` will be run before any tear-down callbacks in parent
215 216 217 218 219 220
/// groups or at the top level.
///
/// Each callback at the top level or in a given group will be run in the
/// reverse of the order they were declared.
///
/// See also [addTearDown], which adds tear-downs to a running test.
221
void tearDown(dynamic Function() body) {
222 223 224 225 226
  _declarer.tearDown(body);
}

/// Registers a function to be run once before all tests.
///
227
/// The `body` may be asynchronous; if so, it must return a [Future].
228
///
229
/// If this is called within a test group, The `body` will run before all tests
230 231 232 233 234 235 236 237
/// in that group. It will be run after any [setUpAll] callbacks in parent
/// groups or at the top level. It won't be run if none of the tests in the
/// group are run.
///
/// **Note**: This function makes it very easy to accidentally introduce hidden
/// dependencies between tests that should be isolated. In general, you should
/// prefer [setUp], and only use [setUpAll] if the callback is prohibitively
/// slow.
238
void setUpAll(dynamic Function() body) {
239 240 241 242 243
  _declarer.setUpAll(body);
}

/// Registers a function to be run once after all tests.
///
244
/// If this is called within a test group, `body` will run after all tests
245 246 247 248 249 250 251 252
/// in that group. It will be run before any [tearDownAll] callbacks in parent
/// groups or at the top level. It won't be run if none of the tests in the
/// group are run.
///
/// **Note**: This function makes it very easy to accidentally introduce hidden
/// dependencies between tests that should be isolated. In general, you should
/// prefer [tearDown], and only use [tearDownAll] if the callback is
/// prohibitively slow.
253
void tearDownAll(dynamic Function() body) {
254 255 256 257 258 259 260 261 262 263 264 265
  _declarer.tearDownAll(body);
}


/// A reporter that prints each test on its own line.
///
/// This is currently used in place of [CompactReporter] by `lib/test.dart`,
/// which can't transitively import `dart:io` but still needs access to a runner
/// so that test files can be run directly. This means that until issue 6943 is
/// fixed, this must not import `dart:io`.
class _Reporter {
  _Reporter({bool color = true, bool printPath = true})
266 267 268 269 270 271 272
    : _printPath = printPath,
      _green = color ? '\u001b[32m' : '',
      _red = color ? '\u001b[31m' : '',
      _yellow = color ? '\u001b[33m' : '',
      _bold = color ? '\u001b[1m' : '',
      _noColor = color ? '\u001b[0m' : '';

273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
  final List<LiveTest> passed = <LiveTest>[];
  final List<LiveTest> failed = <LiveTest>[];
  final List<Test> skipped = <Test>[];

  /// The terminal escape for green text, or the empty string if this is Windows
  /// or not outputting to a terminal.
  final String _green;

  /// The terminal escape for red text, or the empty string if this is Windows
  /// or not outputting to a terminal.
  final String _red;

  /// The terminal escape for yellow text, or the empty string if this is
  /// Windows or not outputting to a terminal.
  final String _yellow;

  /// The terminal escape for bold text, or the empty string if this is
  /// Windows or not outputting to a terminal.
  final String _bold;

  /// The terminal escape for removing test coloring, or the empty string if
  /// this is Windows or not outputting to a terminal.
  final String _noColor;

  /// Whether the path to each test's suite should be printed.
  final bool _printPath;

  /// A stopwatch that tracks the duration of the full run.
  final Stopwatch _stopwatch = Stopwatch();

  /// The size of `_engine.passed` last time a progress notification was
  /// printed.
305
  int? _lastProgressPassed;
306 307 308

  /// The size of `_engine.skipped` last time a progress notification was
  /// printed.
309
  int? _lastProgressSkipped;
310 311 312

  /// The size of `_engine.failed` last time a progress notification was
  /// printed.
313
  int? _lastProgressFailed;
314 315

  /// The message printed for the last progress notification.
316
  String? _lastProgressMessage;
317 318

  /// The suffix added to the last progress notification.
319
  String? _lastProgressSuffix;
320 321

  /// The set of all subscriptions to various streams.
322
  final Set<StreamSubscription<void>> _subscriptions = <StreamSubscription<void>>{};
323 324 325 326 327 328 329 330 331 332 333 334 335 336 337

  /// A callback called when the engine begins running [liveTest].
  void _onTestStarted(LiveTest liveTest) {
    if (!_stopwatch.isRunning) {
      _stopwatch.start();
    }
    _progressLine(_description(liveTest));
    _subscriptions.add(liveTest.onStateChange.listen((State state) => _onStateChange(liveTest, state)));
    _subscriptions.add(liveTest.onError.listen((AsyncError error) => _onError(liveTest, error.error, error.stackTrace)));
    _subscriptions.add(liveTest.onMessage.listen((Message message) {
      _progressLine(_description(liveTest));
      String text = message.text;
      if (message.type == MessageType.skip) {
        text = '  $_yellow$text$_noColor';
      }
338
      log(text);
339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354
    }));
  }

  /// A callback called when [liveTest]'s state becomes [state].
  void _onStateChange(LiveTest liveTest, State state) {
    if (state.status != Status.complete) {
      return;
    }
  }

  /// A callback called when [liveTest] throws [error].
  void _onError(LiveTest liveTest, Object error, StackTrace stackTrace) {
    if (liveTest.state.status != Status.complete) {
      return;
    }
    _progressLine(_description(liveTest), suffix: ' $_bold$_red[E]$_noColor');
355 356
    log(_indent(error.toString()));
    log(_indent('$stackTrace'));
357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
  }

  /// A callback called when the engine is finished running tests.
  void _onDone() {
    final bool success = failed.isEmpty;
    if (!success) {
      _progressLine('Some tests failed.', color: _red);
    } else if (passed.isEmpty) {
      _progressLine('All tests skipped.');
    } else {
      _progressLine('All tests passed!');
    }
  }

  /// Prints a line representing the current state of the tests.
  ///
  /// [message] goes after the progress report. If [color] is passed, it's used
  /// as the color for [message]. If [suffix] is passed, it's added to the end
  /// of [message].
376
  void _progressLine(String message, { String? color, String? suffix }) {
377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424
    // Print nothing if nothing has changed since the last progress line.
    if (passed.length == _lastProgressPassed &&
        skipped.length == _lastProgressSkipped &&
        failed.length == _lastProgressFailed &&
        message == _lastProgressMessage &&
        // Don't re-print just because a suffix was removed.
        (suffix == null || suffix == _lastProgressSuffix)) {
      return;
    }
    _lastProgressPassed = passed.length;
    _lastProgressSkipped = skipped.length;
    _lastProgressFailed = failed.length;
    _lastProgressMessage = message;
    _lastProgressSuffix = suffix;

    if (suffix != null) {
      message += suffix;
    }
    color ??= '';
    final Duration duration = _stopwatch.elapsed;
    final StringBuffer buffer = StringBuffer();

    // \r moves back to the beginning of the current line.
    buffer.write('${_timeString(duration)} ');
    buffer.write(_green);
    buffer.write('+');
    buffer.write(passed.length);
    buffer.write(_noColor);

    if (skipped.isNotEmpty) {
      buffer.write(_yellow);
      buffer.write(' ~');
      buffer.write(skipped.length);
      buffer.write(_noColor);
    }

    if (failed.isNotEmpty) {
      buffer.write(_red);
      buffer.write(' -');
      buffer.write(failed.length);
      buffer.write(_noColor);
    }

    buffer.write(': ');
    buffer.write(color);
    buffer.write(message);
    buffer.write(_noColor);

425
    log(buffer.toString());
426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445
  }

  /// Returns a representation of [duration] as `MM:SS`.
  String _timeString(Duration duration) {
    final String minutes = duration.inMinutes.toString().padLeft(2, '0');
    final String seconds = (duration.inSeconds % 60).toString().padLeft(2, '0');
    return '$minutes:$seconds';
  }

  /// Returns a description of [liveTest].
  ///
  /// This differs from the test's own description in that it may also include
  /// the suite's name.
  String _description(LiveTest liveTest) {
    String name = liveTest.test.name;
    if (_printPath && liveTest.suite.path != null) {
      name = '${liveTest.suite.path}: $name';
    }
    return name;
  }
446 447 448 449 450 451 452

  /// Print the message to the console.
  void log(String message) {
    // We centralize all the prints in this file through this one method so that
    // in principle we can reroute the output easily should we need to.
    print(message); // ignore: avoid_print
  }
453 454
}

455
String _indent(String string, { int? size, String? first }) {
456 457 458 459
  size ??= first == null ? 2 : first.length;
  return _prefixLines(string, ' ' * size, first: first);
}

460
String _prefixLines(String text, String prefix, { String? first, String? last, String? single }) {
461 462
  first ??= prefix;
  last ??= prefix;
463
  single ??= first;
464 465 466 467 468 469
  final List<String> lines = text.split('\n');
  if (lines.length == 1) {
    return '$single$text';
  }
  final StringBuffer buffer = StringBuffer('$first${lines.first}\n');
  // Write out all but the first and last lines with [prefix].
470
  for (final String line in lines.skip(1).take(lines.length - 2)) {
471 472 473 474 475
    buffer.writeln('$prefix$line');
  }
  buffer.write('$last${lines.last}');
  return buffer.toString();
}