run_test.dart 4.38 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6 7
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:io';

8 9
import 'package:path/path.dart' as path;
import 'package:process/process.dart';
10 11

import 'common.dart';
12 13

void main() {
14
  const ProcessManager processManager = LocalProcessManager();
15

16
  group('run.dart script', () {
17 18 19 20
    Future<ProcessResult> runScript(List<String> testNames,
        [List<String> otherArgs = const <String>[]]) async {
      final String dart = path.absolute(
          path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', 'dart'));
21 22 23
      final ProcessResult scriptProcess = processManager.runSync(<String>[
        dart,
        'bin/run.dart',
24
        ...otherArgs,
25
        for (final String testName in testNames) ...<String>['-t', testName],
26
      ]);
27 28 29
      return scriptProcess;
    }

30 31
    Future<void> expectScriptResult(
        List<String> testNames, int expectedExitCode) async {
32
      final ProcessResult result = await runScript(testNames);
33
      expect(result.exitCode, expectedExitCode,
34 35 36
          reason:
              '[ stderr from test process ]\n\n${result.stderr}\n\n[ end of stderr ]'
              '\n\n[ stdout from test process ]\n\n${result.stdout}\n\n[ end of stdout ]');
37 38
    }

39
    test('exits with code 0 when succeeds', () async {
40
      await expectScriptResult(<String>['smoke_test_success'], 0);
41 42
    });

43
    test('accepts file paths', () async {
44 45
      await expectScriptResult(
          <String>['bin/tasks/smoke_test_success.dart'], 0);
46 47 48
    });

    test('rejects invalid file paths', () async {
49
      await expectScriptResult(<String>['lib/framework/adb.dart'], 1);
50 51 52
    });

    test('exits with code 1 when task throws', () async {
53
      await expectScriptResult(<String>['smoke_test_throws'], 1);
54
    });
55

56
    test('exits with code 1 when fails', () async {
57
      await expectScriptResult(<String>['smoke_test_failure'], 1);
58 59
    });

60
    test('exits with code 1 when fails to connect', () async {
61 62
      await expectScriptResult(<String>['smoke_test_setup_failure'], 1);
    }, skip: true); // https://github.com/flutter/flutter/issues/53707
63

64
    test('exits with code 1 when results are mixed', () async {
65 66
      await expectScriptResult(
        <String>[
67 68
          'smoke_test_failure',
          'smoke_test_success',
69
        ],
70
        1,
71 72
      );
    });
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125

    test('runs A/B test', () async {
      final ProcessResult result = await runScript(
        <String>['smoke_test_success'],
        <String>['--ab=2', '--local-engine=host_debug_unopt'],
      );
      expect(result.exitCode, 0);

      String sectionHeader = !Platform.isWindows
          ? '═════════════════════════╡ ••• A/B results so far ••• ╞═════════════════════════'
          : 'A/B results so far';
      expect(
        result.stdout,
        contains(
          '$sectionHeader\n'
          '\n'
          'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n'
          'metric1\t42.00 (0.00%)\t42.00 (0.00%)\t1.00x\t\n'
          'metric2\t123.00 (0.00%)\t123.00 (0.00%)\t1.00x\t\n',
        ),
      );

      sectionHeader = !Platform.isWindows
          ? '════════════════════════════╡ ••• Raw results ••• ╞═════════════════════════════'
          : 'Raw results';
      expect(
        result.stdout,
        contains(
          '$sectionHeader\n'
          '\n'
          'metric1:\n'
          '  A:\t42.00\t42.00\t\n'
          '  B:\t42.00\t42.00\t\n'
          'metric2:\n'
          '  A:\t123.00\t123.00\t\n'
          '  B:\t123.00\t123.00\t\n',
        ),
      );

      sectionHeader = !Platform.isWindows
          ? '═════════════════════════╡ ••• Final A/B results ••• ╞══════════════════════════'
          : 'Final A/B results';
      expect(
        result.stdout,
        contains(
          '$sectionHeader\n'
          '\n'
          'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n'
          'metric1\t42.00 (0.00%)\t42.00 (0.00%)\t1.00x\t\n'
          'metric2\t123.00 (0.00%)\t123.00 (0.00%)\t1.00x\t\n',
        ),
      );
    });
126 127
  });
}