run_test.dart 5.68 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:io';

7
import 'package:flutter_devicelab/framework/utils.dart' show rm;
8 9
import 'package:path/path.dart' as path;
import 'package:process/process.dart';
10 11

import 'common.dart';
12 13

void main() {
14
  const ProcessManager processManager = LocalProcessManager();
15

16
  group('run.dart script', () {
17 18 19 20
    Future<ProcessResult> runScript(List<String> testNames,
        [List<String> otherArgs = const <String>[]]) async {
      final String dart = path.absolute(
          path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', 'dart'));
21 22 23
      final ProcessResult scriptProcess = processManager.runSync(<String>[
        dart,
        'bin/run.dart',
24
        '--no-terminate-stray-dart-processes',
25
        ...otherArgs,
26
        for (final String testName in testNames) ...<String>['-t', testName],
27
      ]);
28 29 30
      return scriptProcess;
    }

31
    Future<void> expectScriptResult(
32 33
        List<String> testNames,
        int expectedExitCode,
34
        {String? deviceId}
35 36 37 38
      ) async {
      final ProcessResult result = await runScript(testNames, <String>[
        if (deviceId != null) ...<String>['-d', deviceId],
      ]);
39
      expect(result.exitCode, expectedExitCode,
40 41 42
          reason:
              '[ stderr from test process ]\n\n${result.stderr}\n\n[ end of stderr ]'
              '\n\n[ stdout from test process ]\n\n${result.stdout}\n\n[ end of stdout ]');
43 44
    }

45
    test('exits with code 0 when succeeds', () async {
46
      await expectScriptResult(<String>['smoke_test_success'], 0);
47 48
    });

49
    test('accepts file paths', () async {
50 51
      await expectScriptResult(
          <String>['bin/tasks/smoke_test_success.dart'], 0);
52 53 54
    });

    test('rejects invalid file paths', () async {
55
      await expectScriptResult(<String>['lib/framework/adb.dart'], 1);
56 57 58
    });

    test('exits with code 1 when task throws', () async {
59
      await expectScriptResult(<String>['smoke_test_throws'], 1);
60
    });
61

62
    test('exits with code 1 when fails', () async {
63
      await expectScriptResult(<String>['smoke_test_failure'], 1);
64 65
    });

66
    test('exits with code 1 when fails to connect', () async {
67 68
      await expectScriptResult(<String>['smoke_test_setup_failure'], 1);
    }, skip: true); // https://github.com/flutter/flutter/issues/53707
69

70
    test('exits with code 1 when results are mixed', () async {
71 72
      await expectScriptResult(
        <String>[
73 74
          'smoke_test_failure',
          'smoke_test_success',
75
        ],
76
        1,
77 78
      );
    });
79

80 81 82 83 84 85 86 87 88 89 90
    test('exits with code 0 when provided a valid device ID', () async {
      await expectScriptResult(<String>['smoke_test_device'], 0,
        deviceId: 'FAKE');
    });

    test('exits with code 1 when provided a bad device ID', () async {
      await expectScriptResult(<String>['smoke_test_device'], 1,
        deviceId: 'THIS_IS_NOT_VALID');
    });


91
    test('runs A/B test', () async {
92 93 94 95 96
      final Directory tempDirectory = Directory.systemTemp.createTempSync('flutter_devicelab_ab_test.');
      final File abResultsFile = File(path.join(tempDirectory.path, 'test_results.json'));

      expect(abResultsFile.existsSync(), isFalse);

97 98
      final ProcessResult result = await runScript(
        <String>['smoke_test_success'],
99
        <String>['--ab=2', '--local-engine=host_debug_unopt', '--ab-result-file', abResultsFile.path],
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
      );
      expect(result.exitCode, 0);

      String sectionHeader = !Platform.isWindows
          ? '═════════════════════════╡ ••• A/B results so far ••• ╞═════════════════════════'
          : 'A/B results so far';
      expect(
        result.stdout,
        contains(
          '$sectionHeader\n'
          '\n'
          'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n'
          'metric1\t42.00 (0.00%)\t42.00 (0.00%)\t1.00x\t\n'
          'metric2\t123.00 (0.00%)\t123.00 (0.00%)\t1.00x\t\n',
        ),
      );

      sectionHeader = !Platform.isWindows
          ? '════════════════════════════╡ ••• Raw results ••• ╞═════════════════════════════'
          : 'Raw results';
      expect(
        result.stdout,
        contains(
          '$sectionHeader\n'
          '\n'
          'metric1:\n'
          '  A:\t42.00\t42.00\t\n'
          '  B:\t42.00\t42.00\t\n'
          'metric2:\n'
          '  A:\t123.00\t123.00\t\n'
          '  B:\t123.00\t123.00\t\n',
        ),
      );

      sectionHeader = !Platform.isWindows
          ? '═════════════════════════╡ ••• Final A/B results ••• ╞══════════════════════════'
          : 'Final A/B results';
      expect(
        result.stdout,
        contains(
          '$sectionHeader\n'
          '\n'
          'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n'
          'metric1\t42.00 (0.00%)\t42.00 (0.00%)\t1.00x\t\n'
          'metric2\t123.00 (0.00%)\t123.00 (0.00%)\t1.00x\t\n',
        ),
      );
147 148 149

      expect(abResultsFile.existsSync(), isTrue);
      rm(tempDirectory, recursive: true);
150
    });
151 152 153 154 155 156 157 158 159

    test('fails to upload results to Cocoon if flags given', () async {
      // CocoonClient will fail to find test-file, and will not send any http requests.
      final ProcessResult result = await runScript(
        <String>['smoke_test_success'],
        <String>['--service-account-file=test-file', '--task-key=task123'],
      );
      expect(result.exitCode, 1);
    });
160 161
  });
}