runner.dart 7.22 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io';

import 'package:path/path.dart' as path;

import 'customer_test.dart';

Future<bool> runTests({
  int repeat = 1,
  bool skipOnFetchFailure = false,
  bool verbose = false,
  int numberShards = 1,
  int shardIndex = 0,
18
  required List<File> files,
19
}) async {
20
  if (verbose) {
21
    print('Starting run_tests.dart...');
22
  }
23 24 25 26 27 28 29 30 31 32 33 34

  // Best attempt at evenly splitting tests among the shards
  final List<File> shardedFiles = <File>[];
  for (int i = shardIndex; i < files.length; i += numberShards) {
    shardedFiles.add(files[i]);
  }

  int testCount = 0;
  int failures = 0;

  if (verbose) {
    final String s = files.length == 1 ? '' : 's';
35 36 37 38 39 40
    if (numberShards > 1) {
      final String ss = shardedFiles.length == 1 ? '' : 's';
      print('${files.length} file$s specified. ${shardedFiles.length} test$ss in shard #$shardIndex ($numberShards shards total).');
    } else {
      print('${files.length} file$s specified.');
    }
41 42 43 44
    print('');
  }

  if (verbose) {
45 46 47 48 49
    if (numberShards > 1) {
      print('Tests in this shard:');
    } else {
      print('Tests:');
    }
50
    for (final File file in shardedFiles) {
51
      print(file.path);
52
    }
53 54 55 56
  }
  print('');

  for (final File file in shardedFiles) {
57
    if (verbose) {
58
      print('Processing ${file.path}...');
59
    }
60 61

    void printHeader() {
62
      if (!verbose) {
63
        print('Processing ${file.path}...');
64
      }
65 66 67 68 69 70 71 72
    }

    void failure(String message) {
      printHeader();
      print('ERROR: $message');
      failures += 1;
    }

73 74 75 76
    CustomerTest instructions;
    try {
      instructions = CustomerTest(file);
    } on FormatException catch (error) {
77
      failure(error.message);
78 79 80
      print('');
      continue;
    } on FileSystemException catch (error) {
81
      failure(error.message);
82 83 84 85
      print('');
      continue;
    }

86 87
    bool success = true;

88
    final Directory checkout = Directory.systemTemp.createTempSync('flutter_customer_testing.${path.basenameWithoutExtension(file.path)}.');
89
    if (verbose) {
90
      print('Created temporary directory: ${checkout.path}');
91
    }
92
    try {
93
      assert(instructions.fetch.isNotEmpty);
94
      for (final String fetchCommand in instructions.fetch) {
95
        success = await shell(fetchCommand, checkout, verbose: verbose, silentFailure: skipOnFetchFailure, failedCallback: printHeader);
96 97 98 99 100 101 102 103
        if (!success) {
          if (skipOnFetchFailure) {
            if (verbose) {
              print('Skipping (fetch failed).');
            } else {
              print('Skipping ${file.path} (fetch failed).');
            }
          } else {
104
            failure('Failed to fetch repository.');
105 106 107 108
          }
          break;
        }
      }
109 110 111 112
      if (success) {
        final Directory customerRepo = Directory(path.join(checkout.path, 'tests'));
        for (final Directory updateDirectory in instructions.update) {
          final Directory resolvedUpdateDirectory = Directory(path.join(customerRepo.path, updateDirectory.path));
113
          if (verbose) {
114
            print('Updating code in ${resolvedUpdateDirectory.path}...');
115
          }
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
          if (!File(path.join(resolvedUpdateDirectory.path, 'pubspec.yaml')).existsSync()) {
            failure('The directory ${updateDirectory.path}, which was specified as an update directory, does not contain a "pubspec.yaml" file.');
            success = false;
            break;
          }
          success = await shell('flutter packages get', resolvedUpdateDirectory, verbose: verbose, failedCallback: printHeader);
          if (!success) {
            failure('Could not run "flutter pub get" in ${updateDirectory.path}, which was specified as an update directory.');
            break;
          }
          success = await shell('dart fix --apply', resolvedUpdateDirectory, verbose: verbose, failedCallback: printHeader);
          if (!success) {
            failure('Could not run "dart fix" in ${updateDirectory.path}, which was specified as an update directory.');
            break;
          }
        }
        if (success) {
133
          if (verbose) {
134
            print('Running tests...');
135
          }
136 137 138 139 140 141 142 143
          if (instructions.iterations != null && instructions.iterations! < repeat) {
            if (verbose) {
              final String s = instructions.iterations == 1 ? '' : 's';
              print('Limiting to ${instructions.iterations} round$s rather than $repeat rounds because of "iterations" directive.');
            }
            repeat = instructions.iterations!;
          }
          final Stopwatch stopwatch = Stopwatch()..start();
144
          for (int iteration = 0; iteration < repeat; iteration += 1) {
145
            if (verbose && repeat > 1) {
146
              print('Round ${iteration + 1} of $repeat.');
147
            }
148 149 150 151 152 153 154
            for (final String testCommand in instructions.tests) {
              testCount += 1;
              success = await shell(testCommand, customerRepo, verbose: verbose, failedCallback: printHeader);
              if (!success) {
                failure('One or more tests from ${path.basenameWithoutExtension(file.path)} failed.');
                break;
              }
155 156
            }
          }
157
          stopwatch.stop();
158
          if (verbose && success) {
159
            print('Tests finished in ${(stopwatch.elapsed.inSeconds / repeat).toStringAsFixed(2)} seconds per iteration.');
160
          }
161 162 163
        }
      }
    } finally {
164
      if (verbose) {
165
        print('Deleting temporary directory...');
166
      }
167 168 169 170 171 172
      try {
        checkout.deleteSync(recursive: true);
      } on FileSystemException {
        print('Failed to delete "${checkout.path}".');
      }
    }
173 174 175 176
    if (!success) {
      final String s = instructions.contacts.length == 1 ? '' : 's';
      print('Contact$s: ${instructions.contacts.join(", ")}');
    }
177
    if (verbose || !success) {
178
      print('');
179
    }
180 181 182 183 184 185 186 187 188 189 190 191
  }
  if (failures > 0) {
    final String s = failures == 1 ? '' : 's';
    print('$failures failure$s.');
    return false;
  }
  print('$testCount tests all passed!');
  return true;
}

final RegExp _spaces = RegExp(r' +');

192
Future<bool> shell(String command, Directory directory, { bool verbose = false, bool silentFailure = false, void Function()? failedCallback }) async {
193
  if (verbose) {
194
    print('>> $command');
195
  }
196 197 198 199 200 201 202 203 204 205 206
  Process process;
  if (Platform.isWindows) {
    process = await Process.start('CMD.EXE', <String>['/S', '/C', command], workingDirectory: directory.path);
  } else {
    final List<String> segments = command.trim().split(_spaces);
    process = await Process.start(segments.first, segments.skip(1).toList(), workingDirectory: directory.path);
  }
  final List<String> output = <String>[];
  utf8.decoder.bind(process.stdout).transform(const LineSplitter()).listen(verbose ? printLog : output.add);
  utf8.decoder.bind(process.stderr).transform(const LineSplitter()).listen(verbose ? printLog : output.add);
  final bool success = await process.exitCode == 0;
207
  if (success || silentFailure) {
208
    return success;
209
  }
210
  if (!verbose) {
211
    if (failedCallback != null) {
212
      failedCallback();
213
    }
214 215 216 217 218 219 220 221 222
    print('>> $command');
    output.forEach(printLog);
  }
  return success;
}

void printLog(String line) {
  print('| $line'.trimRight());
}