runner.dart 10.5 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6 7 8
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io';

9
import 'package:meta/meta.dart';
10
import 'package:vm_service/vm_service.dart';
11

12
import 'cocoon.dart';
13
import 'devices.dart';
14
import 'task_result.dart';
15 16
import 'utils.dart';

17 18 19 20 21 22 23 24 25 26
/// Run a list of tasks.
///
/// For each task, an auto rerun will be triggered when task fails.
///
/// If the task succeeds the first time, it will be recorded as successful.
///
/// If the task fails first, but gets passed in the end, the
/// test will be recorded as successful but with a flake flag.
///
/// If the task fails all reruns, it will be recorded as failed.
27 28 29
Future<void> runTasks(
  List<String> taskNames, {
  bool exitOnFirstTestFailure = false,
30 31 32 33
  // terminateStrayDartProcesses defaults to false so that tests don't have to specify it.
  // It is set based on the --terminate-stray-dart-processes command line argument in
  // normal execution, and that flag defaults to true.
  bool terminateStrayDartProcesses = false,
34
  bool silent = false,
35 36 37
  String? deviceId,
  String? gitBranch,
  String? localEngine,
38
  String? localEngineHost,
39 40 41 42
  String? localEngineSrcPath,
  String? luciBuilder,
  String? resultsPath,
  List<String>? taskArgs,
43
  bool useEmulator = false,
44 45 46
  @visibleForTesting Map<String, String>? isolateParams,
  @visibleForTesting Function(String) print = print,
  @visibleForTesting List<String>? logs,
47 48
}) async {
  for (final String taskName in taskNames) {
49
    TaskResult result = TaskResult.success(null);
50 51
    int failureCount = 0;
    while (failureCount <= Cocoon.retryNumber) {
52 53 54 55
      result = await rerunTask(
        taskName,
        deviceId: deviceId,
        localEngine: localEngine,
56
        localEngineHost: localEngineHost,
57
        localEngineSrcPath: localEngineSrcPath,
58
        terminateStrayDartProcesses: terminateStrayDartProcesses,
59 60
        silent: silent,
        taskArgs: taskArgs,
61
        resultsPath: resultsPath,
62 63 64
        gitBranch: gitBranch,
        luciBuilder: luciBuilder,
        isolateParams: isolateParams,
65
        useEmulator: useEmulator,
66
      );
67 68

      if (!result.succeeded) {
69 70 71 72
        failureCount += 1;
        if (exitOnFirstTestFailure) {
          break;
        }
73
      } else {
74
        section('Flaky status for "$taskName"');
75 76
        if (failureCount > 0) {
          print('Total ${failureCount+1} executions: $failureCount failures and 1 false positive.');
77
          print('flaky: true');
78 79
          // TODO(ianh): stop ignoring this failure. We should set exitCode=1, and quit
          // if exitOnFirstTestFailure is true.
80
        } else {
81
          print('Test passed on first attempt.');
82 83 84 85
          print('flaky: false');
        }
        break;
      }
86 87 88
    }

    if (!result.succeeded) {
89
      section('Flaky status for "$taskName"');
90
      print('Consistently failed across all $failureCount executions.');
91
      print('flaky: false');
92 93 94 95 96 97 98
      exitCode = 1;
      if (exitOnFirstTestFailure) {
        return;
      }
    }
  }
}
99

100 101 102 103 104 105 106
/// A rerun wrapper for `runTask`.
///
/// This separates reruns in separate sections.
Future<TaskResult> rerunTask(
  String taskName, {
  String? deviceId,
  String? localEngine,
107
  String? localEngineHost,
108
  String? localEngineSrcPath,
109
  bool terminateStrayDartProcesses = false,
110 111 112 113 114
  bool silent = false,
  List<String>? taskArgs,
  String? resultsPath,
  String? gitBranch,
  String? luciBuilder,
115
  bool useEmulator = false,
116 117 118 119 120 121 122
  @visibleForTesting Map<String, String>? isolateParams,
}) async {
  section('Running task "$taskName"');
  final TaskResult result = await runTask(
    taskName,
    deviceId: deviceId,
    localEngine: localEngine,
123
    localEngineHost: localEngineHost,
124
    localEngineSrcPath: localEngineSrcPath,
125
    terminateStrayDartProcesses: terminateStrayDartProcesses,
126 127 128
    silent: silent,
    taskArgs: taskArgs,
    isolateParams: isolateParams,
129
    useEmulator: useEmulator,
130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
  );

  print('Task result:');
  print(const JsonEncoder.withIndent('  ').convert(result));
  section('Finished task "$taskName"');

  if (resultsPath != null) {
    final Cocoon cocoon = Cocoon();
    await cocoon.writeTaskResultToFile(
      builderName: luciBuilder,
      gitBranch: gitBranch,
      result: result,
      resultsPath: resultsPath,
    );
  }
  return result;
}

148 149 150 151 152
/// Runs a task in a separate Dart VM and collects the result using the VM
/// service protocol.
///
/// [taskName] is the name of the task. The corresponding task executable is
/// expected to be found under `bin/tasks`.
153 154 155
///
/// Running the task in [silent] mode will suppress standard output from task
/// processes and only print standard errors.
156 157
///
/// [taskArgs] are passed to the task executable for additional configuration.
158
Future<TaskResult> runTask(
159
  String taskName, {
160
  bool terminateStrayDartProcesses = false,
161
  bool silent = false,
162
  String? localEngine,
163
  String? localEngineHost,
164
  String? localWebSdk,
165 166
  String? localEngineSrcPath,
  String? deviceId,
167
  List<String>? taskArgs,
168
  bool useEmulator = false,
169
  @visibleForTesting Map<String, String>? isolateParams,
170
}) async {
171
  final String taskExecutable = 'bin/tasks/$taskName.dart';
172

173
  if (!file(taskExecutable).existsSync()) {
174
    throw 'Executable Dart file not found: $taskExecutable';
175
  }
176

177 178 179 180 181 182 183
  if (useEmulator) {
    taskArgs ??= <String>[];
    taskArgs
      ..add('--android-emulator')
      ..add('--browser-name=android-chrome');
  }

184 185
  stdout.writeln('Starting process for task: [$taskName]');

186 187 188
  final Process runner = await startProcess(
    dartBin,
    <String>[
189
      '--disable-dart-dev',
190 191 192
      '--enable-vm-service=0', // zero causes the system to choose a free port
      '--no-pause-isolates-on-exit',
      if (localEngine != null) '-DlocalEngine=$localEngine',
193
      if (localEngineHost != null) '-DlocalEngineHost=$localEngineHost',
194
      if (localWebSdk != null) '-DlocalWebSdk=$localWebSdk',
195 196
      if (localEngineSrcPath != null) '-DlocalEngineSrcPath=$localEngineSrcPath',
      taskExecutable,
197
      ...?taskArgs,
198 199
    ],
    environment: <String, String>{
200 201
      if (deviceId != null)
        DeviceIdEnvName: deviceId,
202 203
    },
  );
204 205 206

  bool runnerFinished = false;

207
  unawaited(runner.exitCode.whenComplete(() {
208
    runnerFinished = true;
209
  }));
210

211 212
  final Completer<Uri> uri = Completer<Uri>();

213
  final StreamSubscription<String> stdoutSub = runner.stdout
214 215
      .transform<String>(const Utf8Decoder())
      .transform<String>(const LineSplitter())
216
      .listen((String line) {
217
    if (!uri.isCompleted) {
218
      final Uri? serviceUri = parseServiceUri(line, prefix: RegExp('The Dart VM service is listening on '));
219
      if (serviceUri != null) {
220
        uri.complete(serviceUri);
221
      }
222
    }
223
    if (!silent) {
224
      stdout.writeln('[${DateTime.now()}] [STDOUT] $line');
225
    }
226 227
  });

228
  final StreamSubscription<String> stderrSub = runner.stderr
229 230
      .transform<String>(const Utf8Decoder())
      .transform<String>(const LineSplitter())
231
      .listen((String line) {
232
    stderr.writeln('[${DateTime.now()}] [STDERR] $line');
233 234 235
  });

  try {
236
    final ConnectionResult result = await _connectToRunnerIsolate(await uri.future);
237 238 239
    print('[$taskName] Connected to VM server.');
    isolateParams = isolateParams == null ? <String, String>{} : Map<String, String>.of(isolateParams);
    isolateParams['runProcessCleanup'] = terminateStrayDartProcesses.toString();
240 241 242 243
    final Map<String, dynamic> taskResultJson = (await result.vmService.callServiceExtension(
      'ext.cocoonRunTask',
      args: isolateParams,
      isolateId: result.isolate.id,
244
    )).json!;
245
    final TaskResult taskResult = TaskResult.fromJson(taskResultJson);
246 247
    final int exitCode = await runner.exitCode;
    print('[$taskName] Process terminated with exit code $exitCode.');
248
    return taskResult;
249 250 251
  } catch (error, stack) {
    print('[$taskName] Task runner system failed with exception!\n$error\n$stack');
    rethrow;
252
  } finally {
253 254
    if (!runnerFinished) {
      print('[$taskName] Terminating process...');
255
      runner.kill(ProcessSignal.sigkill);
256
    }
257 258 259 260 261
    await stdoutSub.cancel();
    await stderrSub.cancel();
  }
}

262
Future<ConnectionResult> _connectToRunnerIsolate(Uri vmServiceUri) async {
263 264 265 266 267
  final List<String> pathSegments = <String>[
    // Add authentication code.
    if (vmServiceUri.pathSegments.isNotEmpty) vmServiceUri.pathSegments[0],
    'ws',
  ];
268
  final String url = vmServiceUri.replace(scheme: 'ws', pathSegments: pathSegments).toString();
269 270 271 272 273 274 275 276
  final Stopwatch stopwatch = Stopwatch()..start();

  while (true) {
    try {
      // Make sure VM server is up by successfully opening and closing a socket.
      await (await WebSocket.connect(url)).close();

      // Look up the isolate.
277 278
      final VmService client = await vmServiceConnectUri(url);
      VM vm = await client.getVM();
279
      while (vm.isolates!.isEmpty) {
280 281 282
        await Future<void>.delayed(const Duration(seconds: 1));
        vm = await client.getVM();
      }
283
      final IsolateRef isolate = vm.isolates!.first;
284
      final Response response = await client.callServiceExtension('ext.cocoonRunnerReady', isolateId: isolate.id);
285
      if (response.json!['response'] != 'ready') {
286
        throw 'not ready yet';
287
      }
288
      return ConnectionResult(client, isolate);
289
    } catch (error) {
290
      if (stopwatch.elapsed > const Duration(seconds: 10)) {
291
        print('VM service still not ready after ${stopwatch.elapsed}: $error\nContinuing to retry...');
292
      }
293
      await Future<void>.delayed(const Duration(milliseconds: 50));
294 295 296
    }
  }
}
297 298 299 300 301 302 303 304 305

class ConnectionResult {
  ConnectionResult(this.vmService, this.isolate);

  final VmService vmService;
  final IsolateRef isolate;
}

/// The cocoon client sends an invalid VM service response, we need to intercept it.
306
Future<VmService> vmServiceConnectUri(String wsUri, {Log? log}) async {
307 308 309 310 311 312 313 314 315 316 317 318 319
  final WebSocket socket = await WebSocket.connect(wsUri);
  final StreamController<dynamic> controller = StreamController<dynamic>();
  final Completer<dynamic> streamClosedCompleter = Completer<dynamic>();
  socket.listen(
    (dynamic data) {
      final Map<String, dynamic> rawData = json.decode(data as String) as Map<String, dynamic> ;
      if (rawData['result'] == 'ready') {
        rawData['result'] = <String, dynamic>{'response': 'ready'};
        controller.add(json.encode(rawData));
      } else {
        controller.add(data);
      }
    },
320
    onError: (Object err, StackTrace stackTrace) => controller.addError(err, stackTrace),
321 322 323 324 325 326 327 328 329 330 331
    onDone: () => streamClosedCompleter.complete(),
  );

  return VmService(
    controller.stream,
    (String message) => socket.add(message),
    log: log,
    disposeHandler: () => socket.close(),
    streamClosed: streamClosedCompleter.future,
  );
}