[devicelab] Cocoon client (#68333)

dev/devicelab/bin/run.dart

@@ -9,6 +9,7 @@ import 'package:args/args.dart';
 import 'package:path/path.dart' as path;
 
 import 'package:flutter_devicelab/framework/ab.dart';
+import 'package:flutter_devicelab/framework/cocoon.dart';
 import 'package:flutter_devicelab/framework/manifest.dart';
 import 'package:flutter_devicelab/framework/runner.dart';
 import 'package:flutter_devicelab/framework/task_result.dart';
@@ -18,8 +19,8 @@ ArgResults args;
 
 List<String> _taskNames = <String>[];
 
-/// Suppresses standard output, prints only standard error output.
-bool silent;
+/// The device-id to run test on.
+String deviceId;
 
 /// The build of the local engine to use.
 ///
@@ -32,8 +33,13 @@ String localEngineSrcPath;
 /// Whether to exit on first test failure.
 bool exitOnFirstTestFailure;
 
-/// The device-id to run test on.
-String deviceId;
+/// File containing a service account token.
+///
+/// If passed, the test run results will be uploaded to Flutter infrastructure.
+String serviceAccountTokenFile;
+
+/// Suppresses standard output, prints only standard error output.
+bool silent;
 
 /// Runs tasks.
 ///
@@ -74,11 +80,12 @@ Future<void> main(List<String> rawArgs) async {
     return;
   }
 
-  silent = args['silent'] as bool;
+  deviceId = args['device-id'] as String;
   localEngine = args['local-engine'] as String;
   localEngineSrcPath = args['local-engine-src-path'] as String;
   exitOnFirstTestFailure = args['exit'] as bool;
-  deviceId = args['device-id'] as String;
+  serviceAccountTokenFile = args['service-account-token-file'] as String;
+  silent = args['silent'] as bool;
 
   if (args.wasParsed('ab')) {
     await _runABTest();
@@ -102,6 +109,11 @@ Future<void> _runTasks() async {
     print(const JsonEncoder.withIndent('  ').convert(result));
     section('Finished task "$taskName"');
 
+    if (serviceAccountTokenFile != null) {
+      final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
+      await cocoon.sendTaskResult(taskName: taskName, result: result);
+    }
+
     if (!result.succeeded) {
       exitCode = 1;
       if (exitOnFirstTestFailure) {
@@ -334,6 +346,10 @@ final ArgParser _argParser = ArgParser()
           'on a windows host). Each test publishes its '
           '`required_agent_capabilities`\nin the `manifest.yaml` file.',
   )
+  ..addOption(
+    'service-account-token-file',
+    help: '[Flutter infrastructure] Authentication for uploading results.',
+  )
   ..addOption(
     'stage',
     abbr: 's',

dev/devicelab/lib/framework/cocoon.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert' show json;
+import 'dart:io';
+
+import 'package:file/file.dart';
+import 'package:file/local.dart';
+import 'package:http/http.dart';
+import 'package:logging/logging.dart';
+import 'package:meta/meta.dart';
+
+import 'task_result.dart';
+import 'utils.dart';
+
+/// Class for test runner to interact with Flutter's infrastructure service, Cocoon.
+///
+/// Cocoon assigns bots to run these devicelab tasks on real devices.
+/// To retrieve these results, the test runner needs to send results back so so the database can be updated.
+class Cocoon {
+  Cocoon({
+    String serviceAccountTokenPath,
+    @visibleForTesting Client httpClient,
+    @visibleForTesting FileSystem filesystem,
+  }) : _httpClient = AuthenticatedCocoonClient(serviceAccountTokenPath, httpClient: httpClient, filesystem: filesystem);
+
+
+  /// Client to make http requests to Cocoon.
+  final AuthenticatedCocoonClient _httpClient;
+
+  /// Url used to send results to.
+  static const String baseCocoonApiUrl = 'https://flutter-dashboard.appspot.com/api';
+
+  static final Logger logger = Logger('CocoonClient');
+
+  String get commitSha => _commitSha ?? _readCommitSha();
+  String _commitSha;
+
+  /// Parse the local repo for the current running commit.
+  String _readCommitSha() {
+    final ProcessResult result = Process.runSync('git', <String>['rev-parse', 'HEAD']);
+    if (result.exitCode != 0) {
+      throw Exception(result.stderr);
+    }
+
+    _commitSha = result.stdout as String;
+    return _commitSha;
+  }
+
+  /// Send [TaskResult] to Cocoon.
+  Future<void> sendTaskResult({String taskName, TaskResult result}) async {
+    // Skip logging on test runs
+    Logger.root.level = Level.ALL;
+    Logger.root.onRecord.listen((LogRecord rec) {
+      print('${rec.level.name}: ${rec.time}: ${rec.message}');
+    });
+
+    final Map<String, dynamic> status = <String, dynamic>{
+      'CommitSha': commitSha,
+      'TaskName': taskName,
+      'NewStatus': result.succeeded ? 'Succeeded' : 'Failed',
+    };
+
+    // Make a copy of result data because we may alter it for validation below.
+    status['ResultData'] = result.data;
+
+    final List<String> validScoreKeys = <String>[];
+    if (result.benchmarkScoreKeys != null) {
+      for (final String scoreKey in result.benchmarkScoreKeys) {
+        final Object score = result.data[scoreKey];
+        if (score is num) {
+          // Convert all metrics to double, which provide plenty of precision
+          // without having to add support for multiple numeric types in Cocoon.
+          result.data[scoreKey] = score.toDouble();
+          validScoreKeys.add(scoreKey);
+        }
+      }
+    }
+    status['BenchmarkScoreKeys'] = validScoreKeys;
+
+    final Map<String, dynamic> response = await _sendCocoonRequest('update-task-status', status);
+    if (response['Name'] != null) {
+      logger.info('Updated Cocoon with results from this task');
+    } else {
+      logger.info(response);
+      logger.severe('Failed to updated Cocoon with results from this task');
+    }
+  }
+
+  /// Make an API request to Cocoon.
+  Future<Map<String, dynamic>> _sendCocoonRequest(String apiPath, [dynamic jsonData]) async {
+    final String url = '$baseCocoonApiUrl/$apiPath';
+
+    /// Retry requests to Cocoon as sometimes there are issues with the servers, such
+    /// as version changes to the backend, datastore issues, or latency issues.
+    final Response response = await retry(
+      () => _httpClient.post(url, body: json.encode(jsonData)),
+      retryIf: (Exception e) => e is SocketException || e is TimeoutException || e is ClientException,
+      maxAttempts: 5,
+    );
+    return json.decode(response.body) as Map<String, dynamic>;
+  }
+}
+
+/// [HttpClient] for sending authenticated requests to Cocoon.
+class AuthenticatedCocoonClient extends BaseClient {
+  AuthenticatedCocoonClient(
+    this._serviceAccountTokenPath, {
+    @visibleForTesting Client httpClient,
+    @visibleForTesting FileSystem filesystem,
+  })  : _delegate = httpClient ?? Client(),
+        _fs = filesystem ?? const LocalFileSystem();
+
+  /// Authentication token to have the ability to upload and record test results.
+  ///
+  /// This is intended to only be passed on automated runs on LUCI post-submit.
+  final String _serviceAccountTokenPath;
+
+  /// Underlying [HttpClient] to send requests to.
+  final Client _delegate;
+
+  /// Underlying [FileSystem] to use.
+  final FileSystem _fs;
+
+  /// Value contained in the service account token file that can be used in http requests.
+  String get serviceAccountToken => _serviceAccountToken ?? _readServiceAccountTokenFile();
+  String _serviceAccountToken;
+
+  /// Get [serviceAccountToken] from the given service account file.
+  String _readServiceAccountTokenFile() {
+    return _serviceAccountToken = _fs.file(_serviceAccountTokenPath).readAsStringSync().trim();
+  }
+
+  @override
+  Future<StreamedResponse> send(BaseRequest request) async {
+    request.headers['Service-Account-Token'] = serviceAccountToken;
+    final StreamedResponse response = await _delegate.send(request);
+
+    if (response.statusCode != 200) {
+      throw ClientException(
+          'AuthenticatedClientError:\n'
+          '  URI: ${request.url}\n'
+          '  HTTP Status: ${response.statusCode}\n'
+          '  Response body:\n'
+          '${(await Response.fromStream(response)).body}',
+          request.url);
+    }
+    return response;
+  }
+}

dev/devicelab/lib/framework/utils.dart

@@ -736,3 +736,34 @@ Future<int> gitClone({String path, String repo}) async {
         () => exec('git', <String>['clone', repo]),
   );
 }
+
+/// Call [fn] retrying so long as [retryIf] return `true` for the exception
+/// thrown and [maxAttempts] has not been reached.
+///
+/// If no [retryIf] function is given this will retry any for any [Exception]
+/// thrown. To retry on an [Error], the error must be caught and _rethrown_
+/// as an [Exception].
+///
+/// Waits a constant duration of [delayDuration] between every retry attempt.
+Future<T> retry<T>(
+  FutureOr<T> Function() fn, {
+  FutureOr<bool> Function(Exception) retryIf,
+  int maxAttempts = 5,
+  Duration delayDuration = const Duration(seconds: 3),
+}) async {
+  int attempt = 0;
+  while (true) {
+    attempt++; // first invocation is the first attempt
+    try {
+      return await fn();
+    } on Exception catch (e) {
+      if (attempt >= maxAttempts ||
+          (retryIf != null && !(await retryIf(e)))) {
+        rethrow;
+      }
+    }
+
+    // Sleep for a delay
+    await Future<void>.delayed(delayDuration);
+  }
+}

dev/devicelab/test/cocoon_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:file/file.dart';
+import 'package:file/memory.dart';
+import 'package:http/http.dart';
+import 'package:http/testing.dart';
+
+import 'package:flutter_devicelab/framework/cocoon.dart';
+import 'package:flutter_devicelab/framework/task_result.dart';
+
+import 'common.dart';
+
+void main() {
+  group('Cocoon', () {
+    const String serviceAccountTokenPath = 'test_account_file';
+    const String serviceAccountToken = 'test_token';
+
+    Client mockClient;
+    Cocoon cocoon;
+    FileSystem fs;
+
+    setUp(() {
+      fs = MemoryFileSystem();
+
+      final File serviceAccountFile = fs.file(serviceAccountTokenPath)..createSync();
+      serviceAccountFile.writeAsStringSync(serviceAccountToken);
+    });
+
+    test('sends expected request from successful task', () async {
+      mockClient = MockClient((Request request) async => Response('{}', 200));
+
+      cocoon = Cocoon(
+        serviceAccountTokenPath: serviceAccountTokenPath,
+        filesystem: fs,
+        httpClient: mockClient,
+      );
+
+      final TaskResult result = TaskResult.success(<String, dynamic>{});
+      // This should not throw an error.
+      await cocoon.sendTaskResult(taskName: 'taskAbc', result: result);
+    });
+
+    test('throws client exception on non-200 responses', () async {
+      mockClient = MockClient((Request request) async => Response('', 500));
+
+      cocoon = Cocoon(
+        serviceAccountTokenPath: serviceAccountTokenPath,
+        filesystem: fs,
+        httpClient: mockClient,
+      );
+
+      final TaskResult result = TaskResult.success(<String, dynamic>{});
+      expect(() => cocoon.sendTaskResult(taskName: 'taskAbc', result: result), throwsA(isA<ClientException>()));
+    });
+  });
+
+  group('AuthenticatedCocoonClient', () {
+    const String serviceAccountPath = 'test_account_file';
+    const String serviceAccountToken = 'test_token';
+
+    FileSystem fs;
+
+    setUp(() {
+      fs = MemoryFileSystem();
+      final File serviceAccountFile = fs.file(serviceAccountPath)..createSync();
+      serviceAccountFile.writeAsStringSync(serviceAccountToken);
+    });
+
+    test('reads token from service account file', () {
+      final AuthenticatedCocoonClient client = AuthenticatedCocoonClient(serviceAccountPath, filesystem: fs);
+      expect(client.serviceAccountToken, serviceAccountToken);
+    });
+
+    test('reads token from service account file with whitespace', () {
+      final File serviceAccountFile = fs.file(serviceAccountPath)..createSync();
+      serviceAccountFile.writeAsStringSync(serviceAccountToken + ' \n');
+      final AuthenticatedCocoonClient client = AuthenticatedCocoonClient(serviceAccountPath, filesystem: fs);
+      expect(client.serviceAccountToken, serviceAccountToken);
+    });
+
+    test('throws error when service account file not found', () {
+      final AuthenticatedCocoonClient client = AuthenticatedCocoonClient('idontexist', filesystem: fs);
+      expect(() => client.serviceAccountToken, throwsA(isA<FileSystemException>()));
+    });
+  });
+}

dev/devicelab/test/run_test.dart

@@ -138,5 +138,14 @@ void main() {
         ),
       );
     });
+
+    test('fails to upload results to Cocoon if flags given', () async {
+      // CocoonClient will fail to find test-file, and will not send any http requests.
+      final ProcessResult result = await runScript(
+        <String>['smoke_test_success'],
+        <String>['--service-account-file=test-file', '--task-key=task123'],
+      );
+      expect(result.exitCode, 1);
+    });
   });
 }