cocoon.dart 8.95 KB
Newer Older
1 2 3 4 5
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
6
import 'dart:convert' show Encoding, json;
7 8 9 10 11 12 13 14 15 16 17
import 'dart:io';

import 'package:file/file.dart';
import 'package:file/local.dart';
import 'package:http/http.dart';
import 'package:logging/logging.dart';
import 'package:meta/meta.dart';

import 'task_result.dart';
import 'utils.dart';

18 19 20
typedef ProcessRunSync = ProcessResult Function(
  String,
  List<String>, {
21
  Map<String, String>? environment,
22 23
  bool includeParentEnvironment,
  bool runInShell,
24 25 26
  Encoding? stderrEncoding,
  Encoding? stdoutEncoding,
  String? workingDirectory,
27 28
});

29 30 31
/// Class for test runner to interact with Flutter's infrastructure service, Cocoon.
///
/// Cocoon assigns bots to run these devicelab tasks on real devices.
nt4f04uNd's avatar
nt4f04uNd committed
32
/// To retrieve these results, the test runner needs to send results back so the database can be updated.
33 34
class Cocoon {
  Cocoon({
35 36
    String? serviceAccountTokenPath,
    @visibleForTesting Client? httpClient,
37
    @visibleForTesting this.fs = const LocalFileSystem(),
38
    @visibleForTesting this.processRunSync = Process.runSync,
39
    @visibleForTesting this.requestRetryLimit = 5,
40
    @visibleForTesting this.requestTimeoutLimit = 30,
41
  }) : _httpClient = AuthenticatedCocoonClient(serviceAccountTokenPath, httpClient: httpClient, filesystem: fs);
42 43 44 45

  /// Client to make http requests to Cocoon.
  final AuthenticatedCocoonClient _httpClient;

46 47
  final ProcessRunSync processRunSync;

48 49 50
  /// Url used to send results to.
  static const String baseCocoonApiUrl = 'https://flutter-dashboard.appspot.com/api';

51 52 53
  /// Threshold to auto retry a failed test.
  static const int retryNumber = 2;

54 55 56
  /// Underlying [FileSystem] to use.
  final FileSystem fs;

57 58
  static final Logger logger = Logger('CocoonClient');

59 60 61
  @visibleForTesting
  final int requestRetryLimit;

62 63 64
  @visibleForTesting
  final int requestTimeoutLimit;

65
  String get commitSha => _commitSha ?? _readCommitSha();
66
  String? _commitSha;
67 68 69

  /// Parse the local repo for the current running commit.
  String _readCommitSha() {
70 71 72 73 74 75 76 77
    final ProcessResult result = processRunSync('git', <String>['rev-parse', 'HEAD']);
    if (result.exitCode != 0) {
      throw CocoonException(result.stderr as String);
    }

    return _commitSha = result.stdout as String;
  }

78
  /// Update test status to Cocoon.
79 80
  ///
  /// Flutter infrastructure's workflow is:
81
  /// 1. Run DeviceLab test
82
  /// 2. Request service account token from luci auth (valid for at least 3 minutes)
83
  /// 3. Update test status from (1) to Cocoon
84 85 86
  ///
  /// The `resultsPath` is not available for all tests. When it doesn't show up, we
  /// need to append `CommitBranch`, `CommitSha`, and `BuilderName`.
87
  Future<void> sendTaskStatus({
88 89 90 91 92
    String? resultsPath,
    bool? isTestFlaky,
    String? gitBranch,
    String? builderName,
    String? testStatus,
93
    String? builderBucket,
94
  }) async {
95 96 97 98 99 100 101 102 103 104 105
    Map<String, dynamic> resultsJson = <String, dynamic>{};
    if (resultsPath != null) {
      final File resultFile = fs.file(resultsPath);
      resultsJson = json.decode(await resultFile.readAsString()) as Map<String, dynamic>;
    } else {
      resultsJson['CommitBranch'] = gitBranch;
      resultsJson['CommitSha'] = commitSha;
      resultsJson['BuilderName'] = builderName;
      resultsJson['NewStatus'] = testStatus;
    }
    resultsJson['TestFlaky'] = isTestFlaky ?? false;
106
    if (_shouldUpdateCocoon(resultsJson, builderBucket ?? 'prod')) {
107 108 109 110 111
      await retry(
        () async => _sendUpdateTaskRequest(resultsJson).timeout(Duration(seconds: requestTimeoutLimit)),
        retryIf: (Exception e) => e is SocketException || e is TimeoutException || e is ClientException,
        maxAttempts: requestRetryLimit,
      );
112
    }
113 114
  }

115 116 117 118 119 120
  /// Only post-submit tests on `master` are allowed to update in cocoon.
  bool _shouldUpdateCocoon(Map<String, dynamic> resultJson, String builderBucket) {
    const List<String> supportedBranches = <String>['master'];
    return supportedBranches.contains(resultJson['CommitBranch']) && builderBucket == 'prod';
  }

121 122
  /// Write the given parameters into an update task request and store the JSON in [resultsPath].
  Future<void> writeTaskResultToFile({
123 124 125 126
    String? builderName,
    String? gitBranch,
    required TaskResult result,
    required String resultsPath,
127 128 129 130 131 132 133 134 135 136
  }) async {
    final Map<String, dynamic> updateRequest = _constructUpdateRequest(
      gitBranch: gitBranch,
      builderName: builderName,
      result: result,
    );
    final File resultFile = fs.file(resultsPath);
    if (resultFile.existsSync()) {
      resultFile.deleteSync();
    }
137
    logger.fine('Writing results: ${json.encode(updateRequest)}');
138 139 140 141 142
    resultFile.createSync();
    resultFile.writeAsStringSync(json.encode(updateRequest));
  }

  Map<String, dynamic> _constructUpdateRequest({
143 144 145
    String? builderName,
    required TaskResult result,
    String? gitBranch,
146 147
  }) {
    final Map<String, dynamic> updateRequest = <String, dynamic>{
148
      'CommitBranch': gitBranch,
149
      'CommitSha': commitSha,
150
      'BuilderName': builderName,
151 152
      'NewStatus': result.succeeded ? 'Succeeded' : 'Failed',
    };
153
    logger.fine('Update request: $updateRequest');
154 155

    // Make a copy of result data because we may alter it for validation below.
156
    updateRequest['ResultData'] = result.data;
157 158 159

    final List<String> validScoreKeys = <String>[];
    if (result.benchmarkScoreKeys != null) {
160 161
      for (final String scoreKey in result.benchmarkScoreKeys!) {
        final Object score = result.data![scoreKey] as Object;
162 163 164
        if (score is num) {
          // Convert all metrics to double, which provide plenty of precision
          // without having to add support for multiple numeric types in Cocoon.
165
          result.data![scoreKey] = score.toDouble();
166 167 168 169
          validScoreKeys.add(scoreKey);
        }
      }
    }
170 171 172 173
    updateRequest['BenchmarkScoreKeys'] = validScoreKeys;

    return updateRequest;
  }
174

175
  Future<void> _sendUpdateTaskRequest(Map<String, dynamic> postBody) async {
176
    logger.info('Attempting to send update task request to Cocoon.');
177
    final Map<String, dynamic> response = await _sendCocoonRequest('update-task-status', postBody);
178 179 180 181 182 183 184 185 186 187
    if (response['Name'] != null) {
      logger.info('Updated Cocoon with results from this task');
    } else {
      logger.info(response);
      logger.severe('Failed to updated Cocoon with results from this task');
    }
  }

  /// Make an API request to Cocoon.
  Future<Map<String, dynamic>> _sendCocoonRequest(String apiPath, [dynamic jsonData]) async {
188
    final Uri url = Uri.parse('$baseCocoonApiUrl/$apiPath');
189 190 191 192 193 194

    /// Retry requests to Cocoon as sometimes there are issues with the servers, such
    /// as version changes to the backend, datastore issues, or latency issues.
    final Response response = await retry(
      () => _httpClient.post(url, body: json.encode(jsonData)),
      retryIf: (Exception e) => e is SocketException || e is TimeoutException || e is ClientException,
195
      maxAttempts: requestRetryLimit,
196 197 198 199 200 201 202 203 204
    );
    return json.decode(response.body) as Map<String, dynamic>;
  }
}

/// [HttpClient] for sending authenticated requests to Cocoon.
class AuthenticatedCocoonClient extends BaseClient {
  AuthenticatedCocoonClient(
    this._serviceAccountTokenPath, {
205 206
    @visibleForTesting Client? httpClient,
    @visibleForTesting FileSystem? filesystem,
207 208 209 210 211 212
  })  : _delegate = httpClient ?? Client(),
        _fs = filesystem ?? const LocalFileSystem();

  /// Authentication token to have the ability to upload and record test results.
  ///
  /// This is intended to only be passed on automated runs on LUCI post-submit.
213
  final String? _serviceAccountTokenPath;
214 215 216 217 218 219 220 221 222

  /// Underlying [HttpClient] to send requests to.
  final Client _delegate;

  /// Underlying [FileSystem] to use.
  final FileSystem _fs;

  /// Value contained in the service account token file that can be used in http requests.
  String get serviceAccountToken => _serviceAccountToken ?? _readServiceAccountTokenFile();
223
  String? _serviceAccountToken;
224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246

  /// Get [serviceAccountToken] from the given service account file.
  String _readServiceAccountTokenFile() {
    return _serviceAccountToken = _fs.file(_serviceAccountTokenPath).readAsStringSync().trim();
  }

  @override
  Future<StreamedResponse> send(BaseRequest request) async {
    request.headers['Service-Account-Token'] = serviceAccountToken;
    final StreamedResponse response = await _delegate.send(request);

    if (response.statusCode != 200) {
      throw ClientException(
          'AuthenticatedClientError:\n'
          '  URI: ${request.url}\n'
          '  HTTP Status: ${response.statusCode}\n'
          '  Response body:\n'
          '${(await Response.fromStream(response)).body}',
          request.url);
    }
    return response;
  }
}
247 248 249 250 251 252 253 254 255 256

class CocoonException implements Exception {
  CocoonException(this.message) : assert(message != null);

  /// The message to show to the issuer to explain the error.
  final String message;

  @override
  String toString() => 'CocoonException: $message';
}