cocoon_test.dart 13.9 KB
Newer Older
1 2 3 4
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
import 'dart:async';
6 7 8
import 'dart:convert';
import 'dart:io';

9 10 11 12
import 'package:file/file.dart';
import 'package:file/memory.dart';
import 'package:flutter_devicelab/framework/cocoon.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
13 14
import 'package:http/http.dart';
import 'package:http/testing.dart';
15 16 17 18

import 'common.dart';

void main() {
19
  late ProcessResult processResult;
20
  ProcessResult runSyncStub(String executable, List<String> args,
21 22 23 24 25 26
          {Map<String, String>? environment,
          bool includeParentEnvironment = true,
          bool runInShell = false,
          Encoding? stderrEncoding,
          Encoding? stdoutEncoding,
          String? workingDirectory}) =>
27
      processResult;
28 29 30 31 32

  // Expected test values.
  const String commitSha = 'a4952838bf288a81d8ea11edfd4b4cd649fa94cc';
  const String serviceAccountTokenPath = 'test_account_file';
  const String serviceAccountToken = 'test_token';
33

34
  group('Cocoon', () {
35 36 37
    late Client mockClient;
    late Cocoon cocoon;
    late FileSystem fs;
38 39 40

    setUp(() {
      fs = MemoryFileSystem();
41
      mockClient = MockClient((Request request) async => Response('{}', 200));
42 43 44 45 46

      final File serviceAccountFile = fs.file(serviceAccountTokenPath)..createSync();
      serviceAccountFile.writeAsStringSync(serviceAccountToken);
    });

47
    test('returns expected commit sha', () {
48
      processResult = ProcessResult(1, 0, commitSha, '');
49 50
      cocoon = Cocoon(
        serviceAccountTokenPath: serviceAccountTokenPath,
51
        fs: fs,
52 53 54 55 56 57 58 59
        httpClient: mockClient,
        processRunSync: runSyncStub,
      );

      expect(cocoon.commitSha, commitSha);
    });

    test('throws exception on git cli errors', () {
60
      processResult = ProcessResult(1, 1, '', '');
61 62
      cocoon = Cocoon(
        serviceAccountTokenPath: serviceAccountTokenPath,
63
        fs: fs,
64 65 66 67 68 69 70
        httpClient: mockClient,
        processRunSync: runSyncStub,
      );

      expect(() => cocoon.commitSha, throwsA(isA<CocoonException>()));
    });

71
    test('writes expected update task json', () async {
72
      processResult = ProcessResult(1, 0, commitSha, '');
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106
      final TaskResult result = TaskResult.fromJson(<String, dynamic>{
        'success': true,
        'data': <String, dynamic>{
          'i': 0,
          'j': 0,
          'not_a_metric': 'something',
        },
        'benchmarkScoreKeys': <String>['i', 'j'],
      });

      cocoon = Cocoon(
        fs: fs,
        processRunSync: runSyncStub,
      );

      const String resultsPath = 'results.json';
      await cocoon.writeTaskResultToFile(
        builderName: 'builderAbc',
        gitBranch: 'master',
        result: result,
        resultsPath: resultsPath,
      );

      final String resultJson = fs.file(resultsPath).readAsStringSync();
      const String expectedJson = '{'
          '"CommitBranch":"master",'
          '"CommitSha":"$commitSha",'
          '"BuilderName":"builderAbc",'
          '"NewStatus":"Succeeded",'
          '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
          '"BenchmarkScoreKeys":["i","j"]}';
      expect(resultJson, expectedJson);
    });

107
    test('uploads metrics sends expected post body', () async {
108
      processResult = ProcessResult(1, 0, commitSha, '');
109 110
      const String uploadMetricsRequestWithSpaces =
          '{"CommitBranch":"master","CommitSha":"a4952838bf288a81d8ea11edfd4b4cd649fa94cc","BuilderName":"builder a b c","NewStatus":"Succeeded","ResultData":{},"BenchmarkScoreKeys":[],"TestFlaky":false}';
111 112 113 114 115 116
      final MockClient client = MockClient((Request request) async {
        if (request.body == uploadMetricsRequestWithSpaces) {
          return Response('{}', 200);
        }

        return Response('Expected: $uploadMetricsRequestWithSpaces\nReceived: ${request.body}', 500);
117
      });
118 119 120 121 122 123 124 125 126 127 128 129
      cocoon = Cocoon(
        fs: fs,
        httpClient: client,
        processRunSync: runSyncStub,
        serviceAccountTokenPath: serviceAccountTokenPath,
        requestRetryLimit: 0,
      );

      const String resultsPath = 'results.json';
      const String updateTaskJson = '{'
          '"CommitBranch":"master",'
          '"CommitSha":"$commitSha",'
130
          '"BuilderName":"builder a b c",' //ignore: missing_whitespace_between_adjacent_strings
131 132 133 134
          '"NewStatus":"Succeeded",'
          '"ResultData":{},'
          '"BenchmarkScoreKeys":[]}';
      fs.file(resultsPath).writeAsStringSync(updateTaskJson);
135
      await cocoon.sendTaskStatus(resultsPath: resultsPath);
136 137
    });

138
    test('uploads expected update task payload from results file', () async {
139
      processResult = ProcessResult(1, 0, commitSha, '');
140 141 142 143 144
      cocoon = Cocoon(
        fs: fs,
        httpClient: mockClient,
        processRunSync: runSyncStub,
        serviceAccountTokenPath: serviceAccountTokenPath,
145
        requestRetryLimit: 0,
146 147 148 149 150 151 152 153 154 155 156
      );

      const String resultsPath = 'results.json';
      const String updateTaskJson = '{'
          '"CommitBranch":"master",'
          '"CommitSha":"$commitSha",'
          '"BuilderName":"builderAbc",'
          '"NewStatus":"Succeeded",'
          '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
          '"BenchmarkScoreKeys":["i","j"]}';
      fs.file(resultsPath).writeAsStringSync(updateTaskJson);
157
      await cocoon.sendTaskStatus(resultsPath: resultsPath);
158 159
    });

160 161 162 163 164 165 166 167 168 169 170
    test('Verify retries for task result upload', () async {
      int requestCount = 0;
      mockClient = MockClient((Request request) async {
        requestCount++;
        if (requestCount == 1) {
          return Response('{}', 500);
        } else {
          return Response('{}', 200);
        }
      });

171
      processResult = ProcessResult(1, 0, commitSha, '');
172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
      cocoon = Cocoon(
        fs: fs,
        httpClient: mockClient,
        processRunSync: runSyncStub,
        serviceAccountTokenPath: serviceAccountTokenPath,
        requestRetryLimit: 3,
      );

      const String resultsPath = 'results.json';
      const String updateTaskJson = '{'
          '"CommitBranch":"master",'
          '"CommitSha":"$commitSha",'
          '"BuilderName":"builderAbc",'
          '"NewStatus":"Succeeded",'
          '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
          '"BenchmarkScoreKeys":["i","j"]}';
      fs.file(resultsPath).writeAsStringSync(updateTaskJson);
189
      await cocoon.sendTaskStatus(resultsPath: resultsPath);
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
    });

    test('Verify timeout and retry for task result upload', () async {
      int requestCount = 0;
      const int timeoutValue = 2;
      mockClient = MockClient((Request request) async {
        requestCount++;
        if (requestCount == 1) {
          await Future<void>.delayed(const Duration(seconds: timeoutValue + 2));
          throw Exception('Should not reach this, because timeout should trigger');
        } else {
          return Response('{}', 200);
        }
      });

205
      processResult = ProcessResult(1, 0, commitSha, '');
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
      cocoon = Cocoon(
        fs: fs,
        httpClient: mockClient,
        processRunSync: runSyncStub,
        serviceAccountTokenPath: serviceAccountTokenPath,
        requestRetryLimit: 2,
        requestTimeoutLimit: timeoutValue,
      );

      const String resultsPath = 'results.json';
      const String updateTaskJson = '{'
          '"CommitBranch":"master",'
          '"CommitSha":"$commitSha",'
          '"BuilderName":"builderAbc",'
          '"NewStatus":"Succeeded",'
          '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
          '"BenchmarkScoreKeys":["i","j"]}';
      fs.file(resultsPath).writeAsStringSync(updateTaskJson);
224
      await cocoon.sendTaskStatus(resultsPath: resultsPath);
225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
    });

    test('Verify timeout does not trigger for result upload', () async {
      int requestCount = 0;
      const int timeoutValue = 2;
      mockClient = MockClient((Request request) async {
        requestCount++;
        if (requestCount == 1) {
          await Future<void>.delayed(const Duration(seconds: timeoutValue - 1));
          return Response('{}', 200);
        } else {
          throw Exception('This iteration should not be reached, since timeout should not happen.');
        }
      });

240
      processResult = ProcessResult(1, 0, commitSha, '');
241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
      cocoon = Cocoon(
        fs: fs,
        httpClient: mockClient,
        processRunSync: runSyncStub,
        serviceAccountTokenPath: serviceAccountTokenPath,
        requestRetryLimit: 2,
        requestTimeoutLimit: timeoutValue,
      );

      const String resultsPath = 'results.json';
      const String updateTaskJson = '{'
          '"CommitBranch":"master",'
          '"CommitSha":"$commitSha",'
          '"BuilderName":"builderAbc",'
          '"NewStatus":"Succeeded",'
          '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
          '"BenchmarkScoreKeys":["i","j"]}';
      fs.file(resultsPath).writeAsStringSync(updateTaskJson);
259
      await cocoon.sendTaskStatus(resultsPath: resultsPath);
260 261 262 263 264 265 266 267 268 269 270 271 272
    });

    test('Verify failure without retries for task result upload', () async {
      int requestCount = 0;
      mockClient = MockClient((Request request) async {
        requestCount++;
        if (requestCount == 1) {
          return Response('{}', 500);
        } else {
          return Response('{}', 200);
        }
      });

273
      processResult = ProcessResult(1, 0, commitSha, '');
274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
      cocoon = Cocoon(
        fs: fs,
        httpClient: mockClient,
        processRunSync: runSyncStub,
        serviceAccountTokenPath: serviceAccountTokenPath,
        requestRetryLimit: 0,
      );

      const String resultsPath = 'results.json';
      const String updateTaskJson = '{'
          '"CommitBranch":"master",'
          '"CommitSha":"$commitSha",'
          '"BuilderName":"builderAbc",'
          '"NewStatus":"Succeeded",'
          '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
          '"BenchmarkScoreKeys":["i","j"]}';
      fs.file(resultsPath).writeAsStringSync(updateTaskJson);
291
      expect(() => cocoon.sendTaskStatus(resultsPath: resultsPath), throwsA(isA<ClientException>()));
292 293
    });

294 295 296 297 298
    test('throws client exception on non-200 responses', () async {
      mockClient = MockClient((Request request) async => Response('', 500));

      cocoon = Cocoon(
        serviceAccountTokenPath: serviceAccountTokenPath,
299
        fs: fs,
300
        httpClient: mockClient,
301
        requestRetryLimit: 0,
302 303
      );

304 305 306 307 308 309 310 311 312
      const String resultsPath = 'results.json';
      const String updateTaskJson = '{'
          '"CommitBranch":"master",'
          '"CommitSha":"$commitSha",'
          '"BuilderName":"builderAbc",'
          '"NewStatus":"Succeeded",'
          '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
          '"BenchmarkScoreKeys":["i","j"]}';
      fs.file(resultsPath).writeAsStringSync(updateTaskJson);
313
      expect(() => cocoon.sendTaskStatus(resultsPath: resultsPath), throwsA(isA<ClientException>()));
314
    });
315

316
    test('does not upload results on non-supported branches', () async {
317
      // Any network failure would cause the upload to fail
318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
      mockClient = MockClient((Request request) async => Response('', 500));

      cocoon = Cocoon(
        serviceAccountTokenPath: serviceAccountTokenPath,
        fs: fs,
        httpClient: mockClient,
        requestRetryLimit: 0,
      );

      const String resultsPath = 'results.json';
      const String updateTaskJson = '{'
          '"CommitBranch":"stable",'
          '"CommitSha":"$commitSha",'
          '"BuilderName":"builderAbc",'
          '"NewStatus":"Succeeded",'
          '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
          '"BenchmarkScoreKeys":["i","j"]}';
      fs.file(resultsPath).writeAsStringSync(updateTaskJson);

      // This will fail if it decided to upload results
338 339 340 341
      await cocoon.sendTaskStatus(resultsPath: resultsPath);
    });

    test('does not update for staging test', () async {
342
      // Any network failure would cause the upload to fail
343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363
      mockClient = MockClient((Request request) async => Response('', 500));

      cocoon = Cocoon(
        serviceAccountTokenPath: serviceAccountTokenPath,
        fs: fs,
        httpClient: mockClient,
        requestRetryLimit: 0,
      );

      const String resultsPath = 'results.json';
      const String updateTaskJson = '{'
          '"CommitBranch":"master",'
          '"CommitSha":"$commitSha",'
          '"BuilderName":"builderAbc",'
          '"NewStatus":"Succeeded",'
          '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
          '"BenchmarkScoreKeys":["i","j"]}';
      fs.file(resultsPath).writeAsStringSync(updateTaskJson);

      // This will fail if it decided to upload results
      await cocoon.sendTaskStatus(resultsPath: resultsPath, builderBucket: 'staging');
364
    });
365 366 367
  });

  group('AuthenticatedCocoonClient', () {
368
    late FileSystem fs;
369 370 371

    setUp(() {
      fs = MemoryFileSystem();
372
      final File serviceAccountFile = fs.file(serviceAccountTokenPath)..createSync();
373 374 375 376
      serviceAccountFile.writeAsStringSync(serviceAccountToken);
    });

    test('reads token from service account file', () {
377
      final AuthenticatedCocoonClient client = AuthenticatedCocoonClient(serviceAccountTokenPath, filesystem: fs);
378 379 380 381
      expect(client.serviceAccountToken, serviceAccountToken);
    });

    test('reads token from service account file with whitespace', () {
382
      final File serviceAccountFile = fs.file(serviceAccountTokenPath)..createSync();
383
      serviceAccountFile.writeAsStringSync('$serviceAccountToken \n');
384
      final AuthenticatedCocoonClient client = AuthenticatedCocoonClient(serviceAccountTokenPath, filesystem: fs);
385 386 387 388 389 390 391 392 393
      expect(client.serviceAccountToken, serviceAccountToken);
    });

    test('throws error when service account file not found', () {
      final AuthenticatedCocoonClient client = AuthenticatedCocoonClient('idontexist', filesystem: fs);
      expect(() => client.serviceAccountToken, throwsA(isA<FileSystemException>()));
    });
  });
}