Unverified Commit b67e2642 authored by keyonghan's avatar keyonghan Committed by GitHub

Revert "Skip staging test update to cocoon in test runner (#88835)" (#88971)

This reverts commit 884dfc26.
parent b0950fd5
...@@ -41,14 +41,21 @@ class UploadResultsCommand extends Command<void> { ...@@ -41,14 +41,21 @@ class UploadResultsCommand extends Command<void> {
final String? testStatus = argResults!['test-status'] as String?; final String? testStatus = argResults!['test-status'] as String?;
final String? commitTime = argResults!['commit-time'] as String?; final String? commitTime = argResults!['commit-time'] as String?;
// Upload metrics to skia perf from test runner when `resultsPath` is specified. // Upload metrics to metrics_center from test runner when `commitTime` is specified. This
if (resultsPath != null) { // is mainly for testing purpose.
await uploadToSkiaPerf(resultsPath, commitTime); // The upload step will be skipped from cocoon once this is validated.
print('Successfully uploaded metrics to skia perf'); // TODO(keyong): remove try block to block test when this is validated to work https://github.com/flutter/flutter/issues/88484
try {
if (commitTime != null) {
await uploadToMetricsCenter(resultsPath, commitTime);
print('Successfully uploaded metrics to metrics center');
}
} on Exception catch (e, stacktrace) {
print('Uploading metrics failure: $e\n\n$stacktrace');
} }
final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile); final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
return cocoon.sendTaskStatus( return cocoon.sendResultsPath(
resultsPath: resultsPath, resultsPath: resultsPath,
isTestFlaky: testFlakyStatus == 'True', isTestFlaky: testFlakyStatus == 'True',
gitBranch: gitBranch, gitBranch: gitBranch,
......
...@@ -75,16 +75,16 @@ class Cocoon { ...@@ -75,16 +75,16 @@ class Cocoon {
return _commitSha = result.stdout as String; return _commitSha = result.stdout as String;
} }
/// Update test status to Cocoon. /// Upload the JSON results in [resultsPath] to Cocoon.
/// ///
/// Flutter infrastructure's workflow is: /// Flutter infrastructure's workflow is:
/// 1. Run DeviceLab test /// 1. Run DeviceLab test, writing results to a known path
/// 2. Request service account token from luci auth (valid for at least 3 minutes) /// 2. Request service account token from luci auth (valid for at least 3 minutes)
/// 3. Update test status from (1) to Cocoon /// 3. Upload results from (1) to Cocoon
/// ///
/// The `resultsPath` is not available for all tests. When it doesn't show up, we /// The `resultsPath` is not available for all tests. When it doesn't show up, we
/// need to append `CommitBranch`, `CommitSha`, and `BuilderName`. /// need to append `CommitBranch`, `CommitSha`, and `BuilderName`.
Future<void> sendTaskStatus({ Future<void> sendResultsPath({
String? resultsPath, String? resultsPath,
bool? isTestFlaky, bool? isTestFlaky,
String? gitBranch, String? gitBranch,
...@@ -102,7 +102,8 @@ class Cocoon { ...@@ -102,7 +102,8 @@ class Cocoon {
resultsJson['NewStatus'] = testStatus; resultsJson['NewStatus'] = testStatus;
} }
resultsJson['TestFlaky'] = isTestFlaky ?? false; resultsJson['TestFlaky'] = isTestFlaky ?? false;
if (_shouldUpdateCocoon(resultsJson)) { const List<String> supportedBranches = <String>['master'];
if (supportedBranches.contains(resultsJson['CommitBranch'])) {
await retry( await retry(
() async => _sendUpdateTaskRequest(resultsJson).timeout(Duration(seconds: requestTimeoutLimit)), () async => _sendUpdateTaskRequest(resultsJson).timeout(Duration(seconds: requestTimeoutLimit)),
retryIf: (Exception e) => e is SocketException || e is TimeoutException || e is ClientException, retryIf: (Exception e) => e is SocketException || e is TimeoutException || e is ClientException,
...@@ -111,13 +112,6 @@ class Cocoon { ...@@ -111,13 +112,6 @@ class Cocoon {
} }
} }
/// Only post-submit tests on `master` are allowed to update in cocoon.
bool _shouldUpdateCocoon(Map<String, dynamic> resultJson) {
const List<String> supportedBranches = <String>['master'];
return supportedBranches.contains(resultJson['CommitBranch']) &&
!resultJson['BuilderName'].toString().toLowerCase().contains('staging');
}
/// Write the given parameters into an update task request and store the JSON in [resultsPath]. /// Write the given parameters into an update task request and store the JSON in [resultsPath].
Future<void> writeTaskResultToFile({ Future<void> writeTaskResultToFile({
String? builderName, String? builderName,
......
...@@ -45,7 +45,6 @@ Future<FlutterDestination> connectFlutterDestination() async { ...@@ -45,7 +45,6 @@ Future<FlutterDestination> connectFlutterDestination() async {
/// ] /// ]
/// } /// }
List<MetricPoint> parse(Map<String, dynamic> resultsJson) { List<MetricPoint> parse(Map<String, dynamic> resultsJson) {
print('Results to upload to skia perf: $resultsJson');
final List<String> scoreKeys = final List<String> scoreKeys =
(resultsJson['BenchmarkScoreKeys'] as List<dynamic>?)?.cast<String>() ?? const <String>[]; (resultsJson['BenchmarkScoreKeys'] as List<dynamic>?)?.cast<String>() ?? const <String>[];
final Map<String, dynamic> resultData = final Map<String, dynamic> resultData =
...@@ -71,13 +70,8 @@ List<MetricPoint> parse(Map<String, dynamic> resultsJson) { ...@@ -71,13 +70,8 @@ List<MetricPoint> parse(Map<String, dynamic> resultsJson) {
return metricPoints; return metricPoints;
} }
/// Upload JSON results to skia perf. /// Upload test metrics to metrics center.
/// Future<void> uploadToMetricsCenter(String? resultsPath, String? commitTime) async {
/// Flutter infrastructure's workflow is:
/// 1. Run DeviceLab test, writing results to a known path
/// 2. Request service account token from luci auth (valid for at least 3 minutes)
/// 3. Upload results from (1) to skia perf.
Future<void> uploadToSkiaPerf(String? resultsPath, String? commitTime) async {
int commitTimeSinceEpoch; int commitTimeSinceEpoch;
if (resultsPath == null) { if (resultsPath == null) {
return; return;
......
...@@ -132,7 +132,7 @@ void main() { ...@@ -132,7 +132,7 @@ void main() {
'"ResultData":{},' '"ResultData":{},'
'"BenchmarkScoreKeys":[]}'; '"BenchmarkScoreKeys":[]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson); fs.file(resultsPath).writeAsStringSync(updateTaskJson);
await cocoon.sendTaskStatus(resultsPath: resultsPath); await cocoon.sendResultsPath(resultsPath: resultsPath);
}); });
test('uploads expected update task payload from results file', () async { test('uploads expected update task payload from results file', () async {
...@@ -154,7 +154,7 @@ void main() { ...@@ -154,7 +154,7 @@ void main() {
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}'; '"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson); fs.file(resultsPath).writeAsStringSync(updateTaskJson);
await cocoon.sendTaskStatus(resultsPath: resultsPath); await cocoon.sendResultsPath(resultsPath: resultsPath);
}); });
test('Verify retries for task result upload', () async { test('Verify retries for task result upload', () async {
...@@ -186,7 +186,7 @@ void main() { ...@@ -186,7 +186,7 @@ void main() {
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}'; '"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson); fs.file(resultsPath).writeAsStringSync(updateTaskJson);
await cocoon.sendTaskStatus(resultsPath: resultsPath); await cocoon.sendResultsPath(resultsPath: resultsPath);
}); });
test('Verify timeout and retry for task result upload', () async { test('Verify timeout and retry for task result upload', () async {
...@@ -221,7 +221,7 @@ void main() { ...@@ -221,7 +221,7 @@ void main() {
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}'; '"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson); fs.file(resultsPath).writeAsStringSync(updateTaskJson);
await cocoon.sendTaskStatus(resultsPath: resultsPath); await cocoon.sendResultsPath(resultsPath: resultsPath);
}); });
test('Verify timeout does not trigger for result upload', () async { test('Verify timeout does not trigger for result upload', () async {
...@@ -256,7 +256,7 @@ void main() { ...@@ -256,7 +256,7 @@ void main() {
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}'; '"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson); fs.file(resultsPath).writeAsStringSync(updateTaskJson);
await cocoon.sendTaskStatus(resultsPath: resultsPath); await cocoon.sendResultsPath(resultsPath: resultsPath);
}); });
test('Verify failure without retries for task result upload', () async { test('Verify failure without retries for task result upload', () async {
...@@ -288,7 +288,7 @@ void main() { ...@@ -288,7 +288,7 @@ void main() {
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}'; '"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson); fs.file(resultsPath).writeAsStringSync(updateTaskJson);
expect(() => cocoon.sendTaskStatus(resultsPath: resultsPath), throwsA(isA<ClientException>())); expect(() => cocoon.sendResultsPath(resultsPath: resultsPath), throwsA(isA<ClientException>()));
}); });
test('throws client exception on non-200 responses', () async { test('throws client exception on non-200 responses', () async {
...@@ -310,10 +310,10 @@ void main() { ...@@ -310,10 +310,10 @@ void main() {
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}'; '"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson); fs.file(resultsPath).writeAsStringSync(updateTaskJson);
expect(() => cocoon.sendTaskStatus(resultsPath: resultsPath), throwsA(isA<ClientException>())); expect(() => cocoon.sendResultsPath(resultsPath: resultsPath), throwsA(isA<ClientException>()));
}); });
test('does not update on non-supported branches', () async { test('does not upload results on non-supported branches', () async {
// Any network failure would cause the upoad to fail // Any network failure would cause the upoad to fail
mockClient = MockClient((Request request) async => Response('', 500)); mockClient = MockClient((Request request) async => Response('', 500));
...@@ -335,32 +335,7 @@ void main() { ...@@ -335,32 +335,7 @@ void main() {
fs.file(resultsPath).writeAsStringSync(updateTaskJson); fs.file(resultsPath).writeAsStringSync(updateTaskJson);
// This will fail if it decided to upload results // This will fail if it decided to upload results
await cocoon.sendTaskStatus(resultsPath: resultsPath); await cocoon.sendResultsPath(resultsPath: resultsPath);
});
test('does not update for staging test', () async {
// Any network failure would cause the upoad to fail
mockClient = MockClient((Request request) async => Response('', 500));
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
fs: fs,
httpClient: mockClient,
requestRetryLimit: 0,
);
const String resultsPath = 'results.json';
const String updateTaskJson = '{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"Linux_staging_test",'
'"NewStatus":"Succeeded",'
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson);
// This will fail if it decided to upload results
await cocoon.sendTaskStatus(resultsPath: resultsPath);
}); });
}); });
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment