Collect metrics - load balancing tests (#108752)

dev/bots/test.dart

@@ -63,6 +63,7 @@ import 'package:path/path.dart' as path;
 import 'browser.dart';
 import 'run_command.dart';
 import 'service_worker_test.dart';
+import 'tool_subsharding.dart';
 import 'utils.dart';
 
 typedef ShardRunner = Future<void> Function();
@@ -464,6 +465,7 @@ Future<void> _runIntegrationToolTests() async {
     _toolsPath,
     forceSingleCore: true,
     testPaths: _selectIndexOfTotalSubshard<String>(allTests),
+    collectMetrics: true,
   );
 }
 
@@ -1736,6 +1738,7 @@ Future<void> _runDartTest(String workingDirectory, {
   bool includeLocalEngineEnv = false,
   bool ensurePrecompiledTool = true,
   bool shuffleTests = true,
+  bool collectMetrics = false,
 }) async {
   int? cpus;
   final String? cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml
@@ -1757,6 +1760,8 @@ Future<void> _runDartTest(String workingDirectory, {
     cpus = 1;
   }
 
+  const LocalFileSystem fileSystem = LocalFileSystem();
+  final File metricFile = fileSystem.file(path.join(flutterRoot, 'metrics.json'));
   final List<String> args = <String>[
     'run',
     'test',
@@ -1771,6 +1776,8 @@ Future<void> _runDartTest(String workingDirectory, {
     if (testPaths != null)
       for (final String testPath in testPaths)
         testPath,
+    if (collectMetrics)
+      '--file-reporter=json:${metricFile.path}',
   ];
   final Map<String, String> environment = <String, String>{
     'FLUTTER_ROOT': flutterRoot,
@@ -1795,6 +1802,23 @@ Future<void> _runDartTest(String workingDirectory, {
     environment: environment,
     removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
   );
+
+  if (collectMetrics) {
+    try {
+      final List<String> testList = <String>[];
+      final Map<int, TestSpecs> allTestSpecs = generateMetrics(metricFile);
+      for (final TestSpecs testSpecs in allTestSpecs.values) {
+        testList.add(testSpecs.toJson());
+      }
+      if (testList.isNotEmpty) {
+        final String testJson = json.encode(testList);
+        final File testResults = fileSystem.file(path.join(flutterRoot, 'test_results.json'));
+        testResults.writeAsStringSync(testJson);
+      }
+    } on fs.FileSystemException catch (e){
+      print('Failed to generate metrics: $e');
+    }
+  }
 }
 
 Future<void> _runFlutterTest(String workingDirectory, {

dev/bots/test/tool_subsharding_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:file/memory.dart';
+
+import '../tool_subsharding.dart';
+import 'common.dart';
+
+void main() {
+  group('generateMetrics', () {
+    late MemoryFileSystem fileSystem;
+
+    setUp(() {
+      fileSystem = MemoryFileSystem.test();
+    });
+
+    test('empty metrics', () async {
+      final File file = fileSystem.file('success_file');
+      const String output = '''
+      {"missing": "entry"}
+      {"other": true}''';
+      file.writeAsStringSync(output);
+      final Map<int, TestSpecs> result = generateMetrics(file);
+      expect(result, isEmpty);
+    });
+
+    test('have metrics', () async {
+      final File file = fileSystem.file('success_file');
+      const String output = '''
+      {"protocolVersion":"0.1.1","runnerVersion":"1.21.6","pid":93376,"type":"start","time":0}
+      {"suite":{"id":0,"platform":"vm","path":"test/general.shard/project_validator_result_test.dart"},"type":"suite","time":0}
+      {"count":1,"time":12,"type":"allSuites"}
+      {"testID":1,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":4798}
+      {"test":{"id":4,"name":"ProjectValidatorResult success status","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":50,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4803}
+      {"testID":4,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4837}
+      {"suite":{"id":1,"platform":"vm","path":"other_path"},"type":"suite","time":1000}
+      {"test":{"id":5,"name":"ProjectValidatorResult success status with warning","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":60,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4837}
+      {"testID":5,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4839}
+      {"test":{"id":6,"name":"ProjectValidatorResult error status","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":71,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4839}
+      {"testID":6,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4841}
+      {"group":{"id":7,"suiteID":0,"parentID":2,"name":"ProjectValidatorTask","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":4841}
+      {"test":{"id":8,"name":"ProjectValidatorTask error status","suiteID":0,"groupIDs":[2,7],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":89,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4842}
+      {"testID":8,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4860}
+      {"group":{"id":7,"suiteID":1,"parentID":2,"name":"ProjectValidatorTask","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000}
+      {"success":true,"type":"done","time":4870}''';
+      file.writeAsStringSync(output);
+      final Map<int, TestSpecs> result = generateMetrics(file);
+      expect(result, contains(0));
+      expect(result, contains(1));
+      expect(result[0]!.path, 'test/general.shard/project_validator_result_test.dart');
+      expect(result[0]!.milliseconds, 4841);
+      expect(result[1]!.path, 'other_path');
+      expect(result[1]!.milliseconds, 4000);
+    });
+
+    test('missing success entry', () async {
+      final File file = fileSystem.file('success_file');
+      const String output = '''
+      {"suite":{"id":1,"platform":"vm","path":"other_path"},"type":"suite","time":1000}
+      {"group":{"id":7,"suiteID":1,"parentID":2,"name":"name","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000}''';
+      file.writeAsStringSync(output);
+      final Map<int, TestSpecs> result = generateMetrics(file);
+      expect(result, isEmpty);
+    });
+  });
+}

dev/bots/tool_subsharding.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+class TestSpecs {
+
+  TestSpecs({
+    required this.path,
+    required this.startTime,
+  });
+
+  final String path;
+  int startTime;
+  int? _endTime;
+
+  int get milliseconds {
+    return endTime - startTime;
+  }
+
+  set endTime(int value) {
+    _endTime = value;
+  }
+
+  int get endTime {
+    if (_endTime == null) {
+      return 0;
+    }
+    return _endTime!;
+  }
+
+  String toJson() {
+    return json.encode(
+      <String, String>{'path': path, 'runtime': milliseconds.toString()}
+    );
+  }
+}
+
+/// Intended to parse the output file of `dart test --file-reporter json:file_name
+Map<int, TestSpecs> generateMetrics(File metrics) {
+  final Map<int, TestSpecs> allTestSpecs = <int, TestSpecs>{};
+  if (!metrics.existsSync()) {
+    return allTestSpecs;
+  }
+
+  bool success = false;
+  for(final String metric in metrics.readAsLinesSync()) {
+    final Map<String, dynamic> entry = json.decode(metric) as Map<String, dynamic>;
+    if (entry.containsKey('suite')) {
+      final Map<dynamic, dynamic> suite = entry['suite'] as Map<dynamic, dynamic>;
+      allTestSpecs[suite['id'] as int] = TestSpecs(
+        path: suite['path'] as String,
+        startTime: entry['time'] as int,
+      );
+    } else if (_isMetricDone(entry, allTestSpecs)) {
+      final Map<dynamic, dynamic> group = entry['group'] as Map<dynamic, dynamic>;
+      final int suiteID = group['suiteID'] as int;
+      final TestSpecs testSpec = allTestSpecs[suiteID]!;
+      testSpec.endTime = entry['time'] as int;
+    } else if (entry.containsKey('success') && entry['success'] == true) {
+      success = true;
+    }
+  }
+
+  if (!success) { // means that not all tests succeeded therefore no metrics are stored
+    return <int, TestSpecs>{};
+  }
+  return allTestSpecs;
+}
+
+bool _isMetricDone(Map<String, dynamic> entry, Map<int, TestSpecs> allTestSpecs) {
+  if (entry.containsKey('group') && entry['type'] as String == 'group') {
+    final Map<dynamic, dynamic> group = entry['group'] as Map<dynamic, dynamic>;
+    return allTestSpecs.containsKey(group['suiteID'] as int);
+  }
+  return false;
+}