Add SkiaPerfPoint and FlutterEngineMetricPoint (#70153)

Unit tests that cover their translations are also added.

dev/benchmarks/metrics_center/lib/src/flutter.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:metrics_center/src/common.dart';
+
+/// Convenient class to capture the benchmarks in the Flutter engine repo.
+class FlutterEngineMetricPoint extends MetricPoint {
+  FlutterEngineMetricPoint(
+    String name,
+    double value,
+    String gitRevision, {
+    Map<String, String> moreTags = const <String, String>{},
+  }) : super(
+          value,
+          <String, String>{
+            kNameKey: name,
+            kGithubRepoKey: kFlutterEngineRepo,
+            kGitRevisionKey: gitRevision,
+          }..addAll(moreTags),
+        );
+}

dev/benchmarks/metrics_center/lib/src/github_helper.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This file uses Dart 2.12 semantics. This is needed as we can't upgrade
+// the SDK constraint to `>=2.12.0-0` before the deps are ready.
+// @dart=2.12
+
+import 'package:github/github.dart';
+
+/// Singleton class to query some Github info with an in-memory cache.
+class GithubHelper {
+  /// Return the singleton helper.
+  factory GithubHelper() {
+    return _singleton;
+  }
+
+  GithubHelper._internal();
+
+  /// The result is cached in memory so querying the same thing again in the
+  /// same process is fast.
+  ///
+  /// Our unit test requires that calling this method 1000 times for the same
+  /// `githubRepo` and `sha` should be done in 1 second.
+  Future<DateTime> getCommitDateTime(String githubRepo, String sha) async {
+    final String key = '$githubRepo/commit/$sha';
+    if (_commitDateTimeCache[key] == null) {
+      final RepositoryCommit commit = await _github.repositories
+          .getCommit(RepositorySlug.full(githubRepo), sha);
+      _commitDateTimeCache[key] = commit.commit.committer.date;
+    }
+    return _commitDateTimeCache[key]!;
+  }
+
+  static final GithubHelper _singleton = GithubHelper._internal();
+
+  final GitHub _github = GitHub();
+  final Map<String, DateTime> _commitDateTimeCache = <String, DateTime>{};
+}

dev/benchmarks/metrics_center/lib/src/skiaperf.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This file uses Dart 2.12 semantics. This is needed as we can't upgrade
+// the SDK constraint to `>=2.12.0-0` before the deps are ready.
+// @dart=2.12
+
+import 'package:metrics_center/src/common.dart';
+
+// Skia Perf Format is a JSON file that looks like:
+
+// {
+//     "gitHash": "fe4a4029a080bc955e9588d05a6cd9eb490845d4",
+//     "key": {
+//         "arch": "x86",
+//         "gpu": "GTX660",
+//         "model": "ShuttleA",
+//         "os": "Ubuntu12"
+//     },
+//     "results": {
+//         "ChunkAlloc_PushPop_640_480": {
+//             "nonrendering": {
+//                 "min_ms": 0.01485466666666667,
+//                 "options": {
+//                     "source_type": "bench"
+//                 }
+//             }
+//         },
+//         "DeferredSurfaceCopy_discardable_640_480": {
+//             "565": {
+//                 "min_ms": 2.215988,
+//                 "options": {
+//                     "source_type": "bench"
+//                 }
+//             },
+//     ...
+
+class SkiaPerfPoint extends MetricPoint {
+  SkiaPerfPoint._(this.githubRepo, this.gitHash, this.testName, this.subResult,
+      double value, this._options, this.jsonUrl)
+      : assert(_options[kGithubRepoKey] == null),
+        assert(_options[kGitRevisionKey] == null),
+        assert(_options[kNameKey] == null),
+        super(
+          value,
+          <String, String>{}
+            ..addAll(_options)
+            ..addAll(<String, String>{
+              kGithubRepoKey: githubRepo,
+              kGitRevisionKey: gitHash,
+              kNameKey: testName,
+              kSubResultKey: subResult,
+            }),
+        ) {
+    assert(tags[kGithubRepoKey] != null);
+    assert(tags[kGitRevisionKey] != null);
+    assert(tags[kNameKey] != null);
+  }
+
+  /// Construct [SkiaPerfPoint] from a well-formed [MetricPoint].
+  ///
+  /// The [MetricPoint] must have [kGithubRepoKey], [kGitRevisionKey],
+  /// [kNameKey] in its tags for this to be successful.
+  ///
+  /// If the [MetricPoint] has a tag 'date', that tag will be removed so Skia
+  /// perf can plot multiple metrics with different date as a single trace.
+  /// Skia perf will use the git revision's date instead of this date tag in
+  /// the time axis.
+  factory SkiaPerfPoint.fromPoint(MetricPoint p) {
+    final String githubRepo = p.tags[kGithubRepoKey]!;
+    final String gitHash = p.tags[kGitRevisionKey]!;
+    final String name = p.tags[kNameKey]!;
+    final String subResult = p.tags[kSubResultKey] ?? kSkiaPerfValueKey;
+
+    final Map<String, String> options = <String, String>{}..addEntries(
+        p.tags.entries.where(
+          (MapEntry<String, dynamic> entry) =>
+              entry.key != kGithubRepoKey &&
+              entry.key != kGitRevisionKey &&
+              entry.key != kNameKey &&
+              entry.key != kSubResultKey &&
+              // https://github.com/google/benchmark automatically generates a
+              // 'date' field. If it's included in options, the Skia perf won't
+              // be able to connect different points in a single trace because
+              // the date is always different.
+              entry.key != 'date',
+        ),
+      );
+
+    return SkiaPerfPoint._(
+        githubRepo, gitHash, name, subResult, p.value, options, null);
+  }
+
+  /// In the format of '<owner>/<name>' such as 'flutter/flutter' or
+  /// 'flutter/engine'.
+  final String githubRepo;
+
+  /// SHA such as 'ad20d368ffa09559754e4b2b5c12951341ca3b2d'
+  final String gitHash;
+
+  /// For Flutter devicelab, this is the task name (e.g.,
+  /// 'flutter_gallery__transition_perf'); for Google benchmark, this is the
+  /// benchmark name (e.g., 'BM_ShellShutdown').
+  ///
+  /// In Skia perf web dashboard, this value can be queried and filtered by
+  /// "test".
+  final String testName;
+
+  /// The name of "subResult" comes from the special treatment of "sub_result"
+  /// in SkiaPerf. If not provided, its value will be set to kSkiaPerfValueKey.
+  ///
+  /// When Google benchmarks are converted to SkiaPerfPoint, this subResult
+  /// could be "cpu_time" or "real_time".
+  ///
+  /// When devicelab benchmarks are converted to SkiaPerfPoint, this subResult
+  /// is often the metric name such as "average_frame_build_time_millis" whereas
+  /// the [testName] is the benchmark or task name such as
+  /// "flutter_gallery__transition_perf".
+  final String subResult;
+
+  /// The url to the Skia perf json file in the Google Cloud Storage bucket.
+  ///
+  /// This can be null if the point has been stored in the bucket yet.
+  final String? jsonUrl;
+
+  Map<String, dynamic> _toSubResultJson() {
+    return <String, dynamic>{
+      subResult: value,
+      kSkiaPerfOptionsKey: _options,
+    };
+  }
+
+  /// Convert a list of SkiaPoints with the same git repo and git revision into
+  /// a single json file in the Skia perf format.
+  ///
+  /// The list must be non-empty.
+  static Map<String, dynamic> toSkiaPerfJson(List<SkiaPerfPoint> points) {
+    assert(points.isNotEmpty);
+    assert(() {
+      for (final SkiaPerfPoint p in points) {
+        if (p.githubRepo != points[0].githubRepo ||
+            p.gitHash != points[0].gitHash) {
+          return false;
+        }
+      }
+      return true;
+    }(), 'All points must have same githubRepo and gitHash');
+
+    final Map<String, dynamic> results = <String, dynamic>{};
+    for (final SkiaPerfPoint p in points) {
+      final Map<String, dynamic> subResultJson = p._toSubResultJson();
+      if (results[p.testName] == null) {
+        results[p.testName] = <String, dynamic>{
+          kSkiaPerfDefaultConfig: subResultJson,
+        };
+      } else {
+        // Flutter currently doesn't support having the same name but different
+        // options/configurations. If this actually happens in the future, we
+        // probably can use different values of config (currently there's only
+        // one kSkiaPerfDefaultConfig) to resolve the conflict.
+        assert(results[p.testName][kSkiaPerfDefaultConfig][kSkiaPerfOptionsKey]
+                .toString() ==
+            subResultJson[kSkiaPerfOptionsKey].toString());
+        assert(
+            results[p.testName][kSkiaPerfDefaultConfig][p.subResult] == null);
+        results[p.testName][kSkiaPerfDefaultConfig][p.subResult] = p.value;
+      }
+    }
+
+    return <String, dynamic>{
+      kSkiaPerfGitHashKey: points[0].gitHash,
+      kSkiaPerfResultsKey: results,
+    };
+  }
+
+  // Equivalent to tags without git repo, git hash, and name because those two
+  // are already stored somewhere else.
+  final Map<String, String> _options;
+}
+
+const String kSkiaPerfGitHashKey = 'gitHash';
+const String kSkiaPerfResultsKey = 'results';
+const String kSkiaPerfValueKey = 'value';
+const String kSkiaPerfOptionsKey = 'options';
+const String kSkiaPerfDefaultConfig = 'default';

dev/benchmarks/metrics_center/test/github_helper_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+@Timeout(Duration(seconds: 3600))
+
+import 'package:metrics_center/src/github_helper.dart';
+
+import 'common.dart';
+
+void main() {
+  test('GithubHelper gets correct commit date time', () async {
+    final GithubHelper helper = GithubHelper();
+    expect(
+      await helper.getCommitDateTime(
+        'flutter/flutter',
+        'ad20d368ffa09559754e4b2b5c12951341ca3b2d',
+      ),
+      equals(DateTime.parse('2019-12-06 03:33:01.000Z')),
+    );
+  });
+
+  test('GithubHelper is a singleton', () {
+    final GithubHelper helper1 = GithubHelper();
+    final GithubHelper helper2 = GithubHelper();
+    expect(helper1, equals(helper2));
+  });
+
+  test('GithubHelper can query the same commit 1000 times within 1 second',
+      () async {
+    final DateTime start = DateTime.now();
+    for (int i = 0; i < 1000; i += 1) {
+      await GithubHelper().getCommitDateTime(
+        'flutter/flutter',
+        'ad20d368ffa09559754e4b2b5c12951341ca3b2d',
+      );
+    }
+    final Duration duration = DateTime.now().difference(start);
+    expect(duration, lessThan(const Duration(seconds: 1)));
+  });
+}

dev/benchmarks/metrics_center/test/google_benchmark_test.dart

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// @dart = 2.8
-
 import 'package:metrics_center/src/common.dart';
 import 'package:metrics_center/google_benchmark.dart';
 

dev/benchmarks/metrics_center/test/skiaperf_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+@Timeout(Duration(seconds: 3600))
+
+import 'dart:convert';
+
+import 'package:metrics_center/src/common.dart';
+import 'package:metrics_center/src/flutter.dart';
+import 'package:metrics_center/src/skiaperf.dart';
+
+import 'common.dart';
+
+void main() {
+  const double kValue1 = 1.0;
+  const double kValue2 = 2.0;
+
+  const String kFrameworkRevision1 = '9011cece2595447eea5dd91adaa241c1c9ef9a33';
+  const String kTaskName = 'analyzer_benchmark';
+  const String kMetric1 = 'flutter_repo_batch_maximum';
+  const String kMetric2 = 'flutter_repo_watch_maximum';
+
+  final MetricPoint cocoonPointRev1Metric1 = MetricPoint(
+    kValue1,
+    const <String, String>{
+      kGithubRepoKey: kFlutterFrameworkRepo,
+      kGitRevisionKey: kFrameworkRevision1,
+      kNameKey: kTaskName,
+      kSubResultKey: kMetric1,
+      kUnitKey: 's',
+    },
+  );
+
+  final MetricPoint cocoonPointRev1Metric2 = MetricPoint(
+    kValue2,
+    const <String, String>{
+      kGithubRepoKey: kFlutterFrameworkRepo,
+      kGitRevisionKey: kFrameworkRevision1,
+      kNameKey: kTaskName,
+      kSubResultKey: kMetric2,
+      kUnitKey: 's',
+    },
+  );
+
+  final MetricPoint cocoonPointBetaRev1Metric1 = MetricPoint(
+    kValue1,
+    const <String, String>{
+      kGithubRepoKey: kFlutterFrameworkRepo,
+      kGitRevisionKey: kFrameworkRevision1,
+      kNameKey: 'beta/$kTaskName',
+      kSubResultKey: kMetric1,
+      kUnitKey: 's',
+      'branch': 'beta',
+    },
+  );
+
+  final MetricPoint cocoonPointBetaRev1Metric1BadBranch = MetricPoint(
+    kValue1,
+    const <String, String>{
+      kGithubRepoKey: kFlutterFrameworkRepo,
+      kGitRevisionKey: kFrameworkRevision1,
+      kNameKey: kTaskName,
+      kSubResultKey: kMetric1,
+      kUnitKey: 's',
+
+      // If we only add this 'branch' tag without changing the test or sub-result name, an exception
+      // would be thrown as Skia Perf currently only supports the same set of tags for a pair of
+      // kNameKey and kSubResultKey values. So to support branches, one also has to add the branch
+      // name to the test name.
+      'branch': 'beta',
+    },
+  );
+
+  const String engineMetricName = 'BM_PaintRecordInit';
+  const String engineRevision = 'ca799fa8b2254d09664b78ee80c43b434788d112';
+  const double engineValue1 = 101;
+  const double engineValue2 = 102;
+
+  final FlutterEngineMetricPoint enginePoint1 = FlutterEngineMetricPoint(
+    engineMetricName,
+    engineValue1,
+    engineRevision,
+    moreTags: const <String, String>{
+      kSubResultKey: 'cpu_time',
+      kUnitKey: 'ns',
+      'date': '2019-12-17 15:14:14',
+      'num_cpus': '56',
+      'mhz_per_cpu': '2594',
+      'cpu_scaling_enabled': 'true',
+      'library_build_type': 'release',
+    },
+  );
+
+  final FlutterEngineMetricPoint enginePoint2 = FlutterEngineMetricPoint(
+    engineMetricName,
+    engineValue2,
+    engineRevision,
+    moreTags: const <String, String>{
+      kSubResultKey: 'real_time',
+      kUnitKey: 'ns',
+      'date': '2019-12-17 15:14:14',
+      'num_cpus': '56',
+      'mhz_per_cpu': '2594',
+      'cpu_scaling_enabled': 'true',
+      'library_build_type': 'release',
+    },
+  );
+
+  test('Throw if invalid points are converted to SkiaPoint', () {
+    final MetricPoint noGithubRepoPoint = MetricPoint(
+      kValue1,
+      const <String, String>{
+        kGitRevisionKey: kFrameworkRevision1,
+        kNameKey: kTaskName,
+      },
+    );
+
+    final MetricPoint noGitRevisionPoint = MetricPoint(
+      kValue1,
+      const <String, String>{
+        kGithubRepoKey: kFlutterFrameworkRepo,
+        kNameKey: kTaskName,
+      },
+    );
+
+    final MetricPoint noTestNamePoint = MetricPoint(
+      kValue1,
+      const <String, String>{
+        kGithubRepoKey: kFlutterFrameworkRepo,
+        kGitRevisionKey: kFrameworkRevision1,
+      },
+    );
+
+    expect(() => SkiaPerfPoint.fromPoint(noGithubRepoPoint), throwsA(anything));
+    expect(
+        () => SkiaPerfPoint.fromPoint(noGitRevisionPoint), throwsA(anything));
+    expect(() => SkiaPerfPoint.fromPoint(noTestNamePoint), throwsA(anything));
+  });
+
+  test('Correctly convert a metric point from cocoon to SkiaPoint', () {
+    final SkiaPerfPoint skiaPoint1 =
+        SkiaPerfPoint.fromPoint(cocoonPointRev1Metric1);
+    expect(skiaPoint1, isNotNull);
+    expect(skiaPoint1.testName, equals(kTaskName));
+    expect(skiaPoint1.subResult, equals(kMetric1));
+    expect(skiaPoint1.value, equals(cocoonPointRev1Metric1.value));
+    expect(skiaPoint1.jsonUrl, isNull); // Not inserted yet
+  });
+
+  test('Cocoon points correctly encode into Skia perf json format', () {
+    final SkiaPerfPoint p1 = SkiaPerfPoint.fromPoint(cocoonPointRev1Metric1);
+    final SkiaPerfPoint p2 = SkiaPerfPoint.fromPoint(cocoonPointRev1Metric2);
+    final SkiaPerfPoint p3 =
+        SkiaPerfPoint.fromPoint(cocoonPointBetaRev1Metric1);
+
+    const JsonEncoder encoder = JsonEncoder.withIndent('  ');
+
+    expect(
+        encoder
+            .convert(SkiaPerfPoint.toSkiaPerfJson(<SkiaPerfPoint>[p1, p2, p3])),
+        equals('''
+{
+  "gitHash": "9011cece2595447eea5dd91adaa241c1c9ef9a33",
+  "results": {
+    "analyzer_benchmark": {
+      "default": {
+        "flutter_repo_batch_maximum": 1.0,
+        "options": {
+          "unit": "s"
+        },
+        "flutter_repo_watch_maximum": 2.0
+      }
+    },
+    "beta/analyzer_benchmark": {
+      "default": {
+        "flutter_repo_batch_maximum": 1.0,
+        "options": {
+          "branch": "beta",
+          "unit": "s"
+        }
+      }
+    }
+  }
+}'''));
+  });
+
+  test('Engine metric points correctly encode into Skia perf json format', () {
+    const JsonEncoder encoder = JsonEncoder.withIndent('  ');
+    expect(
+      encoder.convert(SkiaPerfPoint.toSkiaPerfJson(<SkiaPerfPoint>[
+        SkiaPerfPoint.fromPoint(enginePoint1),
+        SkiaPerfPoint.fromPoint(enginePoint2),
+      ])),
+      equals(
+        '''
+{
+  "gitHash": "ca799fa8b2254d09664b78ee80c43b434788d112",
+  "results": {
+    "BM_PaintRecordInit": {
+      "default": {
+        "cpu_time": 101.0,
+        "options": {
+          "cpu_scaling_enabled": "true",
+          "library_build_type": "release",
+          "mhz_per_cpu": "2594",
+          "num_cpus": "56",
+          "unit": "ns"
+        },
+        "real_time": 102.0
+      }
+    }
+  }
+}''',
+      ),
+    );
+  });
+
+  test(
+      'Throw if engine points with the same test name but different options are converted to '
+      'Skia perf points', () {
+    final FlutterEngineMetricPoint enginePoint1 = FlutterEngineMetricPoint(
+      'BM_PaintRecordInit',
+      101,
+      'ca799fa8b2254d09664b78ee80c43b434788d112',
+      moreTags: const <String, String>{
+        kSubResultKey: 'cpu_time',
+        kUnitKey: 'ns',
+        'cpu_scaling_enabled': 'true',
+      },
+    );
+    final FlutterEngineMetricPoint enginePoint2 = FlutterEngineMetricPoint(
+      'BM_PaintRecordInit',
+      102,
+      'ca799fa8b2254d09664b78ee80c43b434788d112',
+      moreTags: const <String, String>{
+        kSubResultKey: 'real_time',
+        kUnitKey: 'ns',
+        'cpu_scaling_enabled': 'false',
+      },
+    );
+
+    const JsonEncoder encoder = JsonEncoder.withIndent('  ');
+    expect(
+      () => encoder.convert(SkiaPerfPoint.toSkiaPerfJson(<SkiaPerfPoint>[
+        SkiaPerfPoint.fromPoint(enginePoint1),
+        SkiaPerfPoint.fromPoint(enginePoint2),
+      ])),
+      throwsA(anything),
+    );
+  });
+
+  test(
+      'Throw if two Cocoon metric points with the same name and subResult keys '
+      'but different options are converted to Skia perf points', () {
+    final SkiaPerfPoint p1 = SkiaPerfPoint.fromPoint(cocoonPointRev1Metric1);
+    final SkiaPerfPoint p2 =
+        SkiaPerfPoint.fromPoint(cocoonPointBetaRev1Metric1BadBranch);
+
+    expect(
+      () => SkiaPerfPoint.toSkiaPerfJson(<SkiaPerfPoint>[p1, p2]),
+      throwsA(anything),
+    );
+  });
+}