metrics_center.dart 6.29 KB
Newer Older
1 2 3 4 5 6 7
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io';

8
import 'package:collection/collection.dart';
9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
import 'package:metrics_center/metrics_center.dart';

/// Authenticate and connect to gcloud storage.
///
/// It supports both token and credential authentications.
Future<FlutterDestination> connectFlutterDestination() async {
  const String kTokenPath = 'TOKEN_PATH';
  const String kGcpProject = 'GCP_PROJECT';
  final Map<String, String> env = Platform.environment;
  final bool isTesting = env['IS_TESTING'] == 'true';
  if (env.containsKey(kTokenPath) && env.containsKey(kGcpProject)) {
    return FlutterDestination.makeFromAccessToken(
      File(env[kTokenPath]!).readAsStringSync(),
      env[kGcpProject]!,
      isTesting: isTesting,
    );
  }
  return FlutterDestination.makeFromCredentialsJson(
    jsonDecode(env['BENCHMARK_GCP_CREDENTIALS']!) as Map<String, dynamic>,
    isTesting: isTesting,
  );
}

32
/// Parse results and append additional benchmark tags into Metric Points.
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
///
/// An example of `resultsJson`:
///   {
///     "CommitBranch": "master",
///     "CommitSha": "abc",
///     "BuilderName": "test",
///     "ResultData": {
///       "average_frame_build_time_millis": 0.4550425531914895,
///       "90th_percentile_frame_build_time_millis": 0.473
///     },
///     "BenchmarkScoreKeys": [
///       "average_frame_build_time_millis",
///       "90th_percentile_frame_build_time_millis"
///     ]
///   }
48 49 50 51 52 53 54 55 56
///
/// An example of `benchmarkTags`:
///   {
///     "arch": "intel",
///     "device_type": "Moto G Play",
///     "device_version": "android-25",
///     "host_type": "linux",
///     "host_version": "debian-10.11"
///   }
57
List<MetricPoint> parse(Map<String, dynamic> resultsJson, Map<String, dynamic> benchmarkTags, String taskName) {
58
  print('Results to upload to skia perf: $resultsJson');
59
  print('Benchmark tags to upload to skia perf: $benchmarkTags');
60 61 62 63
  final List<String> scoreKeys =
      (resultsJson['BenchmarkScoreKeys'] as List<dynamic>?)?.cast<String>() ?? const <String>[];
  final Map<String, dynamic> resultData =
      resultsJson['ResultData'] as Map<String, dynamic>? ?? const <String, dynamic>{};
64 65 66 67
  final String gitBranch = (resultsJson['CommitBranch'] as String).trim();
  final String gitSha = (resultsJson['CommitSha'] as String).trim();
  final List<MetricPoint> metricPoints = <MetricPoint>[];
  for (final String scoreKey in scoreKeys) {
68 69 70 71
    Map<String, String> tags = <String, String>{
      kGithubRepoKey: kFlutterFrameworkRepo,
      kGitRevisionKey: gitSha,
      'branch': gitBranch,
72
      kNameKey: taskName,
73 74 75 76 77
      kSubResultKey: scoreKey,
    };
    // Append additional benchmark tags, which will surface in Skia Perf dashboards.
    tags = mergeMaps<String, String>(
        tags, benchmarkTags.map((String key, dynamic value) => MapEntry<String, String>(key, value.toString())));
78
    metricPoints.add(
79 80 81 82 83
      MetricPoint(
        (resultData[scoreKey] as num).toDouble(),
        tags,
      ),
    );
84 85 86 87
  }
  return metricPoints;
}

88 89 90 91 92 93 94 95 96 97 98 99 100
/// Upload metrics to GCS bucket used by Skia Perf.
///
/// Skia Perf picks up all available files under the folder, and
/// is robust to duplicate entries.
///
/// Files will be named based on `taskName`, such as
/// `complex_layout_scroll_perf__timeline_summary_values.json`.
/// If no `taskName` is specified, data will be saved to
/// `default_values.json`.
Future<void> upload(
  FlutterDestination metricsDestination,
  List<MetricPoint> metricPoints,
  int commitTimeSinceEpoch,
101
  String taskName,
102 103 104 105 106 107 108
) async {
  await metricsDestination.update(
    metricPoints,
    DateTime.fromMillisecondsSinceEpoch(
      commitTimeSinceEpoch,
      isUtc: true,
    ),
109
    taskName,
110 111 112
  );
}

113 114 115 116 117 118
/// Upload JSON results to skia perf.
///
/// Flutter infrastructure's workflow is:
/// 1. Run DeviceLab test, writing results to a known path
/// 2. Request service account token from luci auth (valid for at least 3 minutes)
/// 3. Upload results from (1) to skia perf.
119
Future<void> uploadToSkiaPerf(String? resultsPath, String? commitTime, String? taskName, String? benchmarkTags) async {
120 121 122 123 124 125 126 127 128
  int commitTimeSinceEpoch;
  if (resultsPath == null) {
    return;
  }
  if (commitTime != null) {
    commitTimeSinceEpoch = 1000 * int.parse(commitTime);
  } else {
    commitTimeSinceEpoch = DateTime.now().millisecondsSinceEpoch;
  }
129
  taskName = taskName ?? 'default';
130
  final Map<String, dynamic> benchmarkTagsMap = jsonDecode(benchmarkTags ?? '{}') as Map<String, dynamic>;
131 132 133
  final File resultFile = File(resultsPath);
  Map<String, dynamic> resultsJson = <String, dynamic>{};
  resultsJson = json.decode(await resultFile.readAsString()) as Map<String, dynamic>;
134
  final List<MetricPoint> metricPoints = parse(resultsJson, benchmarkTagsMap, taskName);
135
  final FlutterDestination metricsDestination = await connectFlutterDestination();
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
  await upload(
    metricsDestination,
    metricPoints,
    commitTimeSinceEpoch,
    metricFileName(taskName, benchmarkTagsMap),
  );
}

/// Create metric file name based on `taskName`, `arch`, `host type`, and `device type`.
///
/// Same `taskName` may run on different platforms. Considering host/device tags to
/// use different metric file names.
///
/// This affects only the metric file name which contains metric data, and does not affect
/// real host/device tags.
///
/// For example:
///   Old file name: `backdrop_filter_perf__timeline_summary`
///   New file name: `backdrop_filter_perf__timeline_summary_intel_linux_motoG4`
String metricFileName(
  String taskName,
  Map<String, dynamic> benchmarkTagsMap,
) {
  final StringBuffer fileName = StringBuffer(taskName);
  if (benchmarkTagsMap.containsKey('arch')) {
    fileName
      ..write('_')
      ..write(_fileNameFormat(benchmarkTagsMap['arch'] as String));
  }
  if (benchmarkTagsMap.containsKey('host_type')) {
    fileName
      ..write('_')
      ..write(_fileNameFormat(benchmarkTagsMap['host_type'] as String));
  }
  if (benchmarkTagsMap.containsKey('device_type')) {
    fileName
      ..write('_')
      ..write(_fileNameFormat(benchmarkTagsMap['device_type'] as String));
  }
  return fileName.toString();
}

/// Format `fileName` removing non letter and number characters.
String _fileNameFormat(String fileName) {
  return fileName.replaceAll(RegExp('[^a-zA-Z0-9]'), '');
181
}