web_benchmarks.dart 9.44 KB
Newer Older
1 2 3 4 5 6 7 8
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:convert' show json;
import 'dart:io' as io;

9
import 'package:logging/logging.dart';
10 11 12 13 14
import 'package:path/path.dart' as path;
import 'package:shelf/shelf.dart';
import 'package:shelf/shelf_io.dart' as shelf_io;
import 'package:shelf_static/shelf_static.dart';

15 16 17 18
import '../framework/browser.dart';
import '../framework/task_result.dart';
import '../framework/utils.dart';

19 20
/// The port number used by the local benchmark server.
const int benchmarkServerPort = 9999;
21
const int chromeDebugPort = 10000;
22

23 24 25 26 27 28
typedef WebBenchmarkOptions = ({
  String webRenderer,
  bool useWasm,
});

Future<TaskResult> runWebBenchmark(WebBenchmarkOptions benchmarkOptions) async {
29 30
  // Reduce logging level. Otherwise, package:webkit_inspection_protocol is way too spammy.
  Logger.root.level = Level.INFO;
31
  final String macrobenchmarksDirectory = path.join(flutterDirectory.path, 'dev', 'benchmarks', 'macrobenchmarks');
32
  return inDirectory(macrobenchmarksDirectory, () async {
33
    await flutter('clean');
34 35
    await evalFlutter('build', options: <String>[
      'web',
36
      if (benchmarkOptions.useWasm) ...<String>[
37
        '-O4',
38
        '--wasm',
39
        '--no-strip-wasm',
40
      ],
41
      '--dart-define=FLUTTER_WEB_ENABLE_PROFILING=true',
42
      if (!benchmarkOptions.useWasm) '--web-renderer=${benchmarkOptions.webRenderer}',
43
      '--profile',
44
      '--no-web-resources-cdn',
45 46
      '-t',
      'lib/web_benchmarks.dart',
47
    ]);
48 49
    final Completer<List<Map<String, dynamic>>> profileData = Completer<List<Map<String, dynamic>>>();
    final List<Map<String, dynamic>> collectedProfiles = <Map<String, dynamic>>[];
50 51
    List<String>? benchmarks;
    late Iterator<String> benchmarkIterator;
52

53 54 55
    // This future fixes a race condition between the web-page loading and
    // asking to run a benchmark, and us connecting to Chrome's DevTools port.
    // Sometime one wins. Other times, the other wins.
56 57 58
    Future<Chrome>? whenChromeIsReady;
    Chrome? chrome;
    late io.HttpServer server;
59
    Cascade cascade = Cascade();
60
    List<Map<String, dynamic>>? latestPerformanceTrace;
61
    cascade = cascade.add((Request request) async {
62 63 64 65 66 67 68 69
      try {
        chrome ??= await whenChromeIsReady;
        if (request.requestedUri.path.endsWith('/profile-data')) {
          final Map<String, dynamic> profile = json.decode(await request.readAsString()) as Map<String, dynamic>;
          final String benchmarkName = profile['name'] as String;
          if (benchmarkName != benchmarkIterator.current) {
            profileData.completeError(Exception(
              'Browser returned benchmark results from a wrong benchmark.\n'
70
              'Requested to run benchmark ${benchmarkIterator.current}, but '
71 72
              'got results for $benchmarkName.',
            ));
73
            unawaited(server.close());
74 75
          }

76 77
          // Trace data is null when the benchmark is not frame-based, such as RawRecorder.
          if (latestPerformanceTrace != null) {
78
            final BlinkTraceSummary traceSummary = BlinkTraceSummary.fromJson(latestPerformanceTrace!)!;
79 80
            profile['totalUiFrame.average'] = traceSummary.averageTotalUIFrameTime.inMicroseconds;
            profile['scoreKeys'] ??= <dynamic>[]; // using dynamic for consistency with JSON
81
            (profile['scoreKeys'] as List<dynamic>).add('totalUiFrame.average');
82
            latestPerformanceTrace = null;
83 84 85 86 87
          }
          collectedProfiles.add(profile);
          return Response.ok('Profile received');
        } else if (request.requestedUri.path.endsWith('/start-performance-tracing')) {
          latestPerformanceTrace = null;
88
          await chrome!.beginRecordingPerformance(request.requestedUri.queryParameters['label']!);
89 90
          return Response.ok('Started performance tracing');
        } else if (request.requestedUri.path.endsWith('/stop-performance-tracing')) {
91
          latestPerformanceTrace = await chrome!.endRecordingPerformance();
92 93 94
          return Response.ok('Stopped performance tracing');
        } else if (request.requestedUri.path.endsWith('/on-error')) {
          final Map<String, dynamic> errorDetails = json.decode(await request.readAsString()) as Map<String, dynamic>;
95
          unawaited(server.close());
96 97 98 99 100 101
          // Keep the stack trace as a string. It's thrown in the browser, not this Dart VM.
          profileData.completeError('${errorDetails['error']}\n${errorDetails['stackTrace']}');
          return Response.ok('');
        } else if (request.requestedUri.path.endsWith('/next-benchmark')) {
          if (benchmarks == null) {
            benchmarks = (json.decode(await request.readAsString()) as List<dynamic>).cast<String>();
102
            benchmarkIterator = benchmarks!.iterator;
103 104 105 106 107 108 109 110 111
          }
          if (benchmarkIterator.moveNext()) {
            final String nextBenchmark = benchmarkIterator.current;
            print('Launching benchmark "$nextBenchmark"');
            return Response.ok(nextBenchmark);
          } else {
            profileData.complete(collectedProfiles);
            return Response.notFound('Finished running benchmarks.');
          }
112 113 114 115 116
        } else if (request.requestedUri.path.endsWith('/print-to-console')) {
          // A passthrough used by
          // `dev/benchmarks/macrobenchmarks/lib/web_benchmarks.dart`
          // to print information.
          final String message = await request.readAsString();
117
          print('[APP] $message');
118
          return Response.ok('Reported.');
119
        } else {
120 121
          return Response.notFound(
              'This request is not handled by the profile-data handler.');
122
        }
123 124 125
      } catch (error, stackTrace) {
        profileData.completeError(error, stackTrace);
        return Response.internalServerError(body: '$error');
126
      }
127
    }).add(createBuildDirectoryHandler(
128
      path.join(macrobenchmarksDirectory, 'build', 'web'),
129 130 131 132 133 134 135
    ));

    server = await io.HttpServer.bind('localhost', benchmarkServerPort);
    try {
      shelf_io.serveRequests(server, cascade.handler);

      final String dartToolDirectory = path.join('$macrobenchmarksDirectory/.dart_tool');
136
      final String userDataDir = io.Directory(dartToolDirectory).createTempSync('flutter_chrome_user_data.').path;
137

138 139 140 141 142 143 144
      // TODO(yjbanov): temporarily disables headful Chrome until we get
      //                devicelab hardware that is able to run it. Our current
      //                GCE VMs can only run in headless mode.
      //                See: https://github.com/flutter/flutter/issues/50164
      final bool isUncalibratedSmokeTest = io.Platform.environment['CALIBRATED'] != 'true';
      // final bool isUncalibratedSmokeTest =
      //     io.Platform.environment['UNCALIBRATED_SMOKE_TEST'] == 'true';
145 146 147 148
      final ChromeOptions options = ChromeOptions(
        url: 'http://localhost:$benchmarkServerPort/index.html',
        userDataDirectory: userDataDir,
        headless: isUncalibratedSmokeTest,
149
        debugPort: chromeDebugPort,
150
        enableWasmGC: benchmarkOptions.useWasm,
151
      );
152

153
      print('Launching Chrome.');
154
      whenChromeIsReady = Chrome.launch(
155 156 157 158
        options,
        onError: (String error) {
          profileData.completeError(Exception(error));
        },
159 160 161 162 163 164 165
        workingDirectory: cwd,
      );

      print('Waiting for the benchmark to report benchmark profile.');
      final Map<String, dynamic> taskResult = <String, dynamic>{};
      final List<String> benchmarkScoreKeys = <String>[];
      final List<Map<String, dynamic>> profiles = await profileData.future;
166

167 168 169
      print('Received profile data');
      for (final Map<String, dynamic> profile in profiles) {
        final String benchmarkName = profile['name'] as String;
170 171 172 173
        if (benchmarkName.isEmpty) {
          throw 'Benchmark name is empty';
        }

174
        final String namespace = '$benchmarkName.${benchmarkOptions.webRenderer}';
175
        final List<String> scoreKeys = List<String>.from(profile['scoreKeys'] as List<dynamic>);
176
        if (scoreKeys.isEmpty) {
177 178 179
          throw 'No score keys in benchmark "$benchmarkName"';
        }
        for (final String scoreKey in scoreKeys) {
180
          if (scoreKey.isEmpty) {
181 182 183 184 185 186 187 188 189 190 191 192
            throw 'Score key is empty in benchmark "$benchmarkName". '
                'Received [${scoreKeys.join(', ')}]';
          }
          benchmarkScoreKeys.add('$namespace.$scoreKey');
        }

        for (final String key in profile.keys) {
          if (key == 'name' || key == 'scoreKeys') {
            continue;
          }
          taskResult['$namespace.$key'] = profile[key];
        }
193 194 195
      }
      return TaskResult.success(taskResult, benchmarkScoreKeys: benchmarkScoreKeys);
    } finally {
196
      unawaited(server.close());
197
      chrome?.stop();
198 199 200
    }
  });
}
201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220

Handler createBuildDirectoryHandler(String buildDirectoryPath) {
  final Handler childHandler = createStaticHandler(buildDirectoryPath);
  return (Request request) async {
    final Response response = await childHandler(request);
    final String? mimeType = response.mimeType;

    // Provide COOP/COEP headers so that the browser loads the page as
    // crossOriginIsolated. This will make sure that we get high-resolution
    // timers for our benchmark measurements.
    if (mimeType == 'text/html' || mimeType == 'text/javascript') {
      return response.change(headers: <String, String>{
        'Cross-Origin-Opener-Policy': 'same-origin',
        'Cross-Origin-Embedder-Policy': 'require-corp',
      });
    } else {
      return response;
    }
  };
}