analyze_size.dart 17.1 KB
Newer Older
1 2 3 4
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5 6
import 'package:archive/archive.dart';
import 'package:archive/archive_io.dart';
7
import 'package:meta/meta.dart';
8
import 'package:unified_analytics/unified_analytics.dart';
9
import 'package:vm_snapshot_analysis/treemap.dart';
10 11

import '../convert.dart';
12
import '../reporting/reporting.dart';
13
import 'common.dart';
14
import 'file_system.dart';
15 16 17 18 19 20
import 'logger.dart';
import 'terminal.dart';

/// A class to analyze APK and AOT snapshot and generate a breakdown of the data.
class SizeAnalyzer {
  SizeAnalyzer({
21 22 23
    required FileSystem fileSystem,
    required Logger logger,
    required Usage flutterUsage,
24
    required Analytics analytics,
25 26
    Pattern appFilenamePattern = 'libapp.so',
  }) : _flutterUsage = flutterUsage,
27
       _analytics = analytics,
28 29 30 31 32 33 34 35
       _fileSystem = fileSystem,
       _logger = logger,
       _appFilenamePattern = appFilenamePattern;

  final FileSystem _fileSystem;
  final Logger _logger;
  final Pattern _appFilenamePattern;
  final Usage _flutterUsage;
36
  final Analytics _analytics;
37
  String? _appFilename;
38 39 40

  static const String aotSnapshotFileName = 'aot-snapshot.json';
  static const int tableWidth = 80;
41 42 43 44
  static const int _kAotSizeMaxDepth = 2;
  static const int _kZipSizeMaxDepth = 1;

  /// Analyze the [aotSnapshot] in an uncompressed output directory.
45 46 47 48 49 50
  Future<Map<String, Object?>> analyzeAotSnapshot({
    required Directory outputDirectory,
    required File aotSnapshot,
    required File precompilerTrace,
    required String type,
    String? excludePath,
51
  }) async {
52 53
    _logger.printStatus('▒' * tableWidth);
    _logger.printStatus('━' * tableWidth);
54 55 56 57 58 59 60
    final _SymbolNode aotAnalysisJson = _parseDirectory(
      outputDirectory,
      outputDirectory.parent.path,
      excludePath,
    );

    // Convert an AOT snapshot file into a map.
61 62 63 64 65 66
    final Object? decodedAotSnapshot = json.decode(aotSnapshot.readAsStringSync());
    if (decodedAotSnapshot == null) {
      throwToolExit('AOT snapshot is invalid for analysis');
    }
    final Map<String, Object?> processedAotSnapshotJson = treemapFromJson(decodedAotSnapshot);
    final _SymbolNode? aotSnapshotJsonRoot = _parseAotSnapshot(processedAotSnapshotJson);
67 68 69 70 71 72 73 74

    for (final _SymbolNode firstLevelPath in aotAnalysisJson.children) {
      _printEntitySize(
        firstLevelPath.name,
        byteSize: firstLevelPath.byteSize,
        level: 1,
      );
      // Print the expansion of lib directory to show more info for `appFilename`.
75
      if (firstLevelPath.name == _fileSystem.path.basename(outputDirectory.path) && aotSnapshotJsonRoot != null) {
76 77 78 79
        _printLibChildrenPaths(firstLevelPath, '', aotSnapshotJsonRoot, _kAotSizeMaxDepth, 0);
      }
    }

80
    _logger.printStatus('▒' * tableWidth);
81

82
    Map<String, Object?> apkAnalysisJson = aotAnalysisJson.toJson();
83 84 85 86 87 88 89

    apkAnalysisJson['type'] = type; // one of apk, aab, ios, macos, windows, or linux.

    apkAnalysisJson = _addAotSnapshotDataToAnalysis(
      apkAnalysisJson: apkAnalysisJson,
      path: _locatedAotFilePath,
      aotSnapshotJson: processedAotSnapshotJson,
90
      precompilerTrace: json.decode(precompilerTrace.readAsStringSync()) as Map<String, Object?>? ?? <String, Object?>{},
91 92 93
    );

    assert(_appFilename != null);
94
    CodeSizeEvent(type, flutterUsage: _flutterUsage).send();
95
    _analytics.send(Event.codeSizeAnalysis(platform: type));
96 97 98
    return apkAnalysisJson;
  }

99 100
  /// Analyzes [apk] and [aotSnapshot] to output a [Map] object that includes
  /// the breakdown of the both files, where the breakdown of [aotSnapshot] is placed
101
  /// under 'lib/arm64-v8a/$_appFilename'.
102
  ///
103 104
  /// [kind] must be one of 'apk' or 'aab'.
  /// The [aotSnapshot] can be either instruction sizes snapshot or a v8 snapshot.
105 106 107 108 109
  Future<Map<String, Object?>> analyzeZipSizeAndAotSnapshot({
    required File zipFile,
    required File aotSnapshot,
    required File precompilerTrace,
    required String kind,
110
  }) async {
111
    assert(kind == 'apk' || kind == 'aab');
112
    _logger.printStatus('▒' * tableWidth);
113
    _printEntitySize(
114 115
      '${zipFile.basename} (total compressed)',
      byteSize: zipFile.lengthSync(),
116 117 118
      level: 0,
      showColor: false,
    );
119
    _logger.printStatus('━' * tableWidth);
120

121
    final _SymbolNode apkAnalysisRoot = _parseUnzipFile(zipFile);
122 123

    // Convert an AOT snapshot file into a map.
124 125 126 127 128 129 130 131 132 133
    final Object? decodedAotSnapshot = json.decode(aotSnapshot.readAsStringSync());
    if (decodedAotSnapshot == null) {
      throwToolExit('AOT snapshot is invalid for analysis');
    }
    final Map<String, Object?> processedAotSnapshotJson = treemapFromJson(decodedAotSnapshot);
    final _SymbolNode? aotSnapshotJsonRoot = _parseAotSnapshot(processedAotSnapshotJson);
    if (aotSnapshotJsonRoot != null) {
      for (final _SymbolNode firstLevelPath in apkAnalysisRoot.children) {
        _printLibChildrenPaths(firstLevelPath, '', aotSnapshotJsonRoot, _kZipSizeMaxDepth, 0);
      }
134
    }
135
    _logger.printStatus('▒' * tableWidth);
136

137
    Map<String, Object?> apkAnalysisJson = apkAnalysisRoot.toJson();
138

139
    apkAnalysisJson['type'] = kind;
140

141
    assert(_appFilename != null);
142
    apkAnalysisJson = _addAotSnapshotDataToAnalysis(
143
      apkAnalysisJson: apkAnalysisJson,
144
      path: _locatedAotFilePath,
145
      aotSnapshotJson: processedAotSnapshotJson,
146
      precompilerTrace: json.decode(precompilerTrace.readAsStringSync()) as Map<String, Object?>? ?? <String, Object?>{},
147
    );
148
    CodeSizeEvent(kind, flutterUsage: _flutterUsage).send();
149
    _analytics.send(Event.codeSizeAnalysis(platform: kind));
150 151 152
    return apkAnalysisJson;
  }

153 154 155
  _SymbolNode _parseUnzipFile(File zipFile) {
    final Archive archive = ZipDecoder().decodeBytes(zipFile.readAsBytesSync());
    final Map<List<String>, int> pathsToSize = <List<String>, int>{};
156

157
    for (final ArchiveFile archiveFile in archive.files) {
158
      final InputStreamBase? rawContent = archiveFile.rawContent;
159 160 161
      if (rawContent != null) {
        pathsToSize[_fileSystem.path.split(archiveFile.name)] = rawContent.length;
      }
162 163 164
    }
    return _buildSymbolTree(pathsToSize);
  }
165

166
  _SymbolNode _parseDirectory(Directory directory, String relativeTo, String? excludePath) {
167
    final Map<List<String>, int> pathsToSize = <List<String>, int>{};
168 169
    for (final File file in directory.listSync(recursive: true).whereType<File>()) {
      if (excludePath != null && file.uri.pathSegments.contains(excludePath)) {
170 171
        continue;
      }
172 173
      final List<String> path = _fileSystem.path.split(
        _fileSystem.path.relative(file.path, from: relativeTo));
174
      pathsToSize[path] = file.lengthSync();
175
    }
176 177 178
    return _buildSymbolTree(pathsToSize);
  }

179
  List<String> _locatedAotFilePath = <String>[];
180

181
  List<String> _buildNodeName(_SymbolNode start, _SymbolNode? parent) {
182 183 184 185 186 187 188
    final List<String> results = <String>[start.name];
    while (parent != null && parent.name != 'Root') {
      results.insert(0, parent.name);
      parent = parent.parent;
    }
    return results;
  }
189

190 191
  _SymbolNode _buildSymbolTree(Map<List<String>, int> pathsToSize) {
     final _SymbolNode rootNode = _SymbolNode('Root');
192
    _SymbolNode currentNode = rootNode;
193

194 195
    for (final List<String> paths in pathsToSize.keys) {
      for (final String path in paths) {
196
        _SymbolNode? childWithPathAsName = currentNode.childByName(path);
197 198 199

        if (childWithPathAsName == null) {
          childWithPathAsName = _SymbolNode(path);
200
          if (matchesPattern(path, pattern: _appFilenamePattern) != null) {
201
            _appFilename = path;
202
            childWithPathAsName.name += ' (Dart AOT)';
203
            _locatedAotFilePath = _buildNodeName(childWithPathAsName, currentNode);
204
          } else if (path == 'libflutter.so') {
205 206 207 208
            childWithPathAsName.name += ' (Flutter Engine)';
          }
          currentNode.addChild(childWithPathAsName);
        }
209
        childWithPathAsName.addSize(pathsToSize[paths] ?? 0);
210 211 212 213 214 215 216 217 218
        currentNode = childWithPathAsName;
      }
      currentNode = rootNode;
    }
    return rootNode;
  }

  /// Prints all children paths for the lib/ directory in an APK.
  ///
219
  /// A brief summary of aot snapshot is printed under 'lib/arm64-v8a/$_appFilename'.
220 221 222 223
  void _printLibChildrenPaths(
    _SymbolNode currentNode,
    String totalPath,
    _SymbolNode aotSnapshotJsonRoot,
224 225
    int maxDepth,
    int currentDepth,
226 227 228
  ) {
    totalPath += currentNode.name;

229 230
    assert(_appFilename != null);
    if (currentNode.children.isNotEmpty
231 232 233
      && currentNode.name != '$_appFilename (Dart AOT)'
      && currentDepth < maxDepth
      && currentNode.byteSize >= 1000) {
234
      for (final _SymbolNode child in currentNode.children) {
235
        _printLibChildrenPaths(child, '$totalPath/', aotSnapshotJsonRoot, maxDepth, currentDepth + 1);
236
      }
237 238
      _leadingPaths = totalPath.split('/')
        ..removeLast();
239
    } else {
240 241 242 243 244 245 246 247 248 249
      // Print total path and size if currentNode does not have any children and is
      // larger than 1KB
      final bool isAotSnapshotPath = _locatedAotFilePath.join('/').contains(totalPath);
      if (currentNode.byteSize >= 1000 || isAotSnapshotPath) {
        _printEntitySize(totalPath, byteSize: currentNode.byteSize, level: 1, emphasis: currentNode.children.isNotEmpty);
        if (isAotSnapshotPath) {
          _printAotSnapshotSummary(aotSnapshotJsonRoot, level: totalPath.split('/').length);
        }
        _leadingPaths = totalPath.split('/')
          ..removeLast();
250 251 252 253 254 255
      }
    }
  }

  /// Go through the AOT gen snapshot size JSON and print out a collapsed summary
  /// for the first package level.
256
  void _printAotSnapshotSummary(_SymbolNode aotSnapshotRoot, {int maxDirectoriesShown = 20, required int level}) {
257 258 259
    _printEntitySize(
      'Dart AOT symbols accounted decompressed size',
      byteSize: aotSnapshotRoot.byteSize,
260 261
      level: level,
      emphasis: true,
262 263 264
    );

    final List<_SymbolNode> sortedSymbols = aotSnapshotRoot.children.toList()
265 266 267
      // Remove entries like  @unknown, @shared, and @stubs as well as private dart libraries
      //  which are not interpretable by end users.
      ..removeWhere((_SymbolNode node) => node.name.startsWith('@') || node.name.startsWith('dart:_'))
268 269
      ..sort((_SymbolNode a, _SymbolNode b) => b.byteSize.compareTo(a.byteSize));
    for (final _SymbolNode node in sortedSymbols.take(maxDirectoriesShown)) {
270 271 272 273 274 275 276 277 278
      // Node names will have an extra leading `package:*` name, remove it to
      // avoid extra nesting.
      _printEntitySize(_formatExtraLeadingPackages(node.name), byteSize: node.byteSize, level: level + 1);
    }
  }

  String _formatExtraLeadingPackages(String name) {
    if (!name.startsWith('package')) {
      return name;
279
    }
280 281 282 283 284 285
    final List<String> chunks = name.split('/');
    if (chunks.length < 2) {
      return name;
    }
    chunks.removeAt(0);
    return chunks.join('/');
286 287 288
  }

  /// Adds breakdown of aot snapshot data as the children of the node at the given path.
289 290 291 292 293
  Map<String, Object?> _addAotSnapshotDataToAnalysis({
    required Map<String, Object?> apkAnalysisJson,
    required List<String> path,
    required Map<String, Object?> aotSnapshotJson,
    required Map<String, Object?> precompilerTrace,
294
  }) {
295
    Map<String, Object?> currentLevel = apkAnalysisJson;
296
    currentLevel['precompiler-trace'] = precompilerTrace;
297
    while (path.isNotEmpty) {
298 299 300 301
      final List<Map<String, Object?>>? children = currentLevel['children'] as List<Map<String, Object?>>?;
      final Map<String, Object?> childWithPathAsName = children?.firstWhere(
        (Map<String, Object?> child) => (child['n'] as String?) == path.first,
      ) ?? <String, Object?>{};
302 303 304 305 306 307 308
      path.removeAt(0);
      currentLevel = childWithPathAsName;
    }
    currentLevel['children'] = aotSnapshotJson['children'];
    return apkAnalysisJson;
  }

309 310
  List<String> _leadingPaths = <String>[];

311 312 313
  /// Print an entity's name with its size on the same line.
  void _printEntitySize(
    String entityName, {
314 315
    required int byteSize,
    required int level,
316
    bool showColor = true,
317
    bool emphasis = false,
318 319 320 321 322 323 324 325 326 327
  }) {
    final String formattedSize = _prettyPrintBytes(byteSize);

    TerminalColor color = TerminalColor.green;
    if (formattedSize.endsWith('MB')) {
      color = TerminalColor.cyan;
    } else if (formattedSize.endsWith('KB')) {
      color = TerminalColor.yellow;
    }

328 329 330
    // Compute any preceding directories, and compare this to the stored
    // directories (in _leadingPaths) for the last entity that was printed. The
    // similarly determines whether or not leading directory information needs to
331 332 333 334 335 336 337 338
    // be printed.
    final List<String> localSegments = entityName.split('/')
        ..removeLast();
    int i = 0;
    while (i < _leadingPaths.length && i < localSegments.length && _leadingPaths[i] == localSegments[i]) {
      i += 1;
    }
    for (; i < localSegments.length; i += 1) {
339
      _logger.printStatus(
340
        '${localSegments[i]}/',
341 342 343 344 345 346
        indent: (level + i) * 2,
        emphasis: true,
      );
    }
    _leadingPaths = localSegments;

347
    final String baseName = _fileSystem.path.basename(entityName);
348
    final int spaceInBetween = tableWidth - (level + i) * 2 - baseName.length - formattedSize.length;
349
    _logger.printStatus(
350
      baseName + ' ' * spaceInBetween,
351 352
      newline: false,
      emphasis: emphasis,
353
      indent: (level + i) * 2,
354
    );
355
    _logger.printStatus(formattedSize, color: showColor ? color : null);
356 357 358 359 360 361 362 363 364 365 366 367 368 369
  }

  String _prettyPrintBytes(int numBytes) {
    const int kB = 1024;
    const int mB = kB * 1024;
    if (numBytes < kB) {
      return '$numBytes B';
    } else if (numBytes < mB) {
      return '${(numBytes / kB).round()} KB';
    } else {
      return '${(numBytes / mB).round()} MB';
    }
  }

370
  _SymbolNode? _parseAotSnapshot(Map<String, Object?> aotSnapshotJson) {
371 372 373 374 375
    final bool isLeafNode = aotSnapshotJson['children'] == null;
    if (!isLeafNode) {
      return _buildNodeWithChildren(aotSnapshotJson);
    } else {
      // TODO(peterdjlee): Investigate why there are leaf nodes with size of null.
376
      final int? byteSize = aotSnapshotJson['value'] as int?;
377 378 379 380 381 382 383 384
      if (byteSize == null) {
        return null;
      }
      return _buildNode(aotSnapshotJson, byteSize);
    }
  }

  _SymbolNode _buildNode(
385
    Map<String, Object?> aotSnapshotJson,
386 387 388
    int byteSize, {
    List<_SymbolNode> children = const <_SymbolNode>[],
  }) {
389
    final String name = aotSnapshotJson['n']! as String;
390 391 392 393 394 395 396 397 398 399 400 401 402 403
    final Map<String, _SymbolNode> childrenMap = <String, _SymbolNode>{};

    for (final _SymbolNode child in children) {
      childrenMap[child.name] = child;
    }

    return _SymbolNode(
      name,
      byteSize: byteSize,
    )..addAllChildren(children);
  }

  /// Builds a node by recursively building all of its children first
  /// in order to calculate the sum of its children's sizes.
404 405
  _SymbolNode? _buildNodeWithChildren(Map<String, Object?> aotSnapshotJson) {
    final List<Object?> rawChildren = aotSnapshotJson['children'] as List<Object?>? ?? <Object?>[];
406 407 408 409
    final List<_SymbolNode> symbolNodeChildren = <_SymbolNode>[];
    int totalByteSize = 0;

    // Given a child, build its subtree.
410 411 412 413 414 415 416 417 418
    for (final Object? child in rawChildren) {
      if (child == null) {
        continue;
      }
      final _SymbolNode? childTreemapNode = _parseAotSnapshot(child as Map<String, Object?>);
      if (childTreemapNode != null) {
        symbolNodeChildren.add(childTreemapNode);
        totalByteSize += childTreemapNode.byteSize;
      }
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438
    }

    // If none of the children matched the diff tree type
    if (totalByteSize == 0) {
      return null;
    } else {
      return _buildNode(
        aotSnapshotJson,
        totalByteSize,
        children: symbolNodeChildren,
      );
    }
  }
}

/// A node class that represents a single symbol for AOT size snapshots.
class _SymbolNode {
  _SymbolNode(
    this.name, {
    this.byteSize = 0,
439
  })  : _children = <String, _SymbolNode>{};
440 441 442 443 444 445 446 447 448

  /// The human friendly identifier for this node.
  String name;

  int byteSize;
  void addSize(int sizeToBeAdded) {
    byteSize += sizeToBeAdded;
  }

449 450
  _SymbolNode? get parent => _parent;
  _SymbolNode? _parent;
451 452 453 454

  Iterable<_SymbolNode> get children => _children.values;
  final Map<String, _SymbolNode> _children;

455
  _SymbolNode? childByName(String name) => _children[name];
456 457 458 459 460 461 462 463 464 465 466 467 468 469 470

  _SymbolNode addChild(_SymbolNode child) {
    assert(child.parent == null);
    assert(!_children.containsKey(child.name),
        'Cannot add duplicate child key ${child.name}');

    child._parent = this;
    _children[child.name] = child;
    return child;
  }

  void addAllChildren(List<_SymbolNode> children) {
    children.forEach(addChild);
  }

471 472
  Map<String, Object?> toJson() {
    final Map<String, Object?> json = <String, Object?>{
473
      'n': name,
474
      'value': byteSize,
475
    };
476
    final List<Map<String, Object?>> childrenAsJson = <Map<String, Object?>>[];
477 478 479 480 481 482 483 484 485
    for (final _SymbolNode child in children) {
      childrenAsJson.add(child.toJson());
    }
    if (childrenAsJson.isNotEmpty) {
      json['children'] = childrenAsJson;
    }
    return json;
  }
}
486 487 488

/// Matches `pattern` against the entirety of `string`.
@visibleForTesting
489 490
Match? matchesPattern(String string, {required Pattern pattern}) {
  final Match? match = pattern.matchAsPrefix(string);
491 492
  return (match != null && match.end == string.length) ? match : null;
}