analyze_size.dart 16.9 KB
Newer Older
1 2 3 4
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5 6
import 'package:archive/archive.dart';
import 'package:archive/archive_io.dart';
7 8
import 'package:meta/meta.dart';
import 'package:vm_snapshot_analysis/treemap.dart';
9 10

import '../convert.dart';
11
import '../reporting/reporting.dart';
12
import 'common.dart';
13
import 'file_system.dart';
14 15 16 17 18 19
import 'logger.dart';
import 'terminal.dart';

/// A class to analyze APK and AOT snapshot and generate a breakdown of the data.
class SizeAnalyzer {
  SizeAnalyzer({
20 21 22
    required FileSystem fileSystem,
    required Logger logger,
    required Usage flutterUsage,
23 24 25 26 27 28 29 30 31 32
    Pattern appFilenamePattern = 'libapp.so',
  }) : _flutterUsage = flutterUsage,
       _fileSystem = fileSystem,
       _logger = logger,
       _appFilenamePattern = appFilenamePattern;

  final FileSystem _fileSystem;
  final Logger _logger;
  final Pattern _appFilenamePattern;
  final Usage _flutterUsage;
33
  String? _appFilename;
34 35 36

  static const String aotSnapshotFileName = 'aot-snapshot.json';
  static const int tableWidth = 80;
37 38 39 40
  static const int _kAotSizeMaxDepth = 2;
  static const int _kZipSizeMaxDepth = 1;

  /// Analyze the [aotSnapshot] in an uncompressed output directory.
41 42 43 44 45 46
  Future<Map<String, Object?>> analyzeAotSnapshot({
    required Directory outputDirectory,
    required File aotSnapshot,
    required File precompilerTrace,
    required String type,
    String? excludePath,
47
  }) async {
48 49
    _logger.printStatus('▒' * tableWidth);
    _logger.printStatus('━' * tableWidth);
50 51 52 53 54 55 56
    final _SymbolNode aotAnalysisJson = _parseDirectory(
      outputDirectory,
      outputDirectory.parent.path,
      excludePath,
    );

    // Convert an AOT snapshot file into a map.
57 58 59 60 61 62
    final Object? decodedAotSnapshot = json.decode(aotSnapshot.readAsStringSync());
    if (decodedAotSnapshot == null) {
      throwToolExit('AOT snapshot is invalid for analysis');
    }
    final Map<String, Object?> processedAotSnapshotJson = treemapFromJson(decodedAotSnapshot);
    final _SymbolNode? aotSnapshotJsonRoot = _parseAotSnapshot(processedAotSnapshotJson);
63 64 65 66 67 68 69 70

    for (final _SymbolNode firstLevelPath in aotAnalysisJson.children) {
      _printEntitySize(
        firstLevelPath.name,
        byteSize: firstLevelPath.byteSize,
        level: 1,
      );
      // Print the expansion of lib directory to show more info for `appFilename`.
71
      if (firstLevelPath.name == _fileSystem.path.basename(outputDirectory.path) && aotSnapshotJsonRoot != null) {
72 73 74 75
        _printLibChildrenPaths(firstLevelPath, '', aotSnapshotJsonRoot, _kAotSizeMaxDepth, 0);
      }
    }

76
    _logger.printStatus('▒' * tableWidth);
77

78
    Map<String, Object?> apkAnalysisJson = aotAnalysisJson.toJson();
79 80 81 82 83 84 85

    apkAnalysisJson['type'] = type; // one of apk, aab, ios, macos, windows, or linux.

    apkAnalysisJson = _addAotSnapshotDataToAnalysis(
      apkAnalysisJson: apkAnalysisJson,
      path: _locatedAotFilePath,
      aotSnapshotJson: processedAotSnapshotJson,
86
      precompilerTrace: json.decode(precompilerTrace.readAsStringSync()) as Map<String, Object?>? ?? <String, Object?>{},
87 88 89
    );

    assert(_appFilename != null);
90
    CodeSizeEvent(type, flutterUsage: _flutterUsage).send();
91 92 93
    return apkAnalysisJson;
  }

94 95
  /// Analyzes [apk] and [aotSnapshot] to output a [Map] object that includes
  /// the breakdown of the both files, where the breakdown of [aotSnapshot] is placed
96
  /// under 'lib/arm64-v8a/$_appFilename'.
97
  ///
98 99
  /// [kind] must be one of 'apk' or 'aab'.
  /// The [aotSnapshot] can be either instruction sizes snapshot or a v8 snapshot.
100 101 102 103 104
  Future<Map<String, Object?>> analyzeZipSizeAndAotSnapshot({
    required File zipFile,
    required File aotSnapshot,
    required File precompilerTrace,
    required String kind,
105
  }) async {
106
    assert(kind == 'apk' || kind == 'aab');
107
    _logger.printStatus('▒' * tableWidth);
108
    _printEntitySize(
109 110
      '${zipFile.basename} (total compressed)',
      byteSize: zipFile.lengthSync(),
111 112 113
      level: 0,
      showColor: false,
    );
114
    _logger.printStatus('━' * tableWidth);
115

116
    final _SymbolNode apkAnalysisRoot = _parseUnzipFile(zipFile);
117 118

    // Convert an AOT snapshot file into a map.
119 120 121 122 123 124 125 126 127 128
    final Object? decodedAotSnapshot = json.decode(aotSnapshot.readAsStringSync());
    if (decodedAotSnapshot == null) {
      throwToolExit('AOT snapshot is invalid for analysis');
    }
    final Map<String, Object?> processedAotSnapshotJson = treemapFromJson(decodedAotSnapshot);
    final _SymbolNode? aotSnapshotJsonRoot = _parseAotSnapshot(processedAotSnapshotJson);
    if (aotSnapshotJsonRoot != null) {
      for (final _SymbolNode firstLevelPath in apkAnalysisRoot.children) {
        _printLibChildrenPaths(firstLevelPath, '', aotSnapshotJsonRoot, _kZipSizeMaxDepth, 0);
      }
129
    }
130
    _logger.printStatus('▒' * tableWidth);
131

132
    Map<String, Object?> apkAnalysisJson = apkAnalysisRoot.toJson();
133

134
    apkAnalysisJson['type'] = kind;
135

136
    assert(_appFilename != null);
137
    apkAnalysisJson = _addAotSnapshotDataToAnalysis(
138
      apkAnalysisJson: apkAnalysisJson,
139
      path: _locatedAotFilePath,
140
      aotSnapshotJson: processedAotSnapshotJson,
141
      precompilerTrace: json.decode(precompilerTrace.readAsStringSync()) as Map<String, Object?>? ?? <String, Object?>{},
142
    );
143
    CodeSizeEvent(kind, flutterUsage: _flutterUsage).send();
144 145 146
    return apkAnalysisJson;
  }

147 148 149
  _SymbolNode _parseUnzipFile(File zipFile) {
    final Archive archive = ZipDecoder().decodeBytes(zipFile.readAsBytesSync());
    final Map<List<String>, int> pathsToSize = <List<String>, int>{};
150

151
    for (final ArchiveFile archiveFile in archive.files) {
152
      final InputStreamBase? rawContent = archiveFile.rawContent;
153 154 155
      if (rawContent != null) {
        pathsToSize[_fileSystem.path.split(archiveFile.name)] = rawContent.length;
      }
156 157 158
    }
    return _buildSymbolTree(pathsToSize);
  }
159

160
  _SymbolNode _parseDirectory(Directory directory, String relativeTo, String? excludePath) {
161
    final Map<List<String>, int> pathsToSize = <List<String>, int>{};
162 163
    for (final File file in directory.listSync(recursive: true).whereType<File>()) {
      if (excludePath != null && file.uri.pathSegments.contains(excludePath)) {
164 165
        continue;
      }
166 167
      final List<String> path = _fileSystem.path.split(
        _fileSystem.path.relative(file.path, from: relativeTo));
168
      pathsToSize[path] = file.lengthSync();
169
    }
170 171 172
    return _buildSymbolTree(pathsToSize);
  }

173
  List<String> _locatedAotFilePath = <String>[];
174

175
  List<String> _buildNodeName(_SymbolNode start, _SymbolNode? parent) {
176 177 178 179 180 181 182
    final List<String> results = <String>[start.name];
    while (parent != null && parent.name != 'Root') {
      results.insert(0, parent.name);
      parent = parent.parent;
    }
    return results;
  }
183

184 185
  _SymbolNode _buildSymbolTree(Map<List<String>, int> pathsToSize) {
     final _SymbolNode rootNode = _SymbolNode('Root');
186
    _SymbolNode currentNode = rootNode;
187

188 189
    for (final List<String> paths in pathsToSize.keys) {
      for (final String path in paths) {
190
        _SymbolNode? childWithPathAsName = currentNode.childByName(path);
191 192 193

        if (childWithPathAsName == null) {
          childWithPathAsName = _SymbolNode(path);
194
          if (matchesPattern(path, pattern: _appFilenamePattern) != null) {
195
            _appFilename = path;
196
            childWithPathAsName.name += ' (Dart AOT)';
197
            _locatedAotFilePath = _buildNodeName(childWithPathAsName, currentNode);
198
          } else if (path == 'libflutter.so') {
199 200 201 202
            childWithPathAsName.name += ' (Flutter Engine)';
          }
          currentNode.addChild(childWithPathAsName);
        }
203
        childWithPathAsName.addSize(pathsToSize[paths] ?? 0);
204 205 206 207 208 209 210 211 212
        currentNode = childWithPathAsName;
      }
      currentNode = rootNode;
    }
    return rootNode;
  }

  /// Prints all children paths for the lib/ directory in an APK.
  ///
213
  /// A brief summary of aot snapshot is printed under 'lib/arm64-v8a/$_appFilename'.
214 215 216 217
  void _printLibChildrenPaths(
    _SymbolNode currentNode,
    String totalPath,
    _SymbolNode aotSnapshotJsonRoot,
218 219
    int maxDepth,
    int currentDepth,
220 221 222
  ) {
    totalPath += currentNode.name;

223 224
    assert(_appFilename != null);
    if (currentNode.children.isNotEmpty
225 226 227
      && currentNode.name != '$_appFilename (Dart AOT)'
      && currentDepth < maxDepth
      && currentNode.byteSize >= 1000) {
228
      for (final _SymbolNode child in currentNode.children) {
229
        _printLibChildrenPaths(child, '$totalPath/', aotSnapshotJsonRoot, maxDepth, currentDepth + 1);
230
      }
231 232
      _leadingPaths = totalPath.split('/')
        ..removeLast();
233
    } else {
234 235 236 237 238 239 240 241 242 243
      // Print total path and size if currentNode does not have any children and is
      // larger than 1KB
      final bool isAotSnapshotPath = _locatedAotFilePath.join('/').contains(totalPath);
      if (currentNode.byteSize >= 1000 || isAotSnapshotPath) {
        _printEntitySize(totalPath, byteSize: currentNode.byteSize, level: 1, emphasis: currentNode.children.isNotEmpty);
        if (isAotSnapshotPath) {
          _printAotSnapshotSummary(aotSnapshotJsonRoot, level: totalPath.split('/').length);
        }
        _leadingPaths = totalPath.split('/')
          ..removeLast();
244 245 246 247 248 249
      }
    }
  }

  /// Go through the AOT gen snapshot size JSON and print out a collapsed summary
  /// for the first package level.
250
  void _printAotSnapshotSummary(_SymbolNode aotSnapshotRoot, {int maxDirectoriesShown = 20, required int level}) {
251 252 253
    _printEntitySize(
      'Dart AOT symbols accounted decompressed size',
      byteSize: aotSnapshotRoot.byteSize,
254 255
      level: level,
      emphasis: true,
256 257 258
    );

    final List<_SymbolNode> sortedSymbols = aotSnapshotRoot.children.toList()
259 260 261
      // Remove entries like  @unknown, @shared, and @stubs as well as private dart libraries
      //  which are not interpretable by end users.
      ..removeWhere((_SymbolNode node) => node.name.startsWith('@') || node.name.startsWith('dart:_'))
262 263
      ..sort((_SymbolNode a, _SymbolNode b) => b.byteSize.compareTo(a.byteSize));
    for (final _SymbolNode node in sortedSymbols.take(maxDirectoriesShown)) {
264 265 266 267 268 269 270 271 272
      // Node names will have an extra leading `package:*` name, remove it to
      // avoid extra nesting.
      _printEntitySize(_formatExtraLeadingPackages(node.name), byteSize: node.byteSize, level: level + 1);
    }
  }

  String _formatExtraLeadingPackages(String name) {
    if (!name.startsWith('package')) {
      return name;
273
    }
274 275 276 277 278 279
    final List<String> chunks = name.split('/');
    if (chunks.length < 2) {
      return name;
    }
    chunks.removeAt(0);
    return chunks.join('/');
280 281 282
  }

  /// Adds breakdown of aot snapshot data as the children of the node at the given path.
283 284 285 286 287
  Map<String, Object?> _addAotSnapshotDataToAnalysis({
    required Map<String, Object?> apkAnalysisJson,
    required List<String> path,
    required Map<String, Object?> aotSnapshotJson,
    required Map<String, Object?> precompilerTrace,
288
  }) {
289
    Map<String, Object?> currentLevel = apkAnalysisJson;
290
    currentLevel['precompiler-trace'] = precompilerTrace;
291
    while (path.isNotEmpty) {
292 293 294 295
      final List<Map<String, Object?>>? children = currentLevel['children'] as List<Map<String, Object?>>?;
      final Map<String, Object?> childWithPathAsName = children?.firstWhere(
        (Map<String, Object?> child) => (child['n'] as String?) == path.first,
      ) ?? <String, Object?>{};
296 297 298 299 300 301 302
      path.removeAt(0);
      currentLevel = childWithPathAsName;
    }
    currentLevel['children'] = aotSnapshotJson['children'];
    return apkAnalysisJson;
  }

303 304
  List<String> _leadingPaths = <String>[];

305 306 307
  /// Print an entity's name with its size on the same line.
  void _printEntitySize(
    String entityName, {
308 309
    required int byteSize,
    required int level,
310
    bool showColor = true,
311
    bool emphasis = false,
312 313 314 315 316 317 318 319 320 321
  }) {
    final String formattedSize = _prettyPrintBytes(byteSize);

    TerminalColor color = TerminalColor.green;
    if (formattedSize.endsWith('MB')) {
      color = TerminalColor.cyan;
    } else if (formattedSize.endsWith('KB')) {
      color = TerminalColor.yellow;
    }

322 323 324
    // Compute any preceding directories, and compare this to the stored
    // directories (in _leadingPaths) for the last entity that was printed. The
    // similarly determines whether or not leading directory information needs to
325 326 327 328 329 330 331 332
    // be printed.
    final List<String> localSegments = entityName.split('/')
        ..removeLast();
    int i = 0;
    while (i < _leadingPaths.length && i < localSegments.length && _leadingPaths[i] == localSegments[i]) {
      i += 1;
    }
    for (; i < localSegments.length; i += 1) {
333
      _logger.printStatus(
334
        '${localSegments[i]}/',
335 336 337 338 339 340
        indent: (level + i) * 2,
        emphasis: true,
      );
    }
    _leadingPaths = localSegments;

341
    final String baseName = _fileSystem.path.basename(entityName);
342
    final int spaceInBetween = tableWidth - (level + i) * 2 - baseName.length - formattedSize.length;
343
    _logger.printStatus(
344
      baseName + ' ' * spaceInBetween,
345 346
      newline: false,
      emphasis: emphasis,
347
      indent: (level + i) * 2,
348
    );
349
    _logger.printStatus(formattedSize, color: showColor ? color : null);
350 351 352 353 354 355 356 357 358 359 360 361 362 363
  }

  String _prettyPrintBytes(int numBytes) {
    const int kB = 1024;
    const int mB = kB * 1024;
    if (numBytes < kB) {
      return '$numBytes B';
    } else if (numBytes < mB) {
      return '${(numBytes / kB).round()} KB';
    } else {
      return '${(numBytes / mB).round()} MB';
    }
  }

364
  _SymbolNode? _parseAotSnapshot(Map<String, Object?> aotSnapshotJson) {
365 366 367 368 369
    final bool isLeafNode = aotSnapshotJson['children'] == null;
    if (!isLeafNode) {
      return _buildNodeWithChildren(aotSnapshotJson);
    } else {
      // TODO(peterdjlee): Investigate why there are leaf nodes with size of null.
370
      final int? byteSize = aotSnapshotJson['value'] as int?;
371 372 373 374 375 376 377 378
      if (byteSize == null) {
        return null;
      }
      return _buildNode(aotSnapshotJson, byteSize);
    }
  }

  _SymbolNode _buildNode(
379
    Map<String, Object?> aotSnapshotJson,
380 381 382
    int byteSize, {
    List<_SymbolNode> children = const <_SymbolNode>[],
  }) {
383
    final String name = aotSnapshotJson['n']! as String;
384 385 386 387 388 389 390 391 392 393 394 395 396 397
    final Map<String, _SymbolNode> childrenMap = <String, _SymbolNode>{};

    for (final _SymbolNode child in children) {
      childrenMap[child.name] = child;
    }

    return _SymbolNode(
      name,
      byteSize: byteSize,
    )..addAllChildren(children);
  }

  /// Builds a node by recursively building all of its children first
  /// in order to calculate the sum of its children's sizes.
398 399
  _SymbolNode? _buildNodeWithChildren(Map<String, Object?> aotSnapshotJson) {
    final List<Object?> rawChildren = aotSnapshotJson['children'] as List<Object?>? ?? <Object?>[];
400 401 402 403
    final List<_SymbolNode> symbolNodeChildren = <_SymbolNode>[];
    int totalByteSize = 0;

    // Given a child, build its subtree.
404 405 406 407 408 409 410 411 412
    for (final Object? child in rawChildren) {
      if (child == null) {
        continue;
      }
      final _SymbolNode? childTreemapNode = _parseAotSnapshot(child as Map<String, Object?>);
      if (childTreemapNode != null) {
        symbolNodeChildren.add(childTreemapNode);
        totalByteSize += childTreemapNode.byteSize;
      }
413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444
    }

    // If none of the children matched the diff tree type
    if (totalByteSize == 0) {
      return null;
    } else {
      return _buildNode(
        aotSnapshotJson,
        totalByteSize,
        children: symbolNodeChildren,
      );
    }
  }
}

/// A node class that represents a single symbol for AOT size snapshots.
class _SymbolNode {
  _SymbolNode(
    this.name, {
    this.byteSize = 0,
  })  : assert(name != null),
        assert(byteSize != null),
        _children = <String, _SymbolNode>{};

  /// The human friendly identifier for this node.
  String name;

  int byteSize;
  void addSize(int sizeToBeAdded) {
    byteSize += sizeToBeAdded;
  }

445 446
  _SymbolNode? get parent => _parent;
  _SymbolNode? _parent;
447 448 449 450

  Iterable<_SymbolNode> get children => _children.values;
  final Map<String, _SymbolNode> _children;

451
  _SymbolNode? childByName(String name) => _children[name];
452 453 454 455 456 457 458 459 460 461 462 463 464 465 466

  _SymbolNode addChild(_SymbolNode child) {
    assert(child.parent == null);
    assert(!_children.containsKey(child.name),
        'Cannot add duplicate child key ${child.name}');

    child._parent = this;
    _children[child.name] = child;
    return child;
  }

  void addAllChildren(List<_SymbolNode> children) {
    children.forEach(addChild);
  }

467 468
  Map<String, Object?> toJson() {
    final Map<String, Object?> json = <String, Object?>{
469
      'n': name,
470
      'value': byteSize,
471
    };
472
    final List<Map<String, Object?>> childrenAsJson = <Map<String, Object?>>[];
473 474 475 476 477 478 479 480 481
    for (final _SymbolNode child in children) {
      childrenAsJson.add(child.toJson());
    }
    if (childrenAsJson.isNotEmpty) {
      json['children'] = childrenAsJson;
    }
    return json;
  }
}
482 483 484

/// Matches `pattern` against the entirety of `string`.
@visibleForTesting
485 486
Match? matchesPattern(String string, {required Pattern pattern}) {
  final Match? match = pattern.matchAsPrefix(string);
487 488
  return (match != null && match.end == string.length) ? match : null;
}