analyze_size.dart 16.2 KB
Newer Older
1 2 3 4
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5 6
import 'package:archive/archive.dart';
import 'package:archive/archive_io.dart';
7 8
import 'package:meta/meta.dart';
import 'package:vm_snapshot_analysis/treemap.dart';
9 10

import '../convert.dart';
11 12
import '../reporting/reporting.dart';
import 'file_system.dart';
13 14 15 16 17 18
import 'logger.dart';
import 'terminal.dart';

/// A class to analyze APK and AOT snapshot and generate a breakdown of the data.
class SizeAnalyzer {
  SizeAnalyzer({
19 20 21 22 23 24 25 26 27 28 29 30 31 32
    @required FileSystem fileSystem,
    @required Logger logger,
    // TODO(jonahwilliams): migrate to required once this has rolled into google3.
    Usage flutterUsage,
    Pattern appFilenamePattern = 'libapp.so',
  }) : _flutterUsage = flutterUsage,
       _fileSystem = fileSystem,
       _logger = logger,
       _appFilenamePattern = appFilenamePattern;

  final FileSystem _fileSystem;
  final Logger _logger;
  final Pattern _appFilenamePattern;
  final Usage _flutterUsage;
33
  String _appFilename;
34 35 36

  static const String aotSnapshotFileName = 'aot-snapshot.json';
  static const int tableWidth = 80;
37 38 39 40 41 42 43 44 45 46 47
  static const int _kAotSizeMaxDepth = 2;
  static const int _kZipSizeMaxDepth = 1;

  /// Analyze the [aotSnapshot] in an uncompressed output directory.
  Future<Map<String, dynamic>> analyzeAotSnapshot({
    @required Directory outputDirectory,
    @required File aotSnapshot,
    @required File precompilerTrace,
    @required String type,
    String excludePath,
  }) async {
48 49
    _logger.printStatus('▒' * tableWidth);
    _logger.printStatus('━' * tableWidth);
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
    final _SymbolNode aotAnalysisJson = _parseDirectory(
      outputDirectory,
      outputDirectory.parent.path,
      excludePath,
    );

    // Convert an AOT snapshot file into a map.
    final Map<String, dynamic> processedAotSnapshotJson = treemapFromJson(
      json.decode(aotSnapshot.readAsStringSync()),
    );
    final _SymbolNode aotSnapshotJsonRoot = _parseAotSnapshot(processedAotSnapshotJson);

    for (final _SymbolNode firstLevelPath in aotAnalysisJson.children) {
      _printEntitySize(
        firstLevelPath.name,
        byteSize: firstLevelPath.byteSize,
        level: 1,
      );
      // Print the expansion of lib directory to show more info for `appFilename`.
69
      if (firstLevelPath.name == _fileSystem.path.basename(outputDirectory.path)) {
70 71 72 73
        _printLibChildrenPaths(firstLevelPath, '', aotSnapshotJsonRoot, _kAotSizeMaxDepth, 0);
      }
    }

74
    _logger.printStatus('▒' * tableWidth);
75 76 77 78 79 80 81 82 83 84 85 86 87

    Map<String, dynamic> apkAnalysisJson = aotAnalysisJson.toJson();

    apkAnalysisJson['type'] = type; // one of apk, aab, ios, macos, windows, or linux.

    apkAnalysisJson = _addAotSnapshotDataToAnalysis(
      apkAnalysisJson: apkAnalysisJson,
      path: _locatedAotFilePath,
      aotSnapshotJson: processedAotSnapshotJson,
      precompilerTrace: json.decode(precompilerTrace.readAsStringSync()) as Map<String, Object>,
    );

    assert(_appFilename != null);
88
    CodeSizeEvent(type, flutterUsage: _flutterUsage).send();
89 90 91
    return apkAnalysisJson;
  }

92 93
  /// Analyzes [apk] and [aotSnapshot] to output a [Map] object that includes
  /// the breakdown of the both files, where the breakdown of [aotSnapshot] is placed
94
  /// under 'lib/arm64-v8a/$_appFilename'.
95
  ///
96 97 98 99
  /// [kind] must be one of 'apk' or 'aab'.
  /// The [aotSnapshot] can be either instruction sizes snapshot or a v8 snapshot.
  Future<Map<String, dynamic>> analyzeZipSizeAndAotSnapshot({
    @required File zipFile,
100
    @required File aotSnapshot,
101 102
    @required File precompilerTrace,
    @required String kind,
103
  }) async {
104
    assert(kind == 'apk' || kind == 'aab');
105
    _logger.printStatus('▒' * tableWidth);
106
    _printEntitySize(
107 108
      '${zipFile.basename} (total compressed)',
      byteSize: zipFile.lengthSync(),
109 110 111
      level: 0,
      showColor: false,
    );
112
    _logger.printStatus('━' * tableWidth);
113

114
    final _SymbolNode apkAnalysisRoot = _parseUnzipFile(zipFile);
115 116 117 118 119 120 121

    // Convert an AOT snapshot file into a map.
    final Map<String, dynamic> processedAotSnapshotJson = treemapFromJson(
      json.decode(aotSnapshot.readAsStringSync()),
    );
    final _SymbolNode aotSnapshotJsonRoot = _parseAotSnapshot(processedAotSnapshotJson);
    for (final _SymbolNode firstLevelPath in apkAnalysisRoot.children) {
122
      _printLibChildrenPaths(firstLevelPath, '', aotSnapshotJsonRoot, _kZipSizeMaxDepth, 0);
123
    }
124
    _logger.printStatus('▒' * tableWidth);
125 126 127

    Map<String, dynamic> apkAnalysisJson = apkAnalysisRoot.toJson();

128
    apkAnalysisJson['type'] = kind;
129

130
    assert(_appFilename != null);
131
    apkAnalysisJson = _addAotSnapshotDataToAnalysis(
132
      apkAnalysisJson: apkAnalysisJson,
133
      path: _locatedAotFilePath,
134
      aotSnapshotJson: processedAotSnapshotJson,
135
      precompilerTrace: json.decode(precompilerTrace.readAsStringSync()) as Map<String, Object>,
136
    );
137
    CodeSizeEvent(kind, flutterUsage: _flutterUsage).send();
138 139 140
    return apkAnalysisJson;
  }

141 142 143
  _SymbolNode _parseUnzipFile(File zipFile) {
    final Archive archive = ZipDecoder().decodeBytes(zipFile.readAsBytesSync());
    final Map<List<String>, int> pathsToSize = <List<String>, int>{};
144

145
    for (final ArchiveFile archiveFile in archive.files) {
146
      pathsToSize[_fileSystem.path.split(archiveFile.name)] = archiveFile.rawContent.length;
147 148 149
    }
    return _buildSymbolTree(pathsToSize);
  }
150

151
  _SymbolNode _parseDirectory(Directory directory, String relativeTo, String excludePath) {
152
    final Map<List<String>, int> pathsToSize = <List<String>, int>{};
153 154
    for (final File file in directory.listSync(recursive: true).whereType<File>()) {
      if (excludePath != null && file.uri.pathSegments.contains(excludePath)) {
155 156
        continue;
      }
157 158
      final List<String> path = _fileSystem.path.split(
        _fileSystem.path.relative(file.path, from: relativeTo));
159
      pathsToSize[path] = file.lengthSync();
160
    }
161 162 163 164
    return _buildSymbolTree(pathsToSize);
  }

  List<String> _locatedAotFilePath;
165

166 167 168 169 170 171 172 173
  List<String> _buildNodeName(_SymbolNode start, _SymbolNode parent) {
    final List<String> results = <String>[start.name];
    while (parent != null && parent.name != 'Root') {
      results.insert(0, parent.name);
      parent = parent.parent;
    }
    return results;
  }
174

175 176
  _SymbolNode _buildSymbolTree(Map<List<String>, int> pathsToSize) {
     final _SymbolNode rootNode = _SymbolNode('Root');
177
    _SymbolNode currentNode = rootNode;
178

179 180 181 182 183 184
    for (final List<String> paths in pathsToSize.keys) {
      for (final String path in paths) {
        _SymbolNode childWithPathAsName = currentNode.childByName(path);

        if (childWithPathAsName == null) {
          childWithPathAsName = _SymbolNode(path);
185
          if (matchesPattern(path, pattern: _appFilenamePattern) != null) {
186
            _appFilename = path;
187
            childWithPathAsName.name += ' (Dart AOT)';
188
            _locatedAotFilePath = _buildNodeName(childWithPathAsName, currentNode);
189
          } else if (path == 'libflutter.so') {
190 191 192 193 194 195 196 197 198 199 200 201 202 203
            childWithPathAsName.name += ' (Flutter Engine)';
          }
          currentNode.addChild(childWithPathAsName);
        }
        childWithPathAsName.addSize(pathsToSize[paths]);
        currentNode = childWithPathAsName;
      }
      currentNode = rootNode;
    }
    return rootNode;
  }

  /// Prints all children paths for the lib/ directory in an APK.
  ///
204
  /// A brief summary of aot snapshot is printed under 'lib/arm64-v8a/$_appFilename'.
205 206 207 208
  void _printLibChildrenPaths(
    _SymbolNode currentNode,
    String totalPath,
    _SymbolNode aotSnapshotJsonRoot,
209 210
    int maxDepth,
    int currentDepth,
211 212 213
  ) {
    totalPath += currentNode.name;

214 215
    assert(_appFilename != null);
    if (currentNode.children.isNotEmpty
216 217 218
      && currentNode.name != '$_appFilename (Dart AOT)'
      && currentDepth < maxDepth
      && currentNode.byteSize >= 1000) {
219
      for (final _SymbolNode child in currentNode.children) {
220
        _printLibChildrenPaths(child, '$totalPath/', aotSnapshotJsonRoot, maxDepth, currentDepth + 1);
221
      }
222 223
      _leadingPaths = totalPath.split('/')
        ..removeLast();
224
    } else {
225 226 227 228 229 230 231 232 233 234
      // Print total path and size if currentNode does not have any children and is
      // larger than 1KB
      final bool isAotSnapshotPath = _locatedAotFilePath.join('/').contains(totalPath);
      if (currentNode.byteSize >= 1000 || isAotSnapshotPath) {
        _printEntitySize(totalPath, byteSize: currentNode.byteSize, level: 1, emphasis: currentNode.children.isNotEmpty);
        if (isAotSnapshotPath) {
          _printAotSnapshotSummary(aotSnapshotJsonRoot, level: totalPath.split('/').length);
        }
        _leadingPaths = totalPath.split('/')
          ..removeLast();
235 236 237 238 239 240
      }
    }
  }

  /// Go through the AOT gen snapshot size JSON and print out a collapsed summary
  /// for the first package level.
241
  void _printAotSnapshotSummary(_SymbolNode aotSnapshotRoot, {int maxDirectoriesShown = 20, @required int level}) {
242 243 244
    _printEntitySize(
      'Dart AOT symbols accounted decompressed size',
      byteSize: aotSnapshotRoot.byteSize,
245 246
      level: level,
      emphasis: true,
247 248 249
    );

    final List<_SymbolNode> sortedSymbols = aotSnapshotRoot.children.toList()
250 251 252
      // Remove entries like  @unknown, @shared, and @stubs as well as private dart libraries
      //  which are not interpretable by end users.
      ..removeWhere((_SymbolNode node) => node.name.startsWith('@') || node.name.startsWith('dart:_'))
253 254
      ..sort((_SymbolNode a, _SymbolNode b) => b.byteSize.compareTo(a.byteSize));
    for (final _SymbolNode node in sortedSymbols.take(maxDirectoriesShown)) {
255 256 257 258 259 260 261 262 263
      // Node names will have an extra leading `package:*` name, remove it to
      // avoid extra nesting.
      _printEntitySize(_formatExtraLeadingPackages(node.name), byteSize: node.byteSize, level: level + 1);
    }
  }

  String _formatExtraLeadingPackages(String name) {
    if (!name.startsWith('package')) {
      return name;
264
    }
265 266 267 268 269 270
    final List<String> chunks = name.split('/');
    if (chunks.length < 2) {
      return name;
    }
    chunks.removeAt(0);
    return chunks.join('/');
271 272 273
  }

  /// Adds breakdown of aot snapshot data as the children of the node at the given path.
274
  Map<String, dynamic> _addAotSnapshotDataToAnalysis({
275 276 277
    @required Map<String, dynamic> apkAnalysisJson,
    @required List<String> path,
    @required Map<String, dynamic> aotSnapshotJson,
278
    @required Map<String, dynamic> precompilerTrace,
279 280
  }) {
    Map<String, dynamic> currentLevel = apkAnalysisJson;
281
    currentLevel['precompiler-trace'] = precompilerTrace;
282 283 284 285 286 287 288 289 290 291 292 293
    while (path.isNotEmpty) {
      final List<Map<String, dynamic>> children = currentLevel['children'] as List<Map<String, dynamic>>;
      final Map<String, dynamic> childWithPathAsName = children.firstWhere(
        (Map<String, dynamic> child) => child['n'] as String == path.first,
      );
      path.removeAt(0);
      currentLevel = childWithPathAsName;
    }
    currentLevel['children'] = aotSnapshotJson['children'];
    return apkAnalysisJson;
  }

294 295
  List<String> _leadingPaths = <String>[];

296 297 298 299 300 301
  /// Print an entity's name with its size on the same line.
  void _printEntitySize(
    String entityName, {
    @required int byteSize,
    @required int level,
    bool showColor = true,
302
    bool emphasis = false,
303 304 305 306 307 308 309 310 311 312
  }) {
    final String formattedSize = _prettyPrintBytes(byteSize);

    TerminalColor color = TerminalColor.green;
    if (formattedSize.endsWith('MB')) {
      color = TerminalColor.cyan;
    } else if (formattedSize.endsWith('KB')) {
      color = TerminalColor.yellow;
    }

313 314 315
    // Compute any preceding directories, and compare this to the stored
    // directories (in _leadingPaths) for the last entity that was printed. The
    // similarly determines whether or not leading directory information needs to
316 317 318 319 320 321 322 323
    // be printed.
    final List<String> localSegments = entityName.split('/')
        ..removeLast();
    int i = 0;
    while (i < _leadingPaths.length && i < localSegments.length && _leadingPaths[i] == localSegments[i]) {
      i += 1;
    }
    for (; i < localSegments.length; i += 1) {
324
      _logger.printStatus(
325 326 327 328 329 330 331
        localSegments[i] + '/',
        indent: (level + i) * 2,
        emphasis: true,
      );
    }
    _leadingPaths = localSegments;

332
    final String baseName = _fileSystem.path.basename(entityName);
333
    final int spaceInBetween = tableWidth - (level + i) * 2 - baseName.length - formattedSize.length;
334
    _logger.printStatus(
335
      baseName + ' ' * spaceInBetween,
336 337
      newline: false,
      emphasis: emphasis,
338
      indent: (level + i) * 2,
339
    );
340
    _logger.printStatus(formattedSize, color: showColor ? color : null);
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
  }

  String _prettyPrintBytes(int numBytes) {
    const int kB = 1024;
    const int mB = kB * 1024;
    if (numBytes < kB) {
      return '$numBytes B';
    } else if (numBytes < mB) {
      return '${(numBytes / kB).round()} KB';
    } else {
      return '${(numBytes / mB).round()} MB';
    }
  }

  _SymbolNode _parseAotSnapshot(Map<String, dynamic> aotSnapshotJson) {
    final bool isLeafNode = aotSnapshotJson['children'] == null;
    if (!isLeafNode) {
      return _buildNodeWithChildren(aotSnapshotJson);
    } else {
      // TODO(peterdjlee): Investigate why there are leaf nodes with size of null.
      final int byteSize = aotSnapshotJson['value'] as int;
      if (byteSize == null) {
        return null;
      }
      return _buildNode(aotSnapshotJson, byteSize);
    }
  }

  _SymbolNode _buildNode(
    Map<String, dynamic> aotSnapshotJson,
    int byteSize, {
    List<_SymbolNode> children = const <_SymbolNode>[],
  }) {
    final String name = aotSnapshotJson['n'] as String;
    final Map<String, _SymbolNode> childrenMap = <String, _SymbolNode>{};

    for (final _SymbolNode child in children) {
      childrenMap[child.name] = child;
    }

    return _SymbolNode(
      name,
      byteSize: byteSize,
    )..addAllChildren(children);
  }

  /// Builds a node by recursively building all of its children first
  /// in order to calculate the sum of its children's sizes.
  _SymbolNode _buildNodeWithChildren(Map<String, dynamic> aotSnapshotJson) {
    final List<dynamic> rawChildren = aotSnapshotJson['children'] as List<dynamic>;
    final List<_SymbolNode> symbolNodeChildren = <_SymbolNode>[];
    int totalByteSize = 0;

    // Given a child, build its subtree.
    for (final dynamic child in rawChildren) {
      final _SymbolNode childTreemapNode = _parseAotSnapshot(child as Map<String, dynamic>);
      symbolNodeChildren.add(childTreemapNode);
      totalByteSize += childTreemapNode.byteSize;
    }

    // If none of the children matched the diff tree type
    if (totalByteSize == 0) {
      return null;
    } else {
      return _buildNode(
        aotSnapshotJson,
        totalByteSize,
        children: symbolNodeChildren,
      );
    }
  }
}

/// A node class that represents a single symbol for AOT size snapshots.
class _SymbolNode {
  _SymbolNode(
    this.name, {
    this.byteSize = 0,
  })  : assert(name != null),
        assert(byteSize != null),
        _children = <String, _SymbolNode>{};

  /// The human friendly identifier for this node.
  String name;

  int byteSize;
  void addSize(int sizeToBeAdded) {
    byteSize += sizeToBeAdded;
  }

  _SymbolNode get parent => _parent;
  _SymbolNode _parent;

  Iterable<_SymbolNode> get children => _children.values;
  final Map<String, _SymbolNode> _children;

  _SymbolNode childByName(String name) => _children[name];

  _SymbolNode addChild(_SymbolNode child) {
    assert(child.parent == null);
    assert(!_children.containsKey(child.name),
        'Cannot add duplicate child key ${child.name}');

    child._parent = this;
    _children[child.name] = child;
    return child;
  }

  void addAllChildren(List<_SymbolNode> children) {
    children.forEach(addChild);
  }

  Map<String, dynamic> toJson() {
    final Map<String, dynamic> json = <String, dynamic>{
      'n': name,
      'value': byteSize
    };
    final List<Map<String, dynamic>> childrenAsJson = <Map<String, dynamic>>[];
    for (final _SymbolNode child in children) {
      childrenAsJson.add(child.toJson());
    }
    if (childrenAsJson.isNotEmpty) {
      json['children'] = childrenAsJson;
    }
    return json;
  }
}
468 469 470 471 472 473 474

/// Matches `pattern` against the entirety of `string`.
@visibleForTesting
Match matchesPattern(String string, {@required Pattern pattern}) {
  final Match match = pattern.matchAsPrefix(string);
  return (match != null && match.end == string.length) ? match : null;
}