Mega gallery (#3647)

* working on a large benchmark app

* build a 100k saple app

* print sizes of other libs; scale to 60k lines

* test the generated app on travis

dev/benchmarks/.gitignore

+mega_gallery/

dev/dartdoc.dart

@@ -4,121 +4,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:async';
-import 'dart:convert';
-import 'dart:io';
+import 'tools/dartdoc.dart' as dartdoc;
 
-/// This script expects to run with the cwd as the root of the flutter repo. It
-/// will generate documentation for the packages in `packages/`, and leave the
-/// documentation in `dev/docs/doc/api/`.
-main(List<String> args) async {
-  // Create the pubspec.yaml file.
-  StringBuffer buf = new StringBuffer('''
-name: Flutter
-dependencies:
-''');
-  for (String package in _findPackageNames()) {
-    buf.writeln('  $package:');
-    buf.writeln('    path: ../../packages/$package');
-  }
-  new File('dev/docs/pubspec.yaml').writeAsStringSync(buf.toString());
-
-  // Create the library file.
-  Directory libDir = new Directory('dev/docs/lib');
-  libDir.createSync();
-
-  StringBuffer contents = new StringBuffer('library temp_doc;\n\n');
-  for (String libraryRef in _libraryRefs()) {
-    contents.writeln('import \'package:$libraryRef\';');
-  }
-  new File('dev/docs/lib/temp_doc.dart').writeAsStringSync(contents.toString());
-
-  // Run pub.
-  Process process = await Process.start('pub', <String>['get'], workingDirectory: 'dev/docs');
-  _print(process.stdout);
-  _print(process.stderr);
-  int code = await process.exitCode;
-  if (code != 0)
-    exit(code);
-
-  // Generate the documentation; we require dartdoc >= 0.9.4.
-  List<String> args = <String>[
-    'global', 'run', 'dartdoc',
-    '--header', 'styles.html',
-    '--header', 'analytics.html',
-    '--dart-sdk', '../../bin/cache/dart-sdk',
-    '--exclude', 'temp_doc',
-    '--favicon=favicon.ico',
-    '--use-categories'
-  ];
-  for (String libraryRef in _libraryRefs()) {
-    String name = _entityName(libraryRef);
-
-    args.add('--include-external');
-    args.add(name.substring(0, name.length - 5));
-  }
-
-  _findSkyServicesLibraryNames().forEach((String libName) {
-    args.add('--include-external');
-    args.add(libName);
-  });
-
-  process = await Process.start('pub', args, workingDirectory: 'dev/docs');
-  _print(process.stdout);
-  _print(process.stderr);
-  exit(await process.exitCode);
-}
-
-List<String> _findSkyServicesLibraryNames() {
-  Directory skyServicesLocation = new Directory('bin/cache/pkg/sky_services/lib');
-  if (!skyServicesLocation.existsSync()) {
-    throw 'Did not find sky_services package location in '
-          '${skyServicesLocation.path}.';
-  }
-  return skyServicesLocation.listSync(followLinks: false, recursive: true)
-      .where((FileSystemEntity entity) {
-    return entity is File && entity.path.endsWith('.mojom.dart');
-  }).map((FileSystemEntity entity) {
-    String basename = _entityName(entity.path);
-    basename = basename.substring(0, basename.length-('.dart'.length));
-    return basename.replaceAll('.', '_');
-  });
-}
-
-List<String> _findPackageNames() {
-  return _findPackages().map((Directory dir) => _entityName(dir.path)).toList();
-}
-
-List<Directory> _findPackages() {
-  return new Directory('packages')
-    .listSync()
-    .where((FileSystemEntity entity) => entity is Directory)
-    .where((Directory dir) {
-      File pubspec = new File('${dir.path}/pubspec.yaml');
-      bool nodoc = pubspec.readAsStringSync().contains('nodoc: true');
-      return !nodoc;
-    })
-    .toList();
-}
-
-List<String> _libraryRefs() sync* {
-  for (Directory dir in _findPackages()) {
-    String dirName = _entityName(dir.path);
-
-    for (FileSystemEntity file in new Directory('${dir.path}/lib').listSync()) {
-      if (file is File && file.path.endsWith('.dart'))
-        yield '$dirName/${_entityName(file.path)}';
-    }
-  }
-}
-
-String _entityName(String path) {
-  return path.indexOf('/') == -1 ? path : path.substring(path.lastIndexOf('/') + 1);
-}
-
-void _print(Stream<List<int>> stream) {
-  stream
-    .transform(UTF8.decoder)
-    .transform(const LineSplitter())
-    .listen(print);
-}
+// TODO (devoncarew): Remove this once the chromium bot is updated.
+main(List<String> args) => dartdoc.main(args);

dev/tools/.gitignore

+.packages
+pubspec.lock
+packages

dev/tools/dartdoc.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:path/path.dart' as path;
+
+/// This script expects to run with the cwd as the root of the flutter repo. It
+/// will generate documentation for the packages in `//packages/` and write the
+/// documentation to `//dev/docs/doc/api/`.
+main(List<String> args) async {
+  // If we're run from the `tools` dir, set the cwd to the repo root.
+  if (path.basename(Directory.current.path) == 'tools')
+    Directory.current = Directory.current.parent.parent;
+
+  // Create the pubspec.yaml file.
+  StringBuffer buf = new StringBuffer('''
+name: Flutter
+dependencies:
+''');
+  for (String package in _findPackageNames()) {
+    buf.writeln('  $package:');
+    buf.writeln('    path: ../../packages/$package');
+  }
+  new File('dev/docs/pubspec.yaml').writeAsStringSync(buf.toString());
+
+  // Create the library file.
+  Directory libDir = new Directory('dev/docs/lib');
+  libDir.createSync();
+
+  StringBuffer contents = new StringBuffer('library temp_doc;\n\n');
+  for (String libraryRef in _libraryRefs()) {
+    contents.writeln('import \'package:$libraryRef\';');
+  }
+  new File('dev/docs/lib/temp_doc.dart').writeAsStringSync(contents.toString());
+
+  // Run pub.
+  Process process = await Process.start('pub', <String>['get'], workingDirectory: 'dev/docs');
+  _print(process.stdout);
+  _print(process.stderr);
+  int code = await process.exitCode;
+  if (code != 0)
+    exit(code);
+
+  // Generate the documentation; we require dartdoc >= 0.9.4.
+  List<String> args = <String>[
+    'global', 'run', 'dartdoc',
+    '--header', 'styles.html',
+    '--header', 'analytics.html',
+    '--dart-sdk', '../../bin/cache/dart-sdk',
+    '--exclude', 'temp_doc',
+    '--favicon=favicon.ico',
+    '--use-categories'
+  ];
+
+  for (String libraryRef in _libraryRefs()) {
+    String name = path.basename(libraryRef);
+    args.add('--include-external');
+    args.add(name.substring(0, name.length - 5));
+  }
+
+  _findSkyServicesLibraryNames().forEach((String libName) {
+    args.add('--include-external');
+    args.add(libName);
+  });
+
+  process = await Process.start('pub', args, workingDirectory: 'dev/docs');
+  _print(process.stdout);
+  _print(process.stderr);
+  exit(await process.exitCode);
+}
+
+List<String> _findSkyServicesLibraryNames() {
+  Directory skyServicesLocation = new Directory('bin/cache/pkg/sky_services/lib');
+  if (!skyServicesLocation.existsSync()) {
+    throw 'Did not find sky_services package location in ${skyServicesLocation.path}.';
+  }
+  return skyServicesLocation.listSync(followLinks: false, recursive: true)
+      .where((FileSystemEntity entity) {
+    return entity is File && entity.path.endsWith('.mojom.dart');
+  }).map((FileSystemEntity entity) {
+    String basename = path.basename(entity.path);
+    basename = basename.substring(0, basename.length-('.dart'.length));
+    return basename.replaceAll('.', '_');
+  });
+}
+
+List<String> _findPackageNames() {
+  return _findPackages().map((Directory dir) => path.basename(dir.path)).toList();
+}
+
+List<Directory> _findPackages() {
+  return new Directory('packages')
+    .listSync()
+    .where((FileSystemEntity entity) => entity is Directory)
+    .where((Directory dir) {
+      File pubspec = new File('${dir.path}/pubspec.yaml');
+      bool nodoc = pubspec.readAsStringSync().contains('nodoc: true');
+      return !nodoc;
+    })
+    .toList();
+}
+
+List<String> _libraryRefs() sync* {
+  for (Directory dir in _findPackages()) {
+    String dirName = path.basename(dir.path);
+
+    for (FileSystemEntity file in new Directory('${dir.path}/lib').listSync()) {
+      if (file is File && file.path.endsWith('.dart'))
+        yield '$dirName/${path.basename(file.path)}';
+    }
+  }
+}
+
+void _print(Stream<List<int>> stream) {
+  stream
+    .transform(UTF8.decoder)
+    .transform(const LineSplitter())
+    .listen(print);
+}

dev/tools/mega_gallery.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/// Make `n` copies of material_gallery.
+
+import 'dart:io';
+
+import 'package:args/args.dart';
+import 'package:path/path.dart' as path;
+
+/// If no `copies` param is passed in, we scale the generated app up to 60k lines.
+const int kTargetLineCount = 60 * 1024;
+
+void main(List<String> args) {
+  // If we're run from the `tools` dir, set the cwd to the repo root.
+  if (path.basename(Directory.current.path) == 'tools')
+    Directory.current = Directory.current.parent.parent;
+
+  ArgParser argParser = new ArgParser();
+  // ../mega_gallery? dev/benchmarks/mega_gallery?
+  argParser.addOption('out', defaultsTo: _normalize('dev/benchmarks/mega_gallery'));
+  argParser.addOption('copies');
+  argParser.addFlag('delete', negatable: false);
+  argParser.addFlag('help', abbr: 'h', negatable: false);
+
+  ArgResults results = argParser.parse(args);
+
+  if (results['help']) {
+    print('Generate n copies of material_gallery.\n');
+    print('usage: dart mega_gallery.dart <options>');
+    print(argParser.usage);
+    exit(0);
+  }
+
+  Directory source = new Directory(_normalize('examples/material_gallery'));
+  Directory out = new Directory(_normalize(results['out']));
+
+  if (results['delete']) {
+    if (out.existsSync()) {
+      print('Deleting ${out.path}');
+      out.deleteSync(recursive: true);
+    }
+
+    exit(0);
+  }
+
+  int copies;
+  if (!results.wasParsed('copies')) {
+    SourceStats stats = getStatsFor(_dir(source, 'lib'));
+    copies = (kTargetLineCount / stats.lines).round();
+  } else {
+    copies = int.parse(results['copies']);
+  }
+
+  print('Stats:');
+  print('  packages/flutter           : ${getStatsFor(new Directory("packages/flutter"))}');
+  print('  examples/material_gallery  : ${getStatsFor(new Directory("examples/material_gallery"))}');
+  print('');
+
+  print('Making $copies copies of material_gallery:');
+
+  Directory lib = _dir(out, 'lib');
+  if (lib.existsSync())
+    lib.deleteSync(recursive: true);
+
+  // Copy everything that's not a symlink, dot directory, or build/.
+  _copy(source, out);
+
+  // Make n - 1 copies.
+  for (int i = 1; i < copies; i++)
+    _copyGallery(out, i);
+
+  // Create a new entry-point.
+  _createEntry(_file(out, 'lib/main.dart'), copies);
+
+  // Update the pubspec.
+  String pubspec = _file(out, 'pubspec.yaml').readAsStringSync();
+  pubspec = pubspec.replaceAll('../../packages/flutter', '../../../packages/flutter');
+  _file(out, 'pubspec.yaml').writeAsStringSync(pubspec);
+
+  _file(out, '.dartignore').writeAsStringSync('');
+
+  // Count source lines and number of files; tell how to run it.
+  print('  ${path.relative(results["out"])}: ${getStatsFor(out)}');
+}
+
+// TODO(devoncarew): Create an entry-point that builds a UI with all `n` copies.
+void _createEntry(File mainFile, int copies) {
+  StringBuffer imports = new StringBuffer();
+  StringBuffer importRefs = new StringBuffer();
+
+  for (int i = 1; i < copies; i++) {
+    imports.writeln("import 'gallery_$i/main.dart' as main_$i;");
+    importRefs.writeln("  main_$i.main;");
+  }
+
+  String contents = '''
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter/widgets.dart';
+
+import 'gallery/app.dart';
+${imports.toString().trim()}
+
+void main() {
+  // Make sure the imports are not marked as unused.
+  ${importRefs.toString().trim()}
+
+  runApp(new GalleryApp());
+}
+''';
+
+  mainFile.writeAsStringSync(contents);
+}
+
+void _copyGallery(Directory galleryDir, int index) {
+  Directory lib = _dir(galleryDir, 'lib');
+  Directory dest = _dir(lib, 'gallery_$index');
+  dest.createSync();
+
+  // Copy demo/, gallery/, and main.dart.
+  _copy(_dir(lib, 'demo'), _dir(dest, 'demo'));
+  _copy(_dir(lib, 'gallery'), _dir(dest, 'gallery'));
+  _file(dest, 'main.dart').writeAsBytesSync(_file(lib, 'main.dart').readAsBytesSync());
+}
+
+void _copy(Directory source, Directory target) {
+  if (!target.existsSync())
+    target.createSync();
+
+  for (FileSystemEntity entity in source.listSync(followLinks: false)) {
+    String name = path.basename(entity.path);
+
+    if (entity is Directory) {
+      if (name == 'build' || name.startsWith('.'))
+        continue;
+      _copy(entity, new Directory(path.join(target.path, name)));
+    } else if (entity is File) {
+      if (name == '.packages' || name == 'pubspec.lock')
+        continue;
+      File dest = new File(path.join(target.path, name));
+      dest.writeAsBytesSync(entity.readAsBytesSync());
+    }
+  }
+}
+
+Directory _dir(Directory parent, String name) => new Directory(path.join(parent.path, name));
+File _file(Directory parent, String name) => new File(path.join(parent.path, name));
+String _normalize(String filePath) => path.normalize(path.absolute(filePath));
+
+class SourceStats {
+  int files = 0;
+  int lines = 0;
+
+  String toString() => '${_comma(files)} files, ${_comma(lines)} lines';
+}
+
+SourceStats getStatsFor(Directory dir, [SourceStats stats]) {
+  stats ??= new SourceStats();
+
+  for (FileSystemEntity entity in dir.listSync(recursive: false, followLinks: false)) {
+    String name = path.basename(entity.path);
+    if (entity is File && name.endsWith('.dart')) {
+      stats.files += 1;
+      stats.lines += _lineCount(entity);
+    } else if (entity is Directory && !name.startsWith('.')) {
+      getStatsFor(entity, stats);
+    }
+  }
+
+  return stats;
+}
+
+int _lineCount(File file) {
+  return file.readAsLinesSync().where((String line) {
+    line = line.trim();
+    if (line.isEmpty || line.startsWith('//'))
+      return false;
+    return true;
+  }).length;
+}
+
+String _comma(int count) {
+  String str = count.toString();
+  if (str.length > 3)
+    return str.substring(0, str.length - 3) + ',' + str.substring(str.length - 3);
+  return str;
+}

dev/tools/pubspec.yaml

+name: dev_tools
+description: Various repository development tools for flutter.
+
+dependencies:
+  args: ^0.13.4
+  path: ^1.3.0

travis/test.sh

@@ -6,10 +6,15 @@ export PATH="$PWD/bin:$PWD/bin/cache/dart-sdk/bin:$PATH"
 # analyze all the Dart code in the repo
 flutter analyze --flutter-repo
 
+# generate and analyze our large sample app
+dart dev/tools/mega_gallery.dart
+(cd dev/benchmarks/mega_gallery; flutter analyze --watch --benchmark)
+
 # keep the rest of this file in sync with
 # //chrome_infra/build/scripts/slave/recipes/flutter/flutter.py
 # see https://github.com/flutter/flutter/blob/master/infra/README.md
 
+# run tests
 (cd packages/flutter; flutter test)
 (cd packages/flutter_driver; dart -c test/all.dart)
 (cd packages/flutter_sprites; flutter test)