Add a Dart script to prepare the flutter repo for packaging. (#13548)

This is the first step in a two-step process of moving the package preparation step from a recipe in chrome_infra to a dart script in the flutter repo. This will make it easier to make changes to the process. The second step is to change the infra recipe to call this script.

In addition, I added a step to the packaging process to run flutter create for each type of template so that any pub dependencies of the templates get added to the cache that gets packaged (and thus users can run flutter create --offline and have it work).

Note that the actual packaging into a "tar" or "zip" file now happens here, so a developer could actually run this script on their machine to create a package.

dev/bots/prepare_package.dart

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:args/args.dart';
+import 'package:path/path.dart' as path;
+
+const String CHROMIUM_REPO =
+    'https://chromium.googlesource.com/external/github.com/flutter/flutter';
+const String GITHUB_REPO = 'https://github.com/flutter/flutter.git';
+
+/// The type of the process runner function.  This allows us to
+/// inject a fake process runner into the ArchiveCreator for tests.
+typedef ProcessResult ProcessRunner(
+  String executable,
+  List<String> arguments, {
+  String workingDirectory,
+  Map<String, String> environment,
+  bool includeParentEnvironment,
+  bool runInShell,
+  Encoding stdoutEncoding,
+  Encoding stderrEncoding,
+});
+
+/// Error class for when a process fails to run, so we can catch
+/// it and provide something more readable than a stack trace.
+class ProcessFailedException extends Error {
+  ProcessFailedException([this.message, this.exitCode]);
+
+  String message = '';
+  int exitCode = 0;
+
+  @override
+  String toString() => message;
+}
+
+/// Creates a pre-populated Flutter archive from a git repo.
+class ArchiveCreator {
+  /// [tempDir] is the directory to use for creating the archive.  Will place
+  /// several GiB of data there, so it should have available space.
+  /// [outputFile] is the name of the output archive. It should end in either
+  /// ".tar.bz2" or ".zip".
+  /// The runner argument is used to inject a mock of [Process.runSync] for
+  /// testing purposes.
+  ArchiveCreator(this.tempDir, this.outputFile, {ProcessRunner runner})
+      : assert(outputFile.path.toLowerCase().endsWith('.zip') ||
+            outputFile.path.toLowerCase().endsWith('.tar.bz2')),
+        flutterRoot = new Directory(path.join(tempDir.path, 'flutter')),
+        _runner = runner ?? Process.runSync {
+    flutter = path.join(
+      flutterRoot.absolute.path,
+      'bin',
+      Platform.isWindows ? 'flutter.bat' : 'flutter',
+    );
+    environment = new Map<String, String>.from(Platform.environment);
+    environment['PUB_CACHE'] = path.join(flutterRoot.absolute.path, '.pub-cache');
+  }
+
+  final Directory flutterRoot;
+  final Directory tempDir;
+  final File outputFile;
+  final ProcessRunner _runner;
+  String flutter;
+  final String git = Platform.isWindows ? 'git.bat' : 'git';
+  final String zip = Platform.isWindows ? 'zip.exe' : 'zip';
+  final String tar = Platform.isWindows ? 'tar.exe' : 'tar';
+  Map<String, String> environment;
+
+  /// Clone the Flutter repo and make sure that the git environment is sane
+  /// for when the user will unpack it.
+  void checkoutFlutter(String revision) {
+    // We want the user to start out the in the 'master' branch instead of a
+    // detached head. To do that, we need to make sure master points at the
+    // desired revision.
+    runGit(<String>['clone', '-b', 'master', CHROMIUM_REPO], workingDirectory: tempDir);
+    runGit(<String>['reset', '--hard', revision]);
+
+    // Make the origin point to github instead of the chromium mirror.
+    runGit(<String>['remote', 'remove', 'origin']);
+    runGit(<String>['remote', 'add', 'origin', GITHUB_REPO]);
+  }
+
+  /// Prepare the archive repo so that it has all of the caches warmed up and
+  /// is configured for the user to being working.
+  void prepareArchive() {
+    runFlutter(<String>['doctor']);
+    runFlutter(<String>['update-packages']);
+    runFlutter(<String>['precache']);
+    runFlutter(<String>['ide-config']);
+
+    // Create each of the templates, since they will call pub get on
+    // themselves when created, and this will warm the cache with their
+    // dependencies too.
+    for (String template in <String>['app', 'package', 'plugin']) {
+      final String createName = path.join(tempDir.path, 'create_$template');
+      runFlutter(
+        <String>['create', '--template=$template', createName],
+      );
+    }
+
+    // Yes, we could just skip all .packages files when constructing
+    // the archive, but some are checked in, and we don't want to skip
+    // those.
+    runGit(<String>['clean', '-f', '-X', '**/.packages']);
+  }
+
+  /// Create the archive into the given output file.
+  void createArchive() {
+    if (outputFile.path.toLowerCase().endsWith('.zip')) {
+      createZipArchive(outputFile, flutterRoot);
+    } else if (outputFile.path.toLowerCase().endsWith('.tar.bz2')) {
+      createTarArchive(outputFile, flutterRoot);
+    }
+  }
+
+  String _runProcess(String executable, List<String> args, {Directory workingDirectory}) {
+    workingDirectory ??= flutterRoot;
+    stderr.write('Running "$executable ${args.join(' ')}" in ${workingDirectory.path}.\n');
+    ProcessResult result;
+    try {
+      result = _runner(
+        executable,
+        args,
+        workingDirectory: workingDirectory.absolute.path,
+        environment: environment,
+        includeParentEnvironment: false,
+      );
+    } on ProcessException catch (e) {
+      final String message = 'Running "$executable ${args.join(' ')}" in ${workingDirectory.path} '
+          'failed with:\n${e.toString()}\n  PATH: ${environment['PATH']}';
+      throw new ProcessFailedException(message, -1);
+    } catch (e) {
+      rethrow;
+    }
+    stdout.write(result.stdout);
+    stderr.write(result.stderr);
+    if (result.exitCode != 0) {
+      final String message = 'Running "$executable ${args.join(' ')}" in ${workingDirectory.path} '
+          'failed with ${result.exitCode}.';
+      throw new ProcessFailedException(message, result.exitCode);
+    }
+    return result.stdout.trim();
+  }
+
+  String runFlutter(List<String> args) {
+    return _runProcess(flutter, args);
+  }
+
+  String runGit(List<String> args, {Directory workingDirectory}) {
+    return _runProcess(git, args, workingDirectory: workingDirectory);
+  }
+
+  String createZipArchive(File output, Directory source) {
+    final List<String> args = <String>[
+      '-r',
+      '-9',
+      '-q',
+      output.absolute.path,
+      path.basename(source.absolute.path),
+    ];
+
+    return _runProcess(zip, args,
+        workingDirectory: new Directory(path.dirname(source.absolute.path)));
+  }
+
+  String createTarArchive(File output, Directory source) {
+    final List<String> args = <String>[
+      'cjf',
+      output.absolute.path,
+      path.basename(source.absolute.path),
+    ];
+    return _runProcess(tar, args,
+        workingDirectory: new Directory(path.dirname(source.absolute.path)));
+  }
+}
+
+/// Prepares a flutter git repo to be packaged up for distribution.
+/// It mainly serves to populate the .pub-cache with any appropriate Dart
+/// packages, and the flutter cache in bin/cache with the appropriate
+/// dependencies and snapshots.
+void main(List<String> argList) {
+  final ArgParser argParser = new ArgParser();
+  argParser.addOption(
+    'temp_dir',
+    defaultsTo: null,
+    help: 'A location where temporary files may be written. Defaults to a '
+        'directory in the system temp folder. Will write a few GiB of data, '
+        'so it should have sufficient free space.',
+  );
+  argParser.addOption(
+    'revision',
+    defaultsTo: 'master',
+    help: 'The Flutter revision to build the archive with. Defaults to the '
+        "master branch's HEAD revision.",
+  );
+  argParser.addOption(
+    'output',
+    defaultsTo: null,
+    help: 'The path where the output archive should be written. '
+        'The suffix determines the output format: .tar.bz2 or .zip are the '
+        'only formats supported.',
+  );
+  final ArgResults args = argParser.parse(argList);
+
+  void errorExit(String message, {int exitCode = -1}) {
+    stderr.write('Error: $message\n\n');
+    stderr.write('${argParser.usage}\n');
+    exit(exitCode);
+  }
+
+  if (args['revision'].isEmpty) {
+    errorExit('Invalid argument: --revision must be specified.');
+  }
+
+  Directory tmpDir;
+  bool removeTempDir = false;
+  if (args['temp_dir'] == null || args['temp_dir'].isEmpty) {
+    tmpDir = Directory.systemTemp.createTempSync('flutter_');
+    removeTempDir = true;
+  } else {
+    tmpDir = new Directory(args['temp_dir']);
+    if (!tmpDir.existsSync()) {
+      errorExit("Temporary directory ${args['temp_dir']} doesn't exist.");
+    }
+  }
+
+  String outputFileString = args['output'];
+  if (outputFileString == null || outputFileString.isEmpty) {
+    final String suffix = Platform.isWindows ? '.zip' : '.tar.bz2';
+    outputFileString = path.join(tmpDir.path, 'flutter_${args['revision']}$suffix');
+  } else if (!outputFileString.toLowerCase().endsWith('.zip') &&
+      !outputFileString.toLowerCase().endsWith('.tar.bz2')) {
+    errorExit('Output file has unsupported suffix. It should be either ".zip" or ".tar.bz2".');
+  }
+
+  final File outputFile = new File(outputFileString);
+  if (outputFile.existsSync()) {
+    errorExit('Output file ${outputFile.absolute.path} already exists.');
+  }
+
+  final ArchiveCreator preparer = new ArchiveCreator(tmpDir, outputFile);
+  int exitCode = 0;
+  String message;
+  try {
+    preparer.checkoutFlutter(args['revision']);
+    preparer.prepareArchive();
+    preparer.createArchive();
+  } on ProcessFailedException catch (e) {
+    exitCode = e.exitCode;
+    message = e.message;
+  } catch (e) {
+    rethrow;
+  } finally {
+    if (removeTempDir) {
+      tmpDir.deleteSync(recursive: true);
+    }
+    if (exitCode != 0) {
+      errorExit(message, exitCode: exitCode);
+    }
+    exit(0);
+  }
+}

dev/bots/pubspec.yaml

 name: tests_on_bots
-description: Script to run all tests on bots.
+description: Scripts which run on bots.
 
 dependencies:
   path: 1.5.1
+  args: 0.13.7
+
+dev_dependencies:
+  test: 0.12.26
+  mockito: 2.2.1
+
+  async: 1.13.3 # TRANSITIVE DEPENDENCY
+  barback: 0.15.2+13 # TRANSITIVE DEPENDENCY
+  boolean_selector: 1.0.2 # TRANSITIVE DEPENDENCY
+  charcode: 1.1.1 # TRANSITIVE DEPENDENCY
+  collection: 1.14.3 # TRANSITIVE DEPENDENCY
+  convert: 2.0.1 # TRANSITIVE DEPENDENCY
+  crypto: 2.0.2+1 # TRANSITIVE DEPENDENCY
+  glob: 1.1.5 # TRANSITIVE DEPENDENCY
+  http: 0.11.3+14 # TRANSITIVE DEPENDENCY
+  http_multi_server: 2.0.4 # TRANSITIVE DEPENDENCY
+  http_parser: 3.1.1 # TRANSITIVE DEPENDENCY
+  io: 0.3.1 # TRANSITIVE DEPENDENCY
+  js: 0.6.1 # TRANSITIVE DEPENDENCY
+  matcher: 0.12.1+4 # TRANSITIVE DEPENDENCY
+  meta: 1.1.1 # TRANSITIVE DEPENDENCY
+  mime: 0.9.5 # TRANSITIVE DEPENDENCY
+  node_preamble: 1.4.0 # TRANSITIVE DEPENDENCY
+  package_config: 1.0.3 # TRANSITIVE DEPENDENCY
+  package_resolver: 1.0.2 # TRANSITIVE DEPENDENCY
+  pool: 1.3.3 # TRANSITIVE DEPENDENCY
+  pub_semver: 1.3.2 # TRANSITIVE DEPENDENCY
+  shelf: 0.7.1 # TRANSITIVE DEPENDENCY
+  shelf_packages_handler: 1.0.3 # TRANSITIVE DEPENDENCY
+  shelf_static: 0.2.6 # TRANSITIVE DEPENDENCY
+  shelf_web_socket: 0.2.2 # TRANSITIVE DEPENDENCY
+  source_map_stack_trace: 1.1.4 # TRANSITIVE DEPENDENCY
+  source_maps: 0.10.4 # TRANSITIVE DEPENDENCY
+  source_span: 1.4.0 # TRANSITIVE DEPENDENCY
+  stack_trace: 1.9.1 # TRANSITIVE DEPENDENCY
+  stream_channel: 1.6.2 # TRANSITIVE DEPENDENCY
+  string_scanner: 1.0.2 # TRANSITIVE DEPENDENCY
+  term_glyph: 1.0.0 # TRANSITIVE DEPENDENCY
+  typed_data: 1.1.4 # TRANSITIVE DEPENDENCY
+  web_socket_channel: 1.0.6 # TRANSITIVE DEPENDENCY
+  yaml: 2.1.13 # TRANSITIVE DEPENDENCY

dev/bots/test.dart

@@ -162,6 +162,7 @@ Future<Null> _runTests() async {
   await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'));
   await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'));
   await _pubRunTest(path.join(flutterRoot, 'packages', 'flutter_tools'));
+  await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'));
 
   await _runAllDartTests(path.join(flutterRoot, 'dev', 'devicelab'));
   await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));

dev/bots/test/prepare_package_test.dart

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:mockito/mockito.dart';
+import 'package:test/test.dart';
+import 'package:path/path.dart' as path;
+
+import '../prepare_package.dart';
+
+void main() {
+  group('ArchiveCreator', () {
+    ArchiveCreator preparer;
+    Directory tmpDir;
+    Directory flutterDir;
+    File outputFile;
+    MockProcessRunner runner;
+    List<MockProcessResult> results;
+    final List<List<String>> args = <List<String>>[];
+    final List<Map<Symbol, dynamic>> namedArgs = <Map<Symbol, dynamic>>[];
+    final String zipExe = Platform.isWindows ? 'zip.exe' : 'zip';
+    final String tarExe = Platform.isWindows ? 'tar.exe' : 'tar';
+    final String gitExe = Platform.isWindows ? 'git.bat' : 'git';
+    String flutterExe;
+
+    void _verifyCommand(List<dynamic> args, String expected) {
+      final List<String> expectedList = expected.split(' ');
+      final String executable = expectedList.removeAt(0);
+      expect(args[0], executable);
+      expect(args[1], orderedEquals(expectedList));
+    }
+
+    ProcessResult _nextResult(Invocation invocation) {
+      args.add(invocation.positionalArguments);
+      namedArgs.add(invocation.namedArguments);
+      return results.isEmpty ? new MockProcessResult('', '', 0) : results.removeAt(0);
+    }
+
+    void _answerWithResults() {
+      when(
+        runner.call(
+          typed(captureAny),
+          typed(captureAny),
+          environment: typed(captureAny, named: 'environment'),
+          workingDirectory: typed(captureAny, named: 'workingDirectory'),
+          includeParentEnvironment: typed(captureAny, named: 'includeParentEnvironment'),
+        ),
+      ).thenAnswer(_nextResult);
+    }
+
+    setUp(() async {
+      runner = new MockProcessRunner();
+      args.clear();
+      namedArgs.clear();
+      tmpDir = await Directory.systemTemp.createTemp('flutter_');
+      flutterDir = new Directory(path.join(tmpDir.path, 'flutter'));
+      flutterExe =
+          path.join(flutterDir.path, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter');
+    });
+
+    tearDown(() async {
+      await tmpDir.delete(recursive: true);
+    });
+
+    test('sets PUB_CACHE properly', () async {
+      outputFile = new File(path.join(tmpDir.absolute.path, 'flutter_master.tar.bz2'));
+      preparer = new ArchiveCreator(tmpDir, outputFile, runner: runner);
+      _answerWithResults();
+      results = <MockProcessResult>[new MockProcessResult('deadbeef\n', '', 0)];
+      preparer.checkoutFlutter('master');
+      preparer.prepareArchive();
+      preparer.createArchive();
+      expect(
+        verify(runner.call(
+          captureAny,
+          captureAny,
+          workingDirectory: captureAny,
+          environment: captureAny,
+          includeParentEnvironment: typed(captureAny, named: 'includeParentEnvironment'),
+        )).captured[2]['PUB_CACHE'],
+        endsWith(path.join('flutter', '.pub-cache')),
+      );
+    });
+
+    test('calls the right commands for tar output', () async {
+      outputFile = new File(path.join(tmpDir.absolute.path, 'flutter_master.tar.bz2'));
+      preparer = new ArchiveCreator(tmpDir, outputFile, runner: runner);
+      _answerWithResults();
+      results = <MockProcessResult>[new MockProcessResult('deadbeef\n', '', 0)];
+      preparer.checkoutFlutter('master');
+      preparer.prepareArchive();
+      preparer.createArchive();
+      final List<String> commands = <String>[
+        '$gitExe clone -b master https://chromium.googlesource.com/external/github.com/flutter/flutter',
+        '$gitExe reset --hard master',
+        '$gitExe remote remove origin',
+        '$gitExe remote add origin https://github.com/flutter/flutter.git',
+        '$flutterExe doctor',
+        '$flutterExe update-packages',
+        '$flutterExe precache',
+        '$flutterExe ide-config',
+        '$flutterExe create --template=app ${path.join(tmpDir.path, 'create_app')}',
+        '$flutterExe create --template=package ${path.join(tmpDir.path, 'create_package')}',
+        '$flutterExe create --template=plugin ${path.join(tmpDir.path, 'create_plugin')}',
+        '$gitExe clean -f -X **/.packages',
+        '$tarExe cjf ${path.join(tmpDir.path, 'flutter_master.tar.bz2')} flutter',
+      ];
+      int step = 0;
+      for (String command in commands) {
+        _verifyCommand(args[step++], command);
+      }
+    });
+
+    test('calls the right commands for zip output', () async {
+      outputFile = new File(path.join(tmpDir.absolute.path, 'flutter_master.zip'));
+      preparer = new ArchiveCreator(tmpDir, outputFile, runner: runner);
+      _answerWithResults();
+      results = <MockProcessResult>[new MockProcessResult('deadbeef\n', '', 0)];
+      preparer.checkoutFlutter('master');
+      preparer.prepareArchive();
+      preparer.createArchive();
+      final List<String> commands = <String>[
+        '$gitExe clone -b master https://chromium.googlesource.com/external/github.com/flutter/flutter',
+        '$gitExe reset --hard master',
+        '$gitExe remote remove origin',
+        '$gitExe remote add origin https://github.com/flutter/flutter.git',
+        '$flutterExe doctor',
+        '$flutterExe update-packages',
+        '$flutterExe precache',
+        '$flutterExe ide-config',
+        '$flutterExe create --template=app ${path.join(tmpDir.path, 'create_app')}',
+        '$flutterExe create --template=package ${path.join(tmpDir.path, 'create_package')}',
+        '$flutterExe create --template=plugin ${path.join(tmpDir.path, 'create_plugin')}',
+        '$gitExe clean -f -X **/.packages',
+        '$zipExe -r -9 -q ${path.join(tmpDir.path, 'flutter_master.zip')} flutter',
+      ];
+      int step = 0;
+      for (String command in commands) {
+        _verifyCommand(args[step++], command);
+      }
+    });
+
+    test('throws when a command errors out', () async {
+      outputFile = new File(path.join(tmpDir.absolute.path, 'flutter.tar.bz2'));
+      preparer = new ArchiveCreator(
+        tmpDir,
+        outputFile,
+        runner: runner,
+      );
+
+      results = <MockProcessResult>[
+        new MockProcessResult('', '', 0),
+        new MockProcessResult('OMG! OMG! an ERROR!\n', '', -1)
+      ];
+      _answerWithResults();
+      expect(() => preparer.checkoutFlutter('master'),
+          throwsA(const isInstanceOf<ProcessFailedException>()));
+      expect(args.length, 2);
+      _verifyCommand(args[0],
+          '$gitExe clone -b master https://chromium.googlesource.com/external/github.com/flutter/flutter');
+      _verifyCommand(args[1], '$gitExe reset --hard master');
+    });
+  });
+}
+
+class MockProcessRunner extends Mock implements Function {
+  ProcessResult call(
+    String executable,
+    List<String> arguments, {
+    String workingDirectory,
+    Map<String, String> environment,
+    bool includeParentEnvironment,
+    bool runInShell,
+    Encoding stdoutEncoding,
+    Encoding stderrEncoding,
+  });
+}
+
+class MockProcessResult extends Mock implements ProcessResult {
+  MockProcessResult(this.stdout, [this.stderr = '', this.exitCode = 0]);
+
+  @override
+  dynamic stdout = '';
+
+  @override
+  dynamic stderr;
+
+  @override
+  int exitCode;
+}