Unverified Commit 8a2df396 authored by Greg Spencer's avatar Greg Spencer Committed by GitHub

Create packages only for release builds, and publish when created. (#14476)

This changes the publishing of archives so that it happens on the chrome_infra bots when they build a packaged branch instead of as part of the dev_roll process.

It uses the tagged version in the branch, and leaves the git repo that it clones checked out on the branch and hash used to build the package.

It updates metadata located at gs://flutter_infra/releases/releases_.json (where is one of macos, linux, or windows) once published, since it would be complex to do the proper locking to keep them all in one shared .json file safely.

A separate [change to the chrome_infra bots](https://chromium-review.googlesource.com/c/chromium/tools/build/+/902823) was made to instruct them to build packaged for the dev, beta, and release branches (but not master anymore).
parent 4b878dc6
...@@ -4,13 +4,14 @@ ...@@ -4,13 +4,14 @@
import 'dart:async'; import 'dart:async';
import 'dart:convert'; import 'dart:convert';
import 'dart:io'; import 'dart:io' hide Platform;
import 'dart:typed_data'; import 'dart:typed_data';
import 'package:args/args.dart'; import 'package:args/args.dart';
import 'package:http/http.dart' as http; import 'package:http/http.dart' as http;
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
import 'package:process/process.dart'; import 'package:process/process.dart';
import 'package:platform/platform.dart' show Platform, LocalPlatform;
const String CHROMIUM_REPO = const String CHROMIUM_REPO =
'https://chromium.googlesource.com/external/github.com/flutter/flutter'; 'https://chromium.googlesource.com/external/github.com/flutter/flutter';
...@@ -18,21 +19,157 @@ const String GITHUB_REPO = 'https://github.com/flutter/flutter.git'; ...@@ -18,21 +19,157 @@ const String GITHUB_REPO = 'https://github.com/flutter/flutter.git';
const String MINGIT_FOR_WINDOWS_URL = 'https://storage.googleapis.com/flutter_infra/mingit/' const String MINGIT_FOR_WINDOWS_URL = 'https://storage.googleapis.com/flutter_infra/mingit/'
'603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip'; '603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip';
/// Error class for when a process fails to run, so we can catch /// Exception class for when a process fails to run, so we can catch
/// it and provide something more readable than a stack trace. /// it and provide something more readable than a stack trace.
class ProcessFailedException extends Error { class ProcessRunnerException implements Exception {
ProcessFailedException([this.message, this.exitCode]); ProcessRunnerException(this.message, [this.result]);
String message = ''; final String message;
int exitCode = 0; final ProcessResult result;
int get exitCode => result.exitCode ?? -1;
@override @override
String toString() => message; String toString() {
String output = runtimeType.toString();
if (message != null) {
output += ': $message';
}
final String stderr = result?.stderr ?? '';
if (stderr.isNotEmpty) {
output += ':\n${result.stderr}';
}
return output;
}
}
enum Branch { dev, beta, release }
String getBranchName(Branch branch) {
switch (branch) {
case Branch.beta:
return 'beta';
case Branch.dev:
return 'dev';
case Branch.release:
return 'release';
}
return null;
}
Branch fromBranchName(String name) {
switch (name) {
case 'beta':
return Branch.beta;
case 'dev':
return Branch.dev;
case 'release':
return Branch.release;
default:
throw new ArgumentError('Invalid branch name.');
}
}
/// A helper class for classes that want to run a process, optionally have the
/// stderr and stdout reported as the process runs, and capture the stdout
/// properly without dropping any.
class ProcessRunner {
ProcessRunner({
this.processManager: const LocalProcessManager(),
this.subprocessOutput: true,
this.defaultWorkingDirectory,
this.platform: const LocalPlatform(),
}) {
environment = new Map<String, String>.from(platform.environment);
}
/// The platform to use for a starting environment.
final Platform platform;
/// Set [subprocessOutput] to show output as processes run. Stdout from the
/// process will be printed to stdout, and stderr printed to stderr.
final bool subprocessOutput;
/// Set the [processManager] in order to inject a test instance to perform
/// testing.
final ProcessManager processManager;
/// Sets the default directory used when `workingDirectory` is not specified
/// to [runProcess].
final Directory defaultWorkingDirectory;
/// The environment to run processes with.
Map<String, String> environment;
/// Run the command and arguments in `commandLine` as a sub-process from
/// `workingDirectory` if set, or the [defaultWorkingDirectory] if not. Uses
/// [Directory.current] if [defaultWorkingDirectory] is not set.
///
/// Set `failOk` if [runProcess] should not throw an exception when the
/// command completes with a a non-zero exit code.
Future<String> runProcess(
List<String> commandLine, {
Directory workingDirectory,
bool failOk: false,
}) async {
workingDirectory ??= defaultWorkingDirectory ?? Directory.current;
if (subprocessOutput) {
stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n');
}
final List<int> output = <int>[];
final Completer<Null> stdoutComplete = new Completer<Null>();
final Completer<Null> stderrComplete = new Completer<Null>();
Process process;
Future<int> allComplete() async {
await stderrComplete.future;
await stdoutComplete.future;
return process.exitCode;
}
try {
process = await processManager.start(
commandLine,
workingDirectory: workingDirectory.absolute.path,
environment: environment,
);
process.stdout.listen(
(List<int> event) {
output.addAll(event);
if (subprocessOutput) {
stdout.add(event);
}
},
onDone: () async => stdoutComplete.complete(),
);
if (subprocessOutput) {
process.stderr.listen(
(List<int> event) {
stderr.add(event);
},
onDone: () async => stderrComplete.complete(),
);
} else {
stderrComplete.complete();
}
} on ProcessException catch (e) {
final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
'failed with:\n${e.toString()}';
throw new ProcessRunnerException(message);
}
final int exitCode = await allComplete();
if (exitCode != 0 && !failOk) {
final String message =
'Running "${commandLine.join(' ')}" in ${workingDirectory.path} failed';
throw new ProcessRunnerException(
message, new ProcessResult(0, exitCode, null, 'returned $exitCode'));
}
return UTF8.decoder.convert(output).trim();
}
} }
/// Creates a pre-populated Flutter archive from a git repo. /// Creates a pre-populated Flutter archive from a git repo.
class ArchiveCreator { class ArchiveCreator {
/// [_tempDir] is the directory to use for creating the archive. The script /// [tempDir] is the directory to use for creating the archive. The script
/// will place several GiB of data there, so it should have available space. /// will place several GiB of data there, so it should have available space.
/// ///
/// The processManager argument is used to inject a mock of [ProcessManager] for /// The processManager argument is used to inject a mock of [ProcessManager] for
...@@ -40,52 +177,85 @@ class ArchiveCreator { ...@@ -40,52 +177,85 @@ class ArchiveCreator {
/// ///
/// If subprocessOutput is true, then output from processes invoked during /// If subprocessOutput is true, then output from processes invoked during
/// archive creation is echoed to stderr and stdout. /// archive creation is echoed to stderr and stdout.
ArchiveCreator(this._tempDir, {ProcessManager processManager, bool subprocessOutput: true}) ArchiveCreator(
: _flutterRoot = new Directory(path.join(_tempDir.path, 'flutter')), this.tempDir,
_processManager = processManager ?? const LocalProcessManager(), this.outputDir,
_subprocessOutput = subprocessOutput { this.revision,
this.branch, {
ProcessManager processManager,
bool subprocessOutput: true,
this.platform: const LocalPlatform(),
}) : assert(revision.length == 40),
flutterRoot = new Directory(path.join(tempDir.path, 'flutter')),
_processRunner = new ProcessRunner(
processManager: processManager,
subprocessOutput: subprocessOutput,
platform: platform,
) {
_flutter = path.join( _flutter = path.join(
_flutterRoot.absolute.path, flutterRoot.absolute.path,
'bin', 'bin',
'flutter', 'flutter',
); );
_environment = new Map<String, String>.from(Platform.environment); _processRunner.environment['PUB_CACHE'] = path.join(flutterRoot.absolute.path, '.pub-cache');
_environment['PUB_CACHE'] = path.join(_flutterRoot.absolute.path, '.pub-cache');
} }
final Directory _flutterRoot; final Platform platform;
final Directory _tempDir; final Branch branch;
final bool _subprocessOutput; final String revision;
final ProcessManager _processManager; final Directory flutterRoot;
String _flutter; final Directory tempDir;
final Directory outputDir;
final Uri _minGitUri = Uri.parse(MINGIT_FOR_WINDOWS_URL); final Uri _minGitUri = Uri.parse(MINGIT_FOR_WINDOWS_URL);
Map<String, String> _environment; final ProcessRunner _processRunner;
File _outputFile;
String _version;
String _flutter;
/// Get the name of the channel as a string.
String get branchName => getBranchName(branch);
/// Returns a default archive name when given a Git revision. /// Returns a default archive name when given a Git revision.
/// Used when an output filename is not given. /// Used when an output filename is not given.
static String defaultArchiveName(String revision) { String get _archiveName {
final String os = Platform.operatingSystem.toLowerCase(); final String os = platform.operatingSystem.toLowerCase();
final String id = revision.length > 10 ? revision.substring(0, 10) : revision; final String suffix = platform.isWindows ? 'zip' : 'tar.xz';
final String suffix = Platform.isWindows ? 'zip' : 'tar.xz'; return 'flutter_${os}_$_version-$branchName.$suffix';
return 'flutter_${os}_$id.$suffix'; }
/// Checks out the flutter repo and prepares it for other operations.
///
/// Returns the version for this release, as obtained from the git tags.
Future<String> initializeRepo() async {
await _checkoutFlutter();
_version = await _getVersion();
return _version;
} }
/// Performs all of the steps needed to create an archive. /// Performs all of the steps needed to create an archive.
Future<File> createArchive(String revision, File outputFile) async { Future<File> createArchive() async {
await _checkoutFlutter(revision); assert(_version != null, 'Must run initializeRepo before createArchive');
_outputFile = new File(path.join(outputDir.absolute.path, _archiveName));
await _installMinGitIfNeeded(); await _installMinGitIfNeeded();
await _populateCaches(); await _populateCaches();
await _archiveFiles(outputFile); await _archiveFiles(_outputFile);
return outputFile; return _outputFile;
}
/// Returns the version number of this release, according the to tags in
/// the repo.
Future<String> _getVersion() async {
return _runGit(<String>['describe', '--tags', '--abbrev=0']);
} }
/// Clone the Flutter repo and make sure that the git environment is sane /// Clone the Flutter repo and make sure that the git environment is sane
/// for when the user will unpack it. /// for when the user will unpack it.
Future<Null> _checkoutFlutter(String revision) async { Future<Null> _checkoutFlutter() async {
// We want the user to start out the in the 'master' branch instead of a // We want the user to start out the in the specified branch instead of a
// detached head. To do that, we need to make sure master points at the // detached head. To do that, we need to make sure the branch points at the
// desired revision. // desired revision.
await _runGit(<String>['clone', '-b', 'master', CHROMIUM_REPO], workingDirectory: _tempDir); await _runGit(<String>['clone', '-b', branchName, CHROMIUM_REPO], workingDirectory: tempDir);
await _runGit(<String>['reset', '--hard', revision]); await _runGit(<String>['reset', '--hard', revision]);
// Make the origin point to github instead of the chromium mirror. // Make the origin point to github instead of the chromium mirror.
...@@ -95,16 +265,17 @@ class ArchiveCreator { ...@@ -95,16 +265,17 @@ class ArchiveCreator {
/// Retrieve the MinGit executable from storage and unpack it. /// Retrieve the MinGit executable from storage and unpack it.
Future<Null> _installMinGitIfNeeded() async { Future<Null> _installMinGitIfNeeded() async {
if (!Platform.isWindows) { if (!platform.isWindows) {
return; return;
} }
final Uint8List data = await http.readBytes(_minGitUri); final Uint8List data = await http.readBytes(_minGitUri);
final File gitFile = new File(path.join(_tempDir.path, 'mingit.zip')); final File gitFile = new File(path.join(tempDir.absolute.path, 'mingit.zip'));
await gitFile.writeAsBytes(data, flush: true); await gitFile.writeAsBytes(data, flush: true);
final Directory minGitPath = new Directory(path.join(_flutterRoot.path, 'bin', 'mingit')); final Directory minGitPath =
new Directory(path.join(flutterRoot.absolute.path, 'bin', 'mingit'));
await minGitPath.create(recursive: true); await minGitPath.create(recursive: true);
await _unzipArchive(gitFile, currentDirectory: minGitPath); await _unzipArchive(gitFile, workingDirectory: minGitPath);
} }
/// Prepare the archive repo so that it has all of the caches warmed up and /// Prepare the archive repo so that it has all of the caches warmed up and
...@@ -118,8 +289,11 @@ class ArchiveCreator { ...@@ -118,8 +289,11 @@ class ArchiveCreator {
// Create each of the templates, since they will call 'pub get' on // Create each of the templates, since they will call 'pub get' on
// themselves when created, and this will warm the cache with their // themselves when created, and this will warm the cache with their
// dependencies too. // dependencies too.
for (String template in <String>['app', 'package', 'plugin']) { // TODO(gspencer): 'package' is broken on dev branch right now!
final String createName = path.join(_tempDir.path, 'create_$template'); // Add it back in once the following is fixed:
// https://github.com/flutter/flutter/issues/14448
for (String template in <String>['app', 'plugin']) {
final String createName = path.join(tempDir.path, 'create_$template');
await _runFlutter( await _runFlutter(
<String>['create', '--template=$template', createName], <String>['create', '--template=$template', createName],
); );
...@@ -134,111 +308,154 @@ class ArchiveCreator { ...@@ -134,111 +308,154 @@ class ArchiveCreator {
/// Write the archive to the given output file. /// Write the archive to the given output file.
Future<Null> _archiveFiles(File outputFile) async { Future<Null> _archiveFiles(File outputFile) async {
if (outputFile.path.toLowerCase().endsWith('.zip')) { if (outputFile.path.toLowerCase().endsWith('.zip')) {
await _createZipArchive(outputFile, _flutterRoot); await _createZipArchive(outputFile, flutterRoot);
} else if (outputFile.path.toLowerCase().endsWith('.tar.xz')) { } else if (outputFile.path.toLowerCase().endsWith('.tar.xz')) {
await _createTarArchive(outputFile, _flutterRoot); await _createTarArchive(outputFile, flutterRoot);
} }
} }
Future<String> _runFlutter(List<String> args) => _runProcess(<String>[_flutter]..addAll(args)); Future<String> _runFlutter(List<String> args, {Directory workingDirectory}) {
return _processRunner.runProcess(<String>[_flutter]..addAll(args),
workingDirectory: workingDirectory ?? flutterRoot);
}
Future<String> _runGit(List<String> args, {Directory workingDirectory}) { Future<String> _runGit(List<String> args, {Directory workingDirectory}) {
return _runProcess(<String>['git']..addAll(args), workingDirectory: workingDirectory); return _processRunner.runProcess(<String>['git']..addAll(args),
workingDirectory: workingDirectory ?? flutterRoot);
} }
/// Unpacks the given zip file into the currentDirectory (if set), or the /// Unpacks the given zip file into the currentDirectory (if set), or the
/// same directory as the archive. /// same directory as the archive.
/// ///
/// May only be run on Windows (since 7Zip is not available on other platforms). /// May only be run on Windows (since 7Zip is not available on other platforms).
Future<String> _unzipArchive(File archive, {Directory currentDirectory}) { Future<String> _unzipArchive(File archive, {Directory workingDirectory}) {
assert(Platform.isWindows); // 7Zip is only available on Windows. assert(platform.isWindows); // 7Zip is only available on Windows.
currentDirectory ??= new Directory(path.dirname(archive.absolute.path)); workingDirectory ??= new Directory(path.dirname(archive.absolute.path));
final List<String> commandLine = <String>['7za', 'x', archive.absolute.path]; final List<String> commandLine = <String>['7za', 'x', archive.absolute.path];
return _runProcess(commandLine, workingDirectory: currentDirectory); return _processRunner.runProcess(commandLine, workingDirectory: workingDirectory);
} }
/// Create a zip archive from the directory source. /// Create a zip archive from the directory source.
/// ///
/// May only be run on Windows (since 7Zip is not available on other platforms). /// May only be run on Windows (since 7Zip is not available on other platforms).
Future<String> _createZipArchive(File output, Directory source) { Future<String> _createZipArchive(File output, Directory source) {
assert(Platform.isWindows); // 7Zip is only available on Windows. assert(platform.isWindows); // 7Zip is only available on Windows.
final List<String> commandLine = <String>[ final List<String> commandLine = <String>[
'7za', '7za',
'a', 'a',
'-tzip', '-tzip',
'-mx=9', '-mx=9',
output.absolute.path, output.absolute.path,
path.basename(source.absolute.path), path.basename(source.path),
]; ];
return _runProcess(commandLine, return _processRunner.runProcess(commandLine,
workingDirectory: new Directory(path.dirname(source.absolute.path))); workingDirectory: new Directory(path.dirname(source.absolute.path)));
} }
/// Create a tar archive from the directory source. /// Create a tar archive from the directory source.
Future<String> _createTarArchive(File output, Directory source) { Future<String> _createTarArchive(File output, Directory source) {
return _runProcess(<String>[ return _processRunner.runProcess(<String>[
'tar', 'tar',
'cJf', 'cJf',
output.absolute.path, output.absolute.path,
path.basename(source.absolute.path), path.basename(source.absolute.path),
], workingDirectory: new Directory(path.dirname(source.absolute.path))); ], workingDirectory: new Directory(path.dirname(source.absolute.path)));
} }
}
/// Run the command and arguments in commandLine as a sub-process from class ArchivePublisher {
/// workingDirectory if set, or the current directory if not. ArchivePublisher(
Future<String> _runProcess(List<String> commandLine, {Directory workingDirectory}) async { this.tempDir,
workingDirectory ??= _flutterRoot; this.revision,
if (_subprocessOutput) { this.branch,
stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n'); this.version,
} this.outputFile, {
final List<int> output = <int>[]; ProcessManager processManager,
final Completer<Null> stdoutComplete = new Completer<Null>(); bool subprocessOutput: true,
final Completer<Null> stderrComplete = new Completer<Null>(); this.platform: const LocalPlatform(),
Process process; }) : assert(revision.length == 40),
Future<int> allComplete() async { platformName = platform.operatingSystem.toLowerCase(),
await stderrComplete.future; metadataGsPath = '$gsReleaseFolder/releases_${platform.operatingSystem.toLowerCase()}.json',
await stdoutComplete.future; _processRunner = new ProcessRunner(
return process.exitCode; processManager: processManager,
} subprocessOutput: subprocessOutput,
);
static String gsBase = 'gs://flutter_infra';
static String releaseFolder = '/releases';
static String gsReleaseFolder = '$gsBase$releaseFolder';
static String baseUrl = 'https://storage.googleapis.com/flutter_infra';
final Platform platform;
final String platformName;
final String metadataGsPath;
final Branch branch;
final String revision;
final String version;
final Directory tempDir;
final File outputFile;
final ProcessRunner _processRunner;
String get branchName => getBranchName(branch);
String get destinationArchivePath =>
'$branchName/$platformName/${path.basename(outputFile.path)}';
/// Publish the archive to Google Storage.
Future<Null> publishArchive() async {
final String destGsPath = '$gsReleaseFolder/$destinationArchivePath';
await _cloudCopy(outputFile.absolute.path, destGsPath);
assert(tempDir.existsSync());
return _updateMetadata();
}
Future<Null> _updateMetadata() async {
final String currentMetadata = await _runGsUtil(<String>['cat', metadataGsPath]);
if (currentMetadata.isEmpty) {
throw new ProcessRunnerException('Empty metadata received from server');
}
Map<String, dynamic> jsonData;
try { try {
process = await _processManager.start( jsonData = json.decode(currentMetadata);
commandLine, } on FormatException catch (e) {
workingDirectory: workingDirectory.absolute.path, throw new ProcessRunnerException('Unable to parse JSON metadata received from cloud: $e');
environment: _environment,
);
process.stdout.listen(
(List<int> event) {
output.addAll(event);
if (_subprocessOutput) {
stdout.add(event);
} }
},
onDone: () async => stdoutComplete.complete(), // Update the metadata file with the data for this package.
); jsonData['base_url'] = '$baseUrl$releaseFolder';
if (_subprocessOutput) { if (!jsonData.containsKey('current_release')) {
process.stderr.listen( jsonData['current_release'] = <String, String>{};
(List<int> event) {
stderr.add(event);
},
onDone: () async => stderrComplete.complete(),
);
} else {
stderrComplete.complete();
} }
} on ProcessException catch (e) { jsonData['current_release'][branchName] = revision;
final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} ' if (!jsonData.containsKey('releases')) {
'failed with:\n${e.toString()}'; jsonData['releases'] = <String, dynamic>{};
throw new ProcessFailedException(message, -1);
} }
if (!jsonData['releases'].containsKey(revision)) {
jsonData['releases'][revision] = <String, Map<String, String>>{};
}
final Map<String, String> metadata = <String, String>{};
metadata['${platformName}_archive'] = destinationArchivePath;
metadata['release_date'] = new DateTime.now().toUtc().toIso8601String();
metadata['version'] = version;
jsonData['releases'][revision][branchName] = metadata;
final int exitCode = await allComplete(); final File tempFile = new File(path.join(tempDir.absolute.path, 'releases_$platformName.json'));
if (exitCode != 0) { final JsonEncoder encoder = const JsonEncoder.withIndent(' ');
final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} ' tempFile.writeAsStringSync(encoder.convert(jsonData));
'failed with $exitCode.'; await _cloudCopy(tempFile.absolute.path, metadataGsPath);
throw new ProcessFailedException(message, exitCode);
} }
return UTF8.decoder.convert(output).trim();
Future<String> _runGsUtil(List<String> args,
{Directory workingDirectory, bool failOk: false}) async {
return _processRunner.runProcess(
<String>['gsutil']..addAll(args),
workingDirectory: workingDirectory,
failOk: failOk,
);
}
Future<String> _cloudCopy(String src, String dest) async {
await _runGsUtil(<String>['rm', dest], failOk: true);
return _runGsUtil(<String>['cp', src, dest]);
} }
} }
...@@ -256,23 +473,37 @@ Future<Null> main(List<String> argList) async { ...@@ -256,23 +473,37 @@ Future<Null> main(List<String> argList) async {
defaultsTo: null, defaultsTo: null,
help: 'A location where temporary files may be written. Defaults to a ' help: 'A location where temporary files may be written. Defaults to a '
'directory in the system temp folder. Will write a few GiB of data, ' 'directory in the system temp folder. Will write a few GiB of data, '
'so it should have sufficient free space.', 'so it should have sufficient free space. If a temp_dir is not '
'specified, then the default temp_dir will be created, used, and '
'removed automatically.',
); );
argParser.addOption('revision',
defaultsTo: null,
help: 'The Flutter git repo revision to build the '
'archive with. Must be the full 40-character hash. Required.');
argParser.addOption( argParser.addOption(
'revision', 'branch',
defaultsTo: 'master', defaultsTo: null,
help: 'The Flutter revision to build the archive with. Defaults to the ' allowed: Branch.values.map((Branch branch) => getBranchName(branch)),
"master branch's HEAD revision.", help: 'The Flutter branch to build the archive with. Required.',
); );
argParser.addOption( argParser.addOption(
'output', 'output',
defaultsTo: null, defaultsTo: null,
help: 'The path to the file where the output archive should be ' help: 'The path to the directory where the output archive should be '
'written. The output file must end in ".tar.xz" on Linux and Mac, ' 'written. If --output is not specified, the archive will be written to '
'and ".zip" on Windows. If --output is not specified, the archive will ' "the current directory. If the output directory doesn't exist, it, and "
"be written to the current directory. If the output directory doesn't " 'the path to it, will be created.',
'exist, it, and the path to it, will be created.', );
argParser.addFlag(
'publish',
defaultsTo: false,
help: 'The path to the directory where the output archive should be '
'written. If --output is not specified, the archive will be written to '
"the current directory. If the output directory doesn't exist, it, and "
'the path to it, will be created.',
); );
final ArgResults args = argParser.parse(argList); final ArgResults args = argParser.parse(argList);
void errorExit(String message, {int exitCode = -1}) { void errorExit(String message, {int exitCode = -1}) {
...@@ -281,9 +512,17 @@ Future<Null> main(List<String> argList) async { ...@@ -281,9 +512,17 @@ Future<Null> main(List<String> argList) async {
exit(exitCode); exit(exitCode);
} }
if (args['revision'].isEmpty) { final String revision = args['revision'];
if (revision.isEmpty) {
errorExit('Invalid argument: --revision must be specified.'); errorExit('Invalid argument: --revision must be specified.');
} }
if (revision.length != 40) {
errorExit('Invalid argument: --revision must be the entire hash, not just a prefix.');
}
if (args['branch'].isEmpty) {
errorExit('Invalid argument: --branch must be specified.');
}
Directory tempDir; Directory tempDir;
bool removeTempDir = false; bool removeTempDir = false;
...@@ -297,34 +536,34 @@ Future<Null> main(List<String> argList) async { ...@@ -297,34 +536,34 @@ Future<Null> main(List<String> argList) async {
} }
} }
final String output = (args['output'] == null || args['output'].isEmpty) Directory outputDir;
? path.join(path.current, ArchiveCreator.defaultArchiveName(args['revision'])) if (args['output'] == null) {
: args['output']; outputDir = tempDir;
/// Sanity check the output filename.
final String outputFilename = path.basename(output);
if (Platform.isWindows) {
if (!outputFilename.endsWith('.zip')) {
errorExit('The argument to --output must end in .zip on Windows.');
}
} else { } else {
if (!outputFilename.endsWith('.tar.xz')) { outputDir = new Directory(args['output']);
errorExit('The argument to --output must end in .tar.xz on Linux and Mac.'); if (!outputDir.existsSync()) {
outputDir.createSync(recursive: true);
} }
} }
final Directory outputDirectory = new Directory(path.dirname(output)); final Branch branch = fromBranchName(args['branch']);
if (!outputDirectory.existsSync()) { final ArchiveCreator creator = new ArchiveCreator(tempDir, outputDir, revision, branch);
outputDirectory.createSync(recursive: true);
}
final File outputFile = new File(path.join(outputDirectory.absolute.path, outputFilename));
final ArchiveCreator preparer = new ArchiveCreator(tempDir);
int exitCode = 0; int exitCode = 0;
String message; String message;
try { try {
await preparer.createArchive(args['revision'], outputFile); final String version = await creator.initializeRepo();
} on ProcessFailedException catch (e) { final File outputFile = await creator.createArchive();
if (args['publish']) {
final ArchivePublisher publisher = new ArchivePublisher(
tempDir,
revision,
branch,
version,
outputFile,
);
await publisher.publishArchive();
}
} on ProcessRunnerException catch (e) {
exitCode = e.exitCode; exitCode = e.exitCode;
message = e.message; message = e.message;
} finally { } finally {
......
...@@ -29,111 +29,88 @@ class FakeProcessManager extends Mock implements ProcessManager { ...@@ -29,111 +29,88 @@ class FakeProcessManager extends Mock implements ProcessManager {
/// The list of results that will be sent back, organized by the command line /// The list of results that will be sent back, organized by the command line
/// that will produce them. Each command line has a list of returned stdout /// that will produce them. Each command line has a list of returned stdout
/// output that will be returned on each successive call. /// output that will be returned on each successive call.
Map<String, List<ProcessResult>> fakeResults = <String, List<ProcessResult>>{}; Map<String, List<ProcessResult>> _fakeResults = <String, List<ProcessResult>>{};
Map<String, List<ProcessResult>> get fakeResults => _fakeResults;
set fakeResults(Map<String, List<ProcessResult>> value) {
_fakeResults = <String, List<ProcessResult>>{};
for (String key in value.keys) {
_fakeResults[key] = <ProcessResult>[]
..addAll(value[key] ?? <ProcessResult>[new ProcessResult(0, 0, '', '')]);
}
}
/// The list of invocations that occurred, in the order they occurred. /// The list of invocations that occurred, in the order they occurred.
List<Invocation> invocations = <Invocation>[]; List<Invocation> invocations = <Invocation>[];
/// Verify that the given command lines were called, in the given order. /// Verify that the given command lines were called, in the given order, and that the
/// parameters were in the same order.
void verifyCalls(List<String> calls) { void verifyCalls(List<String> calls) {
int index = 0; int index = 0;
expect(invocations.length, equals(calls.length));
for (String call in calls) { for (String call in calls) {
expect(call.split(' '), orderedEquals(invocations[index].positionalArguments[0])); expect(call.split(' '), orderedEquals(invocations[index].positionalArguments[0]));
index++; index++;
} }
expect(invocations.length, equals(calls.length));
} }
/// Sets the list of results that will be returned from each successive call. ProcessResult _popResult(List<String> command) {
void setResults(Map<String, List<String>> results) { final String key = command.join(' ');
final Map<String, List<ProcessResult>> resultCodeUnits = <String, List<ProcessResult>>{};
for (String key in results.keys) {
resultCodeUnits[key] =
results[key].map((String result) => new ProcessResult(0, 0, result, '')).toList();
}
fakeResults = resultCodeUnits;
}
ProcessResult _popResult(String key) {
expect(fakeResults, isNotEmpty); expect(fakeResults, isNotEmpty);
expect(fakeResults, contains(key)); expect(fakeResults, contains(key));
expect(fakeResults[key], isNotEmpty); expect(fakeResults[key], isNotEmpty);
return fakeResults[key].removeAt(0); return fakeResults[key].removeAt(0);
} }
FakeProcess _popProcess(String key) => FakeProcess _popProcess(List<String> command) =>
new FakeProcess(_popResult(key), stdinResults: stdinResults); new FakeProcess(_popResult(command), stdinResults: stdinResults);
Future<Process> _nextProcess(Invocation invocation) async { Future<Process> _nextProcess(Invocation invocation) async {
invocations.add(invocation); invocations.add(invocation);
return new Future<Process>.value(_popProcess(invocation.positionalArguments[0].join(' '))); return new Future<Process>.value(_popProcess(invocation.positionalArguments[0]));
} }
ProcessResult _nextResultSync(Invocation invocation) { ProcessResult _nextResultSync(Invocation invocation) {
invocations.add(invocation); invocations.add(invocation);
return _popResult(invocation.positionalArguments[0].join(' ')); return _popResult(invocation.positionalArguments[0]);
} }
Future<ProcessResult> _nextResult(Invocation invocation) async { Future<ProcessResult> _nextResult(Invocation invocation) async {
invocations.add(invocation); invocations.add(invocation);
return new Future<ProcessResult>.value(_popResult(invocation.positionalArguments[0].join(' '))); return new Future<ProcessResult>.value(_popResult(invocation.positionalArguments[0]));
} }
void _setupMock() { void _setupMock() {
// Note that not all possible types of invocations are covered here, just the ones // Note that not all possible types of invocations are covered here, just the ones
// expected to be called. // expected to be called.
// TODO(gspencer): make this more general so that any call will be captured. // TODO(gspencer): make this more general so that any call will be captured.
when( when(start(
start(
typed(captureAny), typed(captureAny),
environment: typed(captureAny, named: 'environment'), environment: typed(captureAny, named: 'environment'),
workingDirectory: typed(captureAny, named: 'workingDirectory'), workingDirectory: typed(captureAny, named: 'workingDirectory'),
), )).thenAnswer(_nextProcess);
).thenAnswer(_nextProcess);
when( when(start(typed(captureAny))).thenAnswer(_nextProcess);
start(
typed(captureAny),
),
).thenAnswer(_nextProcess);
when( when(run(
run(
typed(captureAny), typed(captureAny),
environment: typed(captureAny, named: 'environment'), environment: typed(captureAny, named: 'environment'),
workingDirectory: typed(captureAny, named: 'workingDirectory'), workingDirectory: typed(captureAny, named: 'workingDirectory'),
), )).thenAnswer(_nextResult);
).thenAnswer(_nextResult);
when( when(run(typed(captureAny))).thenAnswer(_nextResult);
run(
typed(captureAny),
),
).thenAnswer(_nextResult);
when( when(runSync(
runSync(
typed(captureAny), typed(captureAny),
environment: typed(captureAny, named: 'environment'), environment: typed(captureAny, named: 'environment'),
workingDirectory: typed(captureAny, named: 'workingDirectory'), workingDirectory: typed(captureAny, named: 'workingDirectory')
), )).thenAnswer(_nextResultSync);
).thenAnswer(_nextResultSync);
when( when(runSync(typed(captureAny))).thenAnswer(_nextResultSync);
runSync(
typed(captureAny),
),
).thenAnswer(_nextResultSync);
when(killPid(typed(captureAny), typed(captureAny))).thenReturn(true); when(killPid(typed(captureAny), typed(captureAny))).thenReturn(true);
when( when(canRun(captureAny, workingDirectory: typed(captureAny, named: 'workingDirectory')))
canRun(captureAny, .thenReturn(true);
workingDirectory: typed(
captureAny,
named: 'workingDirectory',
)),
).thenReturn(true);
} }
} }
...@@ -190,9 +167,11 @@ class StringStreamConsumer implements StreamConsumer<List<int>> { ...@@ -190,9 +167,11 @@ class StringStreamConsumer implements StreamConsumer<List<int>> {
Future<dynamic> addStream(Stream<List<int>> value) { Future<dynamic> addStream(Stream<List<int>> value) {
streams.add(value); streams.add(value);
completers.add(new Completer<dynamic>()); completers.add(new Completer<dynamic>());
subscriptions.add(value.listen((List<int> data) { subscriptions.add(
value.listen((List<int> data) {
sendString(utf8.decode(data)); sendString(utf8.decode(data));
})); }),
);
subscriptions.last.onDone(() => completers.last.complete(null)); subscriptions.last.onDone(() => completers.last.complete(null));
return new Future<dynamic>.value(null); return new Future<dynamic>.value(null);
} }
......
...@@ -25,11 +25,15 @@ void main() { ...@@ -25,11 +25,15 @@ void main() {
tearDown(() async {}); tearDown(() async {});
test('start works', () async { test('start works', () async {
final Map<String, List<String>> calls = <String, List<String>>{ final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
'gsutil acl get gs://flutter_infra/releases/releases.json': <String>['output1'], 'gsutil acl get gs://flutter_infra/releases/releases.json': <ProcessResult>[
'gsutil cat gs://flutter_infra/releases/releases.json': <String>['test'], new ProcessResult(0, 0, 'output1', '')
],
'gsutil cat gs://flutter_infra/releases/releases.json': <ProcessResult>[
new ProcessResult(0, 0, 'output2', '')
],
}; };
processManager.setResults(calls); processManager.fakeResults = calls;
for (String key in calls.keys) { for (String key in calls.keys) {
final Process process = await processManager.start(key.split(' ')); final Process process = await processManager.start(key.split(' '));
String output = ''; String output = '';
...@@ -37,56 +41,68 @@ void main() { ...@@ -37,56 +41,68 @@ void main() {
output += utf8.decode(item); output += utf8.decode(item);
}); });
await process.exitCode; await process.exitCode;
expect(output, equals(calls[key][0])); expect(output, equals(calls[key][0].stdout));
} }
processManager.verifyCalls(calls.keys); processManager.verifyCalls(calls.keys.toList());
}); });
test('run works', () async { test('run works', () async {
final Map<String, List<String>> calls = <String, List<String>>{ final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
'gsutil acl get gs://flutter_infra/releases/releases.json': <String>['output1'], 'gsutil acl get gs://flutter_infra/releases/releases.json': <ProcessResult>[
'gsutil cat gs://flutter_infra/releases/releases.json': <String>['test'], new ProcessResult(0, 0, 'output1', '')
],
'gsutil cat gs://flutter_infra/releases/releases.json': <ProcessResult>[
new ProcessResult(0, 0, 'output2', '')
],
}; };
processManager.setResults(calls); processManager.fakeResults = calls;
for (String key in calls.keys) { for (String key in calls.keys) {
final ProcessResult result = await processManager.run(key.split(' ')); final ProcessResult result = await processManager.run(key.split(' '));
expect(result.stdout, equals(calls[key][0])); expect(result.stdout, equals(calls[key][0].stdout));
} }
processManager.verifyCalls(calls.keys); processManager.verifyCalls(calls.keys.toList());
}); });
test('runSync works', () async { test('runSync works', () async {
final Map<String, List<String>> calls = <String, List<String>>{ final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
'gsutil acl get gs://flutter_infra/releases/releases.json': <String>['output1'], 'gsutil acl get gs://flutter_infra/releases/releases.json': <ProcessResult>[
'gsutil cat gs://flutter_infra/releases/releases.json': <String>['test'], new ProcessResult(0, 0, 'output1', '')
],
'gsutil cat gs://flutter_infra/releases/releases.json': <ProcessResult>[
new ProcessResult(0, 0, 'output2', '')
],
}; };
processManager.setResults(calls); processManager.fakeResults = calls;
for (String key in calls.keys) { for (String key in calls.keys) {
final ProcessResult result = processManager.runSync(key.split(' ')); final ProcessResult result = processManager.runSync(key.split(' '));
expect(result.stdout, equals(calls[key][0])); expect(result.stdout, equals(calls[key][0].stdout));
} }
processManager.verifyCalls(calls.keys); processManager.verifyCalls(calls.keys.toList());
}); });
test('captures stdin', () async { test('captures stdin', () async {
final Map<String, List<String>> calls = <String, List<String>>{ final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
'gsutil acl get gs://flutter_infra/releases/releases.json': <String>['output1'], 'gsutil acl get gs://flutter_infra/releases/releases.json': <ProcessResult>[
'gsutil cat gs://flutter_infra/releases/releases.json': <String>['test'], new ProcessResult(0, 0, 'output1', '')
],
'gsutil cat gs://flutter_infra/releases/releases.json': <ProcessResult>[
new ProcessResult(0, 0, 'output2', '')
],
}; };
processManager.setResults(calls); processManager.fakeResults = calls;
for (String key in calls.keys) { for (String key in calls.keys) {
final Process process = await processManager.start(key.split(' ')); final Process process = await processManager.start(key.split(' '));
String output = ''; String output = '';
process.stdout.listen((List<int> item) { process.stdout.listen((List<int> item) {
output += utf8.decode(item); output += utf8.decode(item);
}); });
final String testInput = '${calls[key][0]} input'; final String testInput = '${calls[key][0].stdout} input';
process.stdin.add(testInput.codeUnits); process.stdin.add(testInput.codeUnits);
await process.exitCode; await process.exitCode;
expect(output, equals(calls[key][0])); expect(output, equals(calls[key][0].stdout));
expect(stdinCaptured.last, equals(testInput)); expect(stdinCaptured.last, equals(testInput));
} }
processManager.verifyCalls(calls.keys); processManager.verifyCalls(calls.keys.toList());
}); });
}); });
} }
...@@ -2,77 +2,94 @@ ...@@ -2,77 +2,94 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
import 'dart:async'; import 'dart:convert';
import 'dart:io'; import 'dart:io' hide Platform;
import 'package:mockito/mockito.dart'; import 'package:mockito/mockito.dart';
import 'package:test/test.dart'; import 'package:test/test.dart';
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
import 'package:process/process.dart'; import 'package:platform/platform.dart' show FakePlatform;
import '../prepare_package.dart'; import '../prepare_package.dart';
import 'fake_process_manager.dart';
void main() { void main() {
group('ArchiveCreator', () { final String testRef = 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeef';
ArchiveCreator preparer; for (String platformName in <String>['macos', 'linux', 'windows']) {
final FakePlatform platform = new FakePlatform(
operatingSystem: platformName,
environment: <String, String>{},
);
group('ArchiveCreator for $platformName', () {
ArchiveCreator creator;
Directory tmpDir; Directory tmpDir;
Directory flutterDir; Directory flutterDir;
File outputFile; FakeProcessManager processManager;
MockProcessManager processManager;
List<MockProcess> results = <MockProcess>[];
final List<List<String>> args = <List<String>>[]; final List<List<String>> args = <List<String>>[];
final List<Map<Symbol, dynamic>> namedArgs = <Map<Symbol, dynamic>>[]; final List<Map<Symbol, dynamic>> namedArgs = <Map<Symbol, dynamic>>[];
String flutterExe; String flutter;
void _verifyCommand(List<dynamic> args, String expected) {
final List<String> expectedList = expected.split(' ');
expect(args[0], orderedEquals(expectedList));
}
Future<Process> _nextResult(Invocation invocation) async {
args.add(invocation.positionalArguments);
namedArgs.add(invocation.namedArguments);
final Process result = results.isEmpty ? new MockProcess('', '', 0) : results.removeAt(0);
return new Future<Process>.value(result);
}
void _answerWithResults() {
when(
processManager.start(
typed(captureAny),
environment: typed(captureAny, named: 'environment'),
workingDirectory: typed(captureAny, named: 'workingDirectory'),
),
).thenAnswer(_nextResult);
}
setUp(() async { setUp(() async {
processManager = new MockProcessManager(); processManager = new FakeProcessManager();
args.clear(); args.clear();
namedArgs.clear(); namedArgs.clear();
tmpDir = await Directory.systemTemp.createTemp('flutter_'); tmpDir = await Directory.systemTemp.createTemp('flutter_');
outputFile =
new File(path.join(tmpDir.absolute.path, ArchiveCreator.defaultArchiveName('master')));
flutterDir = new Directory(path.join(tmpDir.path, 'flutter')); flutterDir = new Directory(path.join(tmpDir.path, 'flutter'));
flutterDir.createSync(recursive: true); flutterDir.createSync(recursive: true);
flutterExe = creator = new ArchiveCreator(
path.join(flutterDir.path, 'bin', 'flutter'); tmpDir,
tmpDir,
testRef,
Branch.dev,
processManager: processManager,
subprocessOutput: false,
platform: platform,
);
flutter = path.join(creator.flutterRoot.absolute.path, 'bin', 'flutter');
}); });
tearDown(() async { tearDown(() async {
// On Windows, the directory is locked and not able to be deleted, because it is a // On Windows, the directory is locked and not able to be deleted yet. So
// temporary directory. So we just leave some (very small, because we're not actually // we just leave some (very small, because we're not actually building
// building archives here) trash around to be deleted at the next reboot. // archives here) trash around to be deleted at the next reboot.
if (!Platform.isWindows) { if (!platform.isWindows) {
await tmpDir.delete(recursive: true); await tmpDir.delete(recursive: true);
} }
}); });
test('sets PUB_CACHE properly', () async { test('sets PUB_CACHE properly', () async {
preparer = final String createBase = path.join(tmpDir.absolute.path, 'create_');
new ArchiveCreator(tmpDir, processManager: processManager, subprocessOutput: false); final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
_answerWithResults(); 'git clone -b dev https://chromium.googlesource.com/external/github.com/flutter/flutter':
await preparer.createArchive('master', outputFile); null,
'git reset --hard $testRef': null,
'git remote remove origin': null,
'git remote add origin https://github.com/flutter/flutter.git': null,
'git describe --tags --abbrev=0': <ProcessResult>[new ProcessResult(0, 0, 'v1.2.3', '')],
};
if (platform.isWindows) {
calls['7za x ${path.join(tmpDir.path, 'mingit.zip')}'] = null;
}
calls.addAll(<String, List<ProcessResult>>{
'$flutter doctor': null,
'$flutter update-packages': null,
'$flutter precache': null,
'$flutter ide-config': null,
'$flutter create --template=app ${createBase}app': null,
'$flutter create --template=package ${createBase}package': null,
'$flutter create --template=plugin ${createBase}plugin': null,
'git clean -f -X **/.packages': null,
});
final String archiveName = path.join(tmpDir.absolute.path,
'flutter_${platformName}_v1.2.3-dev${platform.isWindows ? '.zip' : '.tar.xz'}');
if (platform.isWindows) {
calls['7za a -tzip -mx=9 $archiveName flutter'] = null;
} else {
calls['tar cJf $archiveName flutter'] = null;
}
processManager.fakeResults = calls;
await creator.initializeRepo();
await creator.createArchive();
expect( expect(
verify(processManager.start( verify(processManager.start(
captureAny, captureAny,
...@@ -84,72 +101,148 @@ void main() { ...@@ -84,72 +101,148 @@ void main() {
}); });
test('calls the right commands for archive output', () async { test('calls the right commands for archive output', () async {
preparer = final String createBase = path.join(tmpDir.absolute.path, 'create_');
new ArchiveCreator(tmpDir, processManager: processManager, subprocessOutput: false); final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
_answerWithResults(); 'git clone -b dev https://chromium.googlesource.com/external/github.com/flutter/flutter':
await preparer.createArchive('master', outputFile); null,
final List<String> commands = <String>[ 'git reset --hard $testRef': null,
'git clone -b master https://chromium.googlesource.com/external/github.com/flutter/flutter', 'git remote remove origin': null,
'git reset --hard master', 'git remote add origin https://github.com/flutter/flutter.git': null,
'git remote remove origin', 'git describe --tags --abbrev=0': <ProcessResult>[new ProcessResult(0, 0, 'v1.2.3', '')],
'git remote add origin https://github.com/flutter/flutter.git', };
]; if (platform.isWindows) {
if (Platform.isWindows) { calls['7za x ${path.join(tmpDir.path, 'mingit.zip')}'] = null;
commands.add('7za x ${path.join(tmpDir.path, 'mingit.zip')}');
} }
commands.addAll(<String>[ calls.addAll(<String, List<ProcessResult>>{
'$flutterExe doctor', '$flutter doctor': null,
'$flutterExe update-packages', '$flutter update-packages': null,
'$flutterExe precache', '$flutter precache': null,
'$flutterExe ide-config', '$flutter ide-config': null,
'$flutterExe create --template=app ${path.join(tmpDir.path, 'create_app')}', '$flutter create --template=app ${createBase}app': null,
'$flutterExe create --template=package ${path.join(tmpDir.path, 'create_package')}', // TODO(gspencer): Re-enable this when package works again:
'$flutterExe create --template=plugin ${path.join(tmpDir.path, 'create_plugin')}', // https://github.com/flutter/flutter/issues/14448
'git clean -f -X **/.packages', // '$flutter create --template=package ${createBase}package': null,
]); '$flutter create --template=plugin ${createBase}plugin': null,
if (Platform.isWindows) { 'git clean -f -X **/.packages': null,
commands.add('7za a -tzip -mx=9 ${outputFile.absolute.path} flutter'); });
final String archiveName = path.join(tmpDir.absolute.path,
'flutter_${platformName}_v1.2.3-dev${platform.isWindows ? '.zip' : '.tar.xz'}');
if (platform.isWindows) {
calls['7za a -tzip -mx=9 $archiveName flutter'] = null;
} else { } else {
commands.add('tar cJf ${outputFile.absolute.path} flutter'); calls['tar cJf $archiveName flutter'] = null;
}
int step = 0;
for (String command in commands) {
_verifyCommand(args[step++], command);
} }
processManager.fakeResults = calls;
creator = new ArchiveCreator(
tmpDir,
tmpDir,
testRef,
Branch.dev,
processManager: processManager,
subprocessOutput: false,
platform: platform,
);
await creator.initializeRepo();
await creator.createArchive();
processManager.verifyCalls(calls.keys.toList());
}); });
test('throws when a command errors out', () async { test('throws when a command errors out', () async {
preparer = final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
new ArchiveCreator(tmpDir, processManager: processManager, subprocessOutput: false); 'git clone -b dev https://chromium.googlesource.com/external/github.com/flutter/flutter':
<ProcessResult>[new ProcessResult(0, 0, 'output1', '')],
results = <MockProcess>[ 'git reset --hard $testRef': <ProcessResult>[new ProcessResult(0, -1, 'output2', '')],
new MockProcess('', '', 0), };
new MockProcess('', "Don't panic.\n", -1) processManager.fakeResults = calls;
]; expect(expectAsync0(creator.initializeRepo),
_answerWithResults(); throwsA(const isInstanceOf<ProcessRunnerException>()));
expect(expectAsync2<Null, String, File>(preparer.createArchive)('master', new File('foo')),
throwsA(const isInstanceOf<ProcessFailedException>()));
}); });
}); });
}
class MockProcessManager extends Mock implements ProcessManager {}
class MockProcess extends Mock implements Process { group('ArchivePublisher for $platformName', () {
MockProcess(this._stdout, [this._stderr, this._exitCode]); FakeProcessManager processManager;
Directory tempDir;
String _stdout; setUp(() async {
String _stderr; processManager = new FakeProcessManager();
int _exitCode; tempDir = await Directory.systemTemp.createTemp('flutter_');
tempDir.createSync();
@override });
Stream<List<int>> get stdout =>
new Stream<List<int>>.fromIterable(<List<int>>[_stdout.codeUnits]);
@override tearDown(() async {
Stream<List<int>> get stderr => // On Windows, the directory is locked and not able to be deleted yet. So
new Stream<List<int>>.fromIterable(<List<int>>[_stderr.codeUnits]); // we just leave some (very small, because we're not actually building
// archives here) trash around to be deleted at the next reboot.
if (!platform.isWindows) {
await tempDir.delete(recursive: true);
}
});
@override test('calls the right processes', () async {
Future<int> get exitCode => new Future<int>.value(_exitCode); final String releasesName = 'releases_$platformName.json';
final String archivePath = path.join(tempDir.absolute.path, 'output_archive');
final String gsArchivePath = 'gs://flutter_infra/releases/dev/$platformName/output_archive';
final String jsonPath = path.join(tempDir.absolute.path, releasesName);
final String gsJsonPath = 'gs://flutter_infra/releases/$releasesName';
final String releasesJson = '''{
"base_url": "https://storage.googleapis.com/flutter_infra/releases",
"current_release": {
"beta": "6da8ec6bd0c4801b80d666869e4069698561c043",
"dev": "f88c60b38c3a5ef92115d24e3da4175b4890daba"
},
"releases": {
"6da8ec6bd0c4801b80d666869e4069698561c043": {
"${platformName}_archive": "dev/linux/flutter_${platformName}_0.21.0-beta.tar.xz",
"release_date": "2017-12-19T10:30:00,847287019-08:00",
"version": "0.21.0-beta"
},
"f88c60b38c3a5ef92115d24e3da4175b4890daba": {
"${platformName}_archive": "dev/linux/flutter_${platformName}_0.22.0-dev.tar.xz",
"release_date": "2018-01-19T13:30:09,728487019-08:00",
"version": "0.22.0-dev"
}
}
}
''';
final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
'gsutil rm $gsArchivePath': null,
'gsutil cp $archivePath $gsArchivePath': null,
'gsutil cat $gsJsonPath': <ProcessResult>[new ProcessResult(0, 0, releasesJson, '')],
'gsutil rm $gsJsonPath': null,
'gsutil cp $jsonPath $gsJsonPath': null,
};
processManager.fakeResults = calls;
final File outputFile = new File(path.join(tempDir.absolute.path, 'output_archive'));
assert(tempDir.existsSync());
final ArchivePublisher publisher = new ArchivePublisher(
tempDir,
testRef,
Branch.dev,
'1.2.3',
outputFile,
processManager: processManager,
subprocessOutput: false,
platform: platform,
);
assert(tempDir.existsSync());
await publisher.publishArchive();
processManager.verifyCalls(calls.keys.toList());
final File releaseFile = new File(jsonPath);
expect(releaseFile.existsSync(), isTrue);
final String contents = releaseFile.readAsStringSync();
// Make sure new data is added.
expect(contents, contains('"dev": "$testRef"'));
expect(contents, contains('"$testRef": {'));
expect(contents, contains('"${platformName}_archive": "dev/$platformName/output_archive"'));
// Make sure existing entries are preserved.
expect(contents, contains('"6da8ec6bd0c4801b80d666869e4069698561c043": {'));
expect(contents, contains('"f88c60b38c3a5ef92115d24e3da4175b4890daba": {'));
expect(contents, contains('"beta": "6da8ec6bd0c4801b80d666869e4069698561c043"'));
// Make sure it's valid JSON, and in the right format.
final Map<String, dynamic> jsonData = json.decode(contents);
final JsonEncoder encoder = const JsonEncoder.withIndent(' ');
expect(contents, equals(encoder.convert(jsonData)));
});
});
}
} }
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'dart:io';
import 'package:path/path.dart' as path;
import 'package:process/process.dart';
class ArchivePublisherException implements Exception {
ArchivePublisherException(this.message, [this.result]);
final String message;
final ProcessResult result;
@override
String toString() {
String output = 'ArchivePublisherException';
if (message != null) {
output += ': $message';
}
final String stderr = result?.stderr ?? '';
if (stderr.isNotEmpty) {
output += ':\n${result.stderr}';
}
return output;
}
}
enum Channel { dev, beta }
/// Publishes the archive created for a particular version and git hash to
/// the releases directory on cloud storage, and updates the metadata for
/// releases.
///
/// See https://github.com/flutter/flutter/wiki/Release-process for more
/// information on the release process.
class ArchivePublisher {
ArchivePublisher(
this.revision,
this.version,
this.channel, {
this.processManager = const LocalProcessManager(),
this.tempDir,
}) : assert(revision.length == 40, 'Git hash must be 40 characters long (i.e. the entire hash).');
/// A git hash describing the revision to publish. It should be the complete
/// hash, not just a prefix.
final String revision;
/// A version number for the release (e.g. "1.2.3").
final String version;
/// The channel to publish to.
// TODO(gspencer): support Channel.beta: it is currently unimplemented.
final Channel channel;
/// Get the name of the channel as a string.
String get channelName {
switch (channel) {
case Channel.beta:
return 'beta';
case Channel.dev:
default:
return 'dev';
}
}
/// The process manager to use for invoking commands. Typically only
/// used for testing purposes.
final ProcessManager processManager;
/// The temporary directory used for this publisher. If not set, one will
/// be created, used, and then removed automatically. If set, it will not be
/// deleted when done: that is left to the caller. Typically used by tests.
Directory tempDir;
static String gsBase = 'gs://flutter_infra';
static String releaseFolder = '/releases';
static String baseUrl = 'https://storage.googleapis.com/flutter_infra';
static String archivePrefix = 'flutter_';
static String releaseNotesPrefix = 'release_notes_';
final String metadataGsPath = '$gsBase$releaseFolder/releases.json';
/// Publishes the archive for the given constructor parameters.
bool publishArchive() {
assert(channel == Channel.dev, 'Channel must be dev (beta not yet supported)');
final List<String> platforms = <String>['linux', 'mac', 'win'];
final Map<String, String> metadata = <String, String>{};
for (String platform in platforms) {
final String src = _builtArchivePath(platform);
final String dest = _destinationArchivePath(platform);
final String srcGsPath = '$gsBase$src';
final String destGsPath = '$gsBase$releaseFolder$dest';
_cloudCopy(srcGsPath, destGsPath);
metadata['${platform}_archive'] = '$channelName/$platform$dest';
}
metadata['release_date'] = new DateTime.now().toUtc().toIso8601String();
metadata['version'] = version;
_updateMetadata(metadata);
return true;
}
/// Checks to make sure the user has access to the Google Storage bucket
/// required to publish. Will throw an [ArchivePublisherException] if not.
void checkForGSUtilAccess() {
// Fetching ACLs requires FULL_CONTROL access.
final ProcessResult result = _runGsUtil(<String>['acl', 'get', metadataGsPath]);
if (result.exitCode != 0) {
throw new ArchivePublisherException(
'GSUtil cannot get ACLs for metadata file $metadataGsPath',
result,
);
}
}
void _updateMetadata(Map<String, String> metadata) {
final ProcessResult result = _runGsUtil(<String>['cat', metadataGsPath]);
if (result.exitCode != 0) {
throw new ArchivePublisherException(
'Unable to get existing metadata at $metadataGsPath', result);
}
final String currentMetadata = result.stdout;
if (currentMetadata.isEmpty) {
throw new ArchivePublisherException('Empty metadata received from server', result);
}
Map<String, dynamic> jsonData;
try {
jsonData = json.decode(currentMetadata);
} on FormatException catch (e) {
throw new ArchivePublisherException('Unable to parse JSON metadata received from cloud: $e');
}
jsonData['current_$channelName'] = revision;
if (!jsonData.containsKey('releases')) {
jsonData['releases'] = <String, dynamic>{};
}
if (jsonData['releases'].containsKey(revision)) {
throw new ArchivePublisherException(
'Revision $revision already exists in metadata! Aborting.');
}
jsonData['releases'][revision] = metadata;
final Directory localTempDir = tempDir ?? Directory.systemTemp.createTempSync('flutter_');
final File tempFile = new File(path.join(localTempDir.absolute.path, 'releases.json'));
const JsonEncoder encoder = const JsonEncoder.withIndent(' ');
tempFile.writeAsStringSync(encoder.convert(jsonData));
_cloudCopy(tempFile.absolute.path, metadataGsPath);
if (tempDir == null) {
localTempDir.delete(recursive: true);
}
}
String _getArchiveSuffix(String platform) {
switch (platform) {
case 'linux':
case 'mac':
return '.tar.xz';
case 'win':
return '.zip';
default:
assert(false, 'platform $platform not recognized.');
return null;
}
}
String _builtArchivePath(String platform) {
final String shortRevision = revision.substring(0, revision.length > 10 ? 10 : revision.length);
final String archivePathBase = '/flutter/$revision/$archivePrefix';
final String suffix = _getArchiveSuffix(platform);
return '$archivePathBase${platform}_$shortRevision$suffix';
}
String _destinationArchivePath(String platform) {
final String archivePathBase = '/$channelName/$platform/$archivePrefix';
final String suffix = _getArchiveSuffix(platform);
return '$archivePathBase${platform}_$version-$channelName$suffix';
}
ProcessResult _runGsUtil(List<String> args) {
return processManager.runSync(<String>['gsutil']..addAll(args));
}
void _cloudCopy(String src, String dest) {
final ProcessResult result = _runGsUtil(<String>['cp', src, dest]);
if (result.exitCode != 0) {
throw new ArchivePublisherException('GSUtil copy command failed: ${result.stderr}', result);
}
}
}
...@@ -10,7 +10,6 @@ ...@@ -10,7 +10,6 @@
import 'dart:io'; import 'dart:io';
import 'package:args/args.dart'; import 'package:args/args.dart';
import 'archive_publisher.dart';
const String kIncrement = 'increment'; const String kIncrement = 'increment';
const String kX = 'x'; const String kX = 'x';
...@@ -114,19 +113,6 @@ void main(List<String> args) { ...@@ -114,19 +113,6 @@ void main(List<String> args) {
final String hash = getGitOutput('rev-parse HEAD', 'Get git hash for $commit'); final String hash = getGitOutput('rev-parse HEAD', 'Get git hash for $commit');
final ArchivePublisher publisher = new ArchivePublisher(hash, version, Channel.dev);
// Check for access early so that we don't try to publish things if the
// user doesn't have access to the metadata file.
try {
publisher.checkForGSUtilAccess();
} on ArchivePublisherException {
print('You do not appear to have the credentials required to update the archive links.');
print('Make sure you have "gsutil" installed, then run "gsutil config".');
print('Talk to @gspencergoog for details on which project to use.');
exit(1);
}
runGit('tag v$version', 'tag the commit with the version label'); runGit('tag v$version', 'tag the commit with the version label');
// PROMPT // PROMPT
...@@ -140,17 +126,6 @@ void main(List<String> args) { ...@@ -140,17 +126,6 @@ void main(List<String> args) {
exit(0); exit(0);
} }
// Publish the archive before pushing the tag so that if something fails in
// the publish step, we can clean up.
try {
publisher.publishArchive();
} on ArchivePublisherException catch (e) {
print('Archive publishing failed.\n$e');
runGit('tag -d v$version', 'remove the tag that was not published');
print('The dev roll has been aborted.');
exit(1);
}
runGit('push upstream v$version', 'publish the version'); runGit('push upstream v$version', 'publish the version');
runGit('push upstream HEAD:dev', 'land the new version on the "dev" branch'); runGit('push upstream HEAD:dev', 'land the new version on the "dev" branch');
print('Flutter version $version has been rolled to the "dev" channel!'); print('Flutter version $version has been rolled to the "dev" channel!');
......
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:test/test.dart';
import 'package:path/path.dart' as path;
import '../lib/archive_publisher.dart';
import 'fake_process_manager.dart';
void main() {
group('ArchivePublisher', () {
final List<String> emptyStdout = <String>[''];
FakeProcessManager processManager;
Directory tempDir;
setUp(() async {
processManager = new FakeProcessManager();
tempDir = await Directory.systemTemp.createTemp('flutter_');
});
tearDown(() async {
// On Windows, the directory is locked and not able to be deleted, because it is a
// temporary directory. So we just leave some (very small, because we're not actually
// building archives here) trash around to be deleted at the next reboot.
if (!Platform.isWindows) {
await tempDir.delete(recursive: true);
}
});
test('calls the right processes', () {
final Map<String, List<String>> calls = <String, List<String>>{
'gsutil acl get gs://flutter_infra/releases/releases.json': emptyStdout,
'gsutil cp gs://flutter_infra/flutter/deadbeef/flutter_linux_deadbeef.tar.xz '
'gs://flutter_infra/releases/dev/linux/flutter_linux_1.2.3-dev.tar.xz': emptyStdout,
'gsutil cp gs://flutter_infra/flutter/deadbeef/flutter_mac_deadbeef.tar.xz '
'gs://flutter_infra/releases/dev/mac/flutter_mac_1.2.3-dev.tar.xz': emptyStdout,
'gsutil cp gs://flutter_infra/flutter/deadbeef/flutter_win_deadbeef.zip '
'gs://flutter_infra/releases/dev/win/flutter_win_1.2.3-dev.zip': emptyStdout,
'gsutil cat gs://flutter_infra/releases/releases.json': <String>[
'''{
"base_url": "https://storage.googleapis.com/flutter_infra/releases",
"current_beta": "6da8ec6bd0c4801b80d666869e4069698561c043",
"current_dev": "f88c60b38c3a5ef92115d24e3da4175b4890daba",
"releases": {
"6da8ec6bd0c4801b80d666869e4069698561c043": {
"linux_archive": "beta/linux/flutter_linux_0.21.0-beta.tar.xz",
"mac_archive": "beta/mac/flutter_mac_0.21.0-beta.tar.xz",
"windows_archive": "beta/win/flutter_win_0.21.0-beta.tar.xz",
"release_date": "2017-12-19T10:30:00,847287019-08:00",
"release_notes": "beta/release_notes_0.21.0-beta.html",
"version": "0.21.0-beta"
},
"f88c60b38c3a5ef92115d24e3da4175b4890daba": {
"linux_archive": "dev/linux/flutter_linux_0.22.0-dev.tar.xz",
"mac_archive": "dev/mac/flutter_mac_0.22.0-dev.tar.xz",
"windows_archive": "dev/win/flutter_win_0.22.0-dev.tar.xz",
"release_date": "2018-01-19T13:30:09,728487019-08:00",
"release_notes": "dev/release_notes_0.22.0-dev.html",
"version": "0.22.0-dev"
}
}
}
'''],
'gsutil cp ${tempDir.path}/releases.json gs://flutter_infra/releases/releases.json':
emptyStdout,
};
processManager.setResults(calls);
new ArchivePublisher('deadbeef', '1.2.3', Channel.dev,
processManager: processManager, tempDir: tempDir)
..publishArchive();
processManager.verifyCalls(calls.keys);
final File outputFile = new File(path.join(tempDir.path, 'releases.json'));
expect(outputFile.existsSync(), isTrue);
final String contents = outputFile.readAsStringSync();
expect(contents, contains('"current_dev": "deadbeef"'));
expect(contents, contains('"deadbeef": {'));
});
});
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment