Skip to content

F
Front-End
Unverified Commit 8a2df396 authored by Greg Spencer's avatar Greg Spencer Committed by GitHub
Browse files
Options

Create packages only for release builds, and publish when created. (#14476) 

This changes the publishing of archives so that it happens on the chrome_infra bots when they build a packaged branch instead of as part of the dev_roll process.

It uses the tagged version in the branch, and leaves the git repo that it clones checked out on the branch and hash used to build the package.

It updates metadata located at gs://flutter_infra/releases/releases_.json (where is one of macos, linux, or windows) once published, since it would be complex to do the proper locking to keep them all in one shared .json file safely.

A separate [change to the chrome_infra bots](https://chromium-review.googlesource.com/c/chromium/tools/build/+/902823) was made to instruct them to build packaged for the dev, beta, and release branches (but not master anymore).
parent 4b878dc6
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 694 additions and 665 deletions
dev/bots/prepare_package.dart
View file @ 8a2df396
This diff is collapsed.
dev/tools/test/fake_process_manager.dart → dev/bots/test/fake_process_manager.dart
View file @ 8a2df396
......@@ -29,111 +29,88 @@ class FakeProcessManager extends Mock implements ProcessManager {
/// The list of results that will be sent back, organized by the command line
/// that will produce them. Each command line has a list of returned stdout
/// output that will be returned on each successive call.
Map<String, List<ProcessResult>> fakeResults = <String, List<ProcessResult>>{};
Map<String, List<ProcessResult>> _fakeResults = <String, List<ProcessResult>>{};
Map<String, List<ProcessResult>> get fakeResults => _fakeResults;
set fakeResults(Map<String, List<ProcessResult>> value) {
_fakeResults = <String, List<ProcessResult>>{};
for (String key in value.keys) {
_fakeResults[key] = <ProcessResult>[]
..addAll(value[key] ?? <ProcessResult>[new ProcessResult(0, 0, '', '')]);
}
}
/// The list of invocations that occurred, in the order they occurred.
List<Invocation> invocations = <Invocation>[];
/// Verify that the given command lines were called, in the given order.
/// Verify that the given command lines were called, in the given order, and that the
/// parameters were in the same order.
void verifyCalls(List<String> calls) {
int index = 0;
expect(invocations.length, equals(calls.length));
for (String call in calls) {
expect(call.split(' '), orderedEquals(invocations[index].positionalArguments[0]));
index++;
}
expect(invocations.length, equals(calls.length));
}
/// Sets the list of results that will be returned from each successive call.
void setResults(Map<String, List<String>> results) {
final Map<String, List<ProcessResult>> resultCodeUnits = <String, List<ProcessResult>>{};
for (String key in results.keys) {
resultCodeUnits[key] =
results[key].map((String result) => new ProcessResult(0, 0, result, '')).toList();
}
fakeResults = resultCodeUnits;
}
ProcessResult _popResult(String key) {
ProcessResult _popResult(List<String> command) {
final String key = command.join(' ');
expect(fakeResults, isNotEmpty);
expect(fakeResults, contains(key));
expect(fakeResults[key], isNotEmpty);
return fakeResults[key].removeAt(0);
}
FakeProcess _popProcess(String key) =>
new FakeProcess(_popResult(key), stdinResults: stdinResults);
FakeProcess _popProcess(List<String> command) =>
new FakeProcess(_popResult(command), stdinResults: stdinResults);
Future<Process> _nextProcess(Invocation invocation) async {
invocations.add(invocation);
return new Future<Process>.value(_popProcess(invocation.positionalArguments[0].join(' ')));
return new Future<Process>.value(_popProcess(invocation.positionalArguments[0]));
}
ProcessResult _nextResultSync(Invocation invocation) {
invocations.add(invocation);
return _popResult(invocation.positionalArguments[0].join(' '));
return _popResult(invocation.positionalArguments[0]);
}
Future<ProcessResult> _nextResult(Invocation invocation) async {
invocations.add(invocation);
return new Future<ProcessResult>.value(_popResult(invocation.positionalArguments[0].join(' ')));
return new Future<ProcessResult>.value(_popResult(invocation.positionalArguments[0]));
}
void _setupMock() {
// Note that not all possible types of invocations are covered here, just the ones
// expected to be called.
// TODO(gspencer): make this more general so that any call will be captured.
when(
start(
typed(captureAny),
environment: typed(captureAny, named: 'environment'),
workingDirectory: typed(captureAny, named: 'workingDirectory'),
),
).thenAnswer(_nextProcess);
when(
start(
typed(captureAny),
),
).thenAnswer(_nextProcess);
when(
run(
typed(captureAny),
environment: typed(captureAny, named: 'environment'),
workingDirectory: typed(captureAny, named: 'workingDirectory'),
),
).thenAnswer(_nextResult);
when(
run(
typed(captureAny),
),
).thenAnswer(_nextResult);
when(
runSync(
typed(captureAny),
environment: typed(captureAny, named: 'environment'),
workingDirectory: typed(captureAny, named: 'workingDirectory'),
),
).thenAnswer(_nextResultSync);
when(
runSync(
typed(captureAny),
),
).thenAnswer(_nextResultSync);
when(start(
typed(captureAny),
environment: typed(captureAny, named: 'environment'),
workingDirectory: typed(captureAny, named: 'workingDirectory'),
)).thenAnswer(_nextProcess);
when(start(typed(captureAny))).thenAnswer(_nextProcess);
when(run(
typed(captureAny),
environment: typed(captureAny, named: 'environment'),
workingDirectory: typed(captureAny, named: 'workingDirectory'),
)).thenAnswer(_nextResult);
when(run(typed(captureAny))).thenAnswer(_nextResult);
when(runSync(
typed(captureAny),
environment: typed(captureAny, named: 'environment'),
workingDirectory: typed(captureAny, named: 'workingDirectory')
)).thenAnswer(_nextResultSync);
when(runSync(typed(captureAny))).thenAnswer(_nextResultSync);
when(killPid(typed(captureAny), typed(captureAny))).thenReturn(true);
when(
canRun(captureAny,
workingDirectory: typed(
captureAny,
named: 'workingDirectory',
)),
).thenReturn(true);
when(canRun(captureAny, workingDirectory: typed(captureAny, named: 'workingDirectory')))
.thenReturn(true);
}
}
......@@ -190,9 +167,11 @@ class StringStreamConsumer implements StreamConsumer<List<int>> {
Future<dynamic> addStream(Stream<List<int>> value) {
streams.add(value);
completers.add(new Completer<dynamic>());
subscriptions.add(value.listen((List<int> data) {
sendString(utf8.decode(data));
}));
subscriptions.add(
value.listen((List<int> data) {
sendString(utf8.decode(data));
}),
);
subscriptions.last.onDone(() => completers.last.complete(null));
return new Future<dynamic>.value(null);
}
......
dev/tools/test/fake_process_manager_test.dart → dev/bots/test/fake_process_manager_test.dart
View file @ 8a2df396
......@@ -25,11 +25,15 @@ void main() {
tearDown(() async {});
test('start works', () async {
final Map<String, List<String>> calls = <String, List<String>>{
'gsutil acl get gs://flutter_infra/releases/releases.json': <String>['output1'],
'gsutil cat gs://flutter_infra/releases/releases.json': <String>['test'],
final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
'gsutil acl get gs://flutter_infra/releases/releases.json': <ProcessResult>[
new ProcessResult(0, 0, 'output1', '')
],
'gsutil cat gs://flutter_infra/releases/releases.json': <ProcessResult>[
new ProcessResult(0, 0, 'output2', '')
],
};
processManager.setResults(calls);
processManager.fakeResults = calls;
for (String key in calls.keys) {
final Process process = await processManager.start(key.split(' '));
String output = '';
......@@ -37,56 +41,68 @@ void main() {
output += utf8.decode(item);
});
await process.exitCode;
expect(output, equals(calls[key][0]));
expect(output, equals(calls[key][0].stdout));
}
processManager.verifyCalls(calls.keys);
processManager.verifyCalls(calls.keys.toList());
});
test('run works', () async {
final Map<String, List<String>> calls = <String, List<String>>{
'gsutil acl get gs://flutter_infra/releases/releases.json': <String>['output1'],
'gsutil cat gs://flutter_infra/releases/releases.json': <String>['test'],
final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
'gsutil acl get gs://flutter_infra/releases/releases.json': <ProcessResult>[
new ProcessResult(0, 0, 'output1', '')
],
'gsutil cat gs://flutter_infra/releases/releases.json': <ProcessResult>[
new ProcessResult(0, 0, 'output2', '')
],
};
processManager.setResults(calls);
processManager.fakeResults = calls;
for (String key in calls.keys) {
final ProcessResult result = await processManager.run(key.split(' '));
expect(result.stdout, equals(calls[key][0]));
expect(result.stdout, equals(calls[key][0].stdout));
}
processManager.verifyCalls(calls.keys);
processManager.verifyCalls(calls.keys.toList());
});
test('runSync works', () async {
final Map<String, List<String>> calls = <String, List<String>>{
'gsutil acl get gs://flutter_infra/releases/releases.json': <String>['output1'],
'gsutil cat gs://flutter_infra/releases/releases.json': <String>['test'],
final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
'gsutil acl get gs://flutter_infra/releases/releases.json': <ProcessResult>[
new ProcessResult(0, 0, 'output1', '')
],
'gsutil cat gs://flutter_infra/releases/releases.json': <ProcessResult>[
new ProcessResult(0, 0, 'output2', '')
],
};
processManager.setResults(calls);
processManager.fakeResults = calls;
for (String key in calls.keys) {
final ProcessResult result = processManager.runSync(key.split(' '));
expect(result.stdout, equals(calls[key][0]));
expect(result.stdout, equals(calls[key][0].stdout));
}
processManager.verifyCalls(calls.keys);
processManager.verifyCalls(calls.keys.toList());
});
test('captures stdin', () async {
final Map<String, List<String>> calls = <String, List<String>>{
'gsutil acl get gs://flutter_infra/releases/releases.json': <String>['output1'],
'gsutil cat gs://flutter_infra/releases/releases.json': <String>['test'],
final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
'gsutil acl get gs://flutter_infra/releases/releases.json': <ProcessResult>[
new ProcessResult(0, 0, 'output1', '')
],
'gsutil cat gs://flutter_infra/releases/releases.json': <ProcessResult>[
new ProcessResult(0, 0, 'output2', '')
],
};
processManager.setResults(calls);
processManager.fakeResults = calls;
for (String key in calls.keys) {
final Process process = await processManager.start(key.split(' '));
String output = '';
process.stdout.listen((List<int> item) {
output += utf8.decode(item);
});
final String testInput = '${calls[key][0]} input';
final String testInput = '${calls[key][0].stdout} input';
process.stdin.add(testInput.codeUnits);
await process.exitCode;
expect(output, equals(calls[key][0]));
expect(output, equals(calls[key][0].stdout));
expect(stdinCaptured.last, equals(testInput));
}
processManager.verifyCalls(calls.keys);
processManager.verifyCalls(calls.keys.toList());
});
});
}
dev/bots/test/prepare_package_test.dart
View file @ 8a2df396
This diff is collapsed.
dev/tools/lib/archive_publisher.dart deleted 100644 → 0
View file @ 4b878dc6
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'dart:io';
import 'package:path/path.dart' as path;
import 'package:process/process.dart';
class ArchivePublisherException implements Exception {
ArchivePublisherException(this.message, [this.result]);
final String message;
final ProcessResult result;
@override
String toString() {
String output = 'ArchivePublisherException';
if (message != null) {
output += ': $message';
}
final String stderr = result?.stderr ?? '';
if (stderr.isNotEmpty) {
output += ':\n${result.stderr}';
}
return output;
}
}
enum Channel { dev, beta }
/// Publishes the archive created for a particular version and git hash to
/// the releases directory on cloud storage, and updates the metadata for
/// releases.
///
/// See https://github.com/flutter/flutter/wiki/Release-process for more
/// information on the release process.
class ArchivePublisher {
ArchivePublisher(
this.revision,
this.version,
this.channel, {
this.processManager = const LocalProcessManager(),
this.tempDir,
}) : assert(revision.length == 40, 'Git hash must be 40 characters long (i.e. the entire hash).');
/// A git hash describing the revision to publish. It should be the complete
/// hash, not just a prefix.
final String revision;
/// A version number for the release (e.g. "1.2.3").
final String version;
/// The channel to publish to.
// TODO(gspencer): support Channel.beta: it is currently unimplemented.
final Channel channel;
/// Get the name of the channel as a string.
String get channelName {
switch (channel) {
case Channel.beta:
return 'beta';
case Channel.dev:
default:
return 'dev';
}
}
/// The process manager to use for invoking commands. Typically only
/// used for testing purposes.
final ProcessManager processManager;
/// The temporary directory used for this publisher. If not set, one will
/// be created, used, and then removed automatically. If set, it will not be
/// deleted when done: that is left to the caller. Typically used by tests.
Directory tempDir;
static String gsBase = 'gs://flutter_infra';
static String releaseFolder = '/releases';
static String baseUrl = 'https://storage.googleapis.com/flutter_infra';
static String archivePrefix = 'flutter_';
static String releaseNotesPrefix = 'release_notes_';
final String metadataGsPath = '$gsBase$releaseFolder/releases.json';
/// Publishes the archive for the given constructor parameters.
bool publishArchive() {
assert(channel == Channel.dev, 'Channel must be dev (beta not yet supported)');
final List<String> platforms = <String>['linux', 'mac', 'win'];
final Map<String, String> metadata = <String, String>{};
for (String platform in platforms) {
final String src = _builtArchivePath(platform);
final String dest = _destinationArchivePath(platform);
final String srcGsPath = '$gsBase$src';
final String destGsPath = '$gsBase$releaseFolder$dest';
_cloudCopy(srcGsPath, destGsPath);
metadata['${platform}_archive'] = '$channelName/$platform$dest';
}
metadata['release_date'] = new DateTime.now().toUtc().toIso8601String();
metadata['version'] = version;
_updateMetadata(metadata);
return true;
}
/// Checks to make sure the user has access to the Google Storage bucket
/// required to publish. Will throw an [ArchivePublisherException] if not.
void checkForGSUtilAccess() {
// Fetching ACLs requires FULL_CONTROL access.
final ProcessResult result = _runGsUtil(<String>['acl', 'get', metadataGsPath]);
if (result.exitCode != 0) {
throw new ArchivePublisherException(
'GSUtil cannot get ACLs for metadata file $metadataGsPath',
result,
);
}
}
void _updateMetadata(Map<String, String> metadata) {
final ProcessResult result = _runGsUtil(<String>['cat', metadataGsPath]);
if (result.exitCode != 0) {
throw new ArchivePublisherException(
'Unable to get existing metadata at $metadataGsPath', result);
}
final String currentMetadata = result.stdout;
if (currentMetadata.isEmpty) {
throw new ArchivePublisherException('Empty metadata received from server', result);
}
Map<String, dynamic> jsonData;
try {
jsonData = json.decode(currentMetadata);
} on FormatException catch (e) {
throw new ArchivePublisherException('Unable to parse JSON metadata received from cloud: $e');
}
jsonData['current_$channelName'] = revision;
if (!jsonData.containsKey('releases')) {
jsonData['releases'] = <String, dynamic>{};
}
if (jsonData['releases'].containsKey(revision)) {
throw new ArchivePublisherException(
'Revision $revision already exists in metadata! Aborting.');
}
jsonData['releases'][revision] = metadata;
final Directory localTempDir = tempDir ?? Directory.systemTemp.createTempSync('flutter_');
final File tempFile = new File(path.join(localTempDir.absolute.path, 'releases.json'));
const JsonEncoder encoder = const JsonEncoder.withIndent(' ');
tempFile.writeAsStringSync(encoder.convert(jsonData));
_cloudCopy(tempFile.absolute.path, metadataGsPath);
if (tempDir == null) {
localTempDir.delete(recursive: true);
}
}
String _getArchiveSuffix(String platform) {
switch (platform) {
case 'linux':
case 'mac':
return '.tar.xz';
case 'win':
return '.zip';
default:
assert(false, 'platform $platform not recognized.');
return null;
}
}
String _builtArchivePath(String platform) {
final String shortRevision = revision.substring(0, revision.length > 10 ? 10 : revision.length);
final String archivePathBase = '/flutter/$revision/$archivePrefix';
final String suffix = _getArchiveSuffix(platform);
return '$archivePathBase${platform}_$shortRevision$suffix';
}
String _destinationArchivePath(String platform) {
final String archivePathBase = '/$channelName/$platform/$archivePrefix';
final String suffix = _getArchiveSuffix(platform);
return '$archivePathBase${platform}_$version-$channelName$suffix';
}
ProcessResult _runGsUtil(List<String> args) {
return processManager.runSync(<String>['gsutil']..addAll(args));
}
void _cloudCopy(String src, String dest) {
final ProcessResult result = _runGsUtil(<String>['cp', src, dest]);
if (result.exitCode != 0) {
throw new ArchivePublisherException('GSUtil copy command failed: ${result.stderr}', result);
}
}
}
dev/tools/lib/roll_dev.dart
View file @ 8a2df396
......@@ -10,7 +10,6 @@
import 'dart:io';
import 'package:args/args.dart';
import 'archive_publisher.dart';
const String kIncrement = 'increment';
const String kX = 'x';
......@@ -114,19 +113,6 @@ void main(List<String> args) {
final String hash = getGitOutput('rev-parse HEAD', 'Get git hash for $commit');
final ArchivePublisher publisher = new ArchivePublisher(hash, version, Channel.dev);
// Check for access early so that we don't try to publish things if the
// user doesn't have access to the metadata file.
try {
publisher.checkForGSUtilAccess();
} on ArchivePublisherException {
print('You do not appear to have the credentials required to update the archive links.');
print('Make sure you have "gsutil" installed, then run "gsutil config".');
print('Talk to @gspencergoog for details on which project to use.');
exit(1);
}
runGit('tag v$version', 'tag the commit with the version label');
// PROMPT
......@@ -140,17 +126,6 @@ void main(List<String> args) {
exit(0);
}
// Publish the archive before pushing the tag so that if something fails in
// the publish step, we can clean up.
try {
publisher.publishArchive();
} on ArchivePublisherException catch (e) {
print('Archive publishing failed.\n$e');
runGit('tag -d v$version', 'remove the tag that was not published');
print('The dev roll has been aborted.');
exit(1);
}
runGit('push upstream v$version', 'publish the version');
runGit('push upstream HEAD:dev', 'land the new version on the "dev" branch');
print('Flutter version $version has been rolled to the "dev" channel!');
......
dev/tools/test/archive_publisher_test.dart deleted 100644 → 0
View file @ 4b878dc6
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:test/test.dart';
import 'package:path/path.dart' as path;
import '../lib/archive_publisher.dart';
import 'fake_process_manager.dart';
void main() {
group('ArchivePublisher', () {
final List<String> emptyStdout = <String>[''];
FakeProcessManager processManager;
Directory tempDir;
setUp(() async {
processManager = new FakeProcessManager();
tempDir = await Directory.systemTemp.createTemp('flutter_');
});
tearDown(() async {
// On Windows, the directory is locked and not able to be deleted, because it is a
// temporary directory. So we just leave some (very small, because we're not actually
// building archives here) trash around to be deleted at the next reboot.
if (!Platform.isWindows) {
await tempDir.delete(recursive: true);
}
});
test('calls the right processes', () {
final Map<String, List<String>> calls = <String, List<String>>{
'gsutil acl get gs://flutter_infra/releases/releases.json': emptyStdout,
'gsutil cp gs://flutter_infra/flutter/deadbeef/flutter_linux_deadbeef.tar.xz '
'gs://flutter_infra/releases/dev/linux/flutter_linux_1.2.3-dev.tar.xz': emptyStdout,
'gsutil cp gs://flutter_infra/flutter/deadbeef/flutter_mac_deadbeef.tar.xz '
'gs://flutter_infra/releases/dev/mac/flutter_mac_1.2.3-dev.tar.xz': emptyStdout,
'gsutil cp gs://flutter_infra/flutter/deadbeef/flutter_win_deadbeef.zip '
'gs://flutter_infra/releases/dev/win/flutter_win_1.2.3-dev.zip': emptyStdout,
'gsutil cat gs://flutter_infra/releases/releases.json': <String>[
'''{
"base_url": "https://storage.googleapis.com/flutter_infra/releases",
"current_beta": "6da8ec6bd0c4801b80d666869e4069698561c043",
"current_dev": "f88c60b38c3a5ef92115d24e3da4175b4890daba",
"releases": {
"6da8ec6bd0c4801b80d666869e4069698561c043": {
"linux_archive": "beta/linux/flutter_linux_0.21.0-beta.tar.xz",
"mac_archive": "beta/mac/flutter_mac_0.21.0-beta.tar.xz",
"windows_archive": "beta/win/flutter_win_0.21.0-beta.tar.xz",
"release_date": "2017-12-19T10:30:00,847287019-08:00",
"release_notes": "beta/release_notes_0.21.0-beta.html",
"version": "0.21.0-beta"
},
"f88c60b38c3a5ef92115d24e3da4175b4890daba": {
"linux_archive": "dev/linux/flutter_linux_0.22.0-dev.tar.xz",
"mac_archive": "dev/mac/flutter_mac_0.22.0-dev.tar.xz",
"windows_archive": "dev/win/flutter_win_0.22.0-dev.tar.xz",
"release_date": "2018-01-19T13:30:09,728487019-08:00",
"release_notes": "dev/release_notes_0.22.0-dev.html",
"version": "0.22.0-dev"
}
}
}
'''],
'gsutil cp ${tempDir.path}/releases.json gs://flutter_infra/releases/releases.json':
emptyStdout,
};
processManager.setResults(calls);
new ArchivePublisher('deadbeef', '1.2.3', Channel.dev,
processManager: processManager, tempDir: tempDir)
..publishArchive();
processManager.verifyCalls(calls.keys);
final File outputFile = new File(path.join(tempDir.path, 'releases.json'));
expect(outputFile.existsSync(), isTrue);
final String contents = outputFile.readAsStringSync();
expect(contents, contains('"current_dev": "deadbeef"'));
expect(contents, contains('"deadbeef": {'));
});
});
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab