Unverified Commit a82255b5 authored by Christopher Fujino's avatar Christopher Fujino Committed by GitHub

[flutter_conductor] updates (#86452)

parent 6406c6d4
......@@ -37,4 +37,7 @@ BIN_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)"
REPO_DIR="$BIN_DIR/../../.."
DART_BIN="$REPO_DIR/bin/dart"
# Ensure pub get has been run in the repo before running the conductor
(cd "$REPO_DIR/dev/conductor"; $DART_BIN pub get 1>&2)
"$DART_BIN" --enable-asserts "$REPO_DIR/dev/conductor/bin/conductor.dart" "$@"
......@@ -86,10 +86,9 @@ String? getValueFromEnvOrArgs(
ArgResults argResults,
Map<String, String> env, {
bool allowNull = false,
}
) {
}) {
final String envName = fromArgToEnvName(name);
if (env[envName] != null ) {
if (env[envName] != null) {
return env[envName];
}
final String? argValue = argResults[name] as String?;
......@@ -139,3 +138,13 @@ List<String> getValuesFromEnvOrArgs(
String fromArgToEnvName(String argName) {
return argName.toUpperCase().replaceAll(r'-', r'_');
}
/// Return a web link for the user to open a new PR.
String getNewPrLink({
required String userName,
required String repoName,
required String candidateBranch,
required String workingBranch,
}) {
return 'https://github.com/flutter/$repoName/compare/$candidateBranch...$userName:$workingBranch?expand=1';
}
......@@ -102,7 +102,7 @@ void runNext({
}
}
if (state.engine.cherrypicks.isEmpty && state.engine.dartRevision.isEmpty) {
if (!requiresEnginePR(state)) {
stdio.printStatus(
'This release has no engine cherrypicks. No Engine PR is necessary.\n',
);
......@@ -110,11 +110,9 @@ void runNext({
}
if (unappliedCherrypicks.isEmpty) {
stdio.printStatus('All engine cherrypicks have been auto-applied by '
'the conductor.\n');
stdio.printStatus('All engine cherrypicks have been auto-applied by the conductor.\n');
} else {
stdio.printStatus(
'There were ${unappliedCherrypicks.length} cherrypicks that were not auto-applied.');
stdio.printStatus('There were ${unappliedCherrypicks.length} cherrypicks that were not auto-applied.');
stdio.printStatus('These must be applied manually in the directory '
'${state.engine.checkoutPath} before proceeding.\n');
}
......@@ -189,13 +187,15 @@ void runNext({
break;
}
}
final EngineRepository engine = EngineRepository(
checkouts,
initialRef: state.engine.candidateBranch,
upstreamRemote: Remote(
final Remote engineUpstreamRemote = Remote(
name: RemoteName.upstream,
url: state.engine.upstream.url,
),
);
final EngineRepository engine = EngineRepository(
checkouts,
// We explicitly want to check out the merged version from upstream
initialRef: '${engineUpstreamRemote.name}/${state.engine.candidateBranch}',
upstreamRemote: engineUpstreamRemote,
previousCheckoutLocation: state.engine.checkoutPath,
);
......@@ -215,7 +215,9 @@ void runNext({
stdio.printStatus('Rolling new engine hash $engineRevision to framework checkout...');
framework.updateEngineRevision(engineRevision);
framework.commit('Update Engine revision to $engineRevision for ${state.releaseChannel} release ${state.releaseVersion}', addFirst: true);
framework.commit(
'Update Engine revision to $engineRevision for ${state.releaseChannel} release ${state.releaseVersion}',
addFirst: true);
if (state.framework.cherrypicks.isEmpty) {
stdio.printStatus(
......@@ -262,7 +264,8 @@ void runNext({
);
final FrameworkRepository framework = FrameworkRepository(
checkouts,
initialRef: state.framework.candidateBranch,
// We explicitly want to check out the merged version from upstream
initialRef: '${upstream.name}/${state.framework.candidateBranch}',
upstreamRemote: upstream,
previousCheckoutLocation: state.framework.checkoutPath,
);
......@@ -288,7 +291,8 @@ void runNext({
);
final FrameworkRepository framework = FrameworkRepository(
checkouts,
initialRef: state.framework.candidateBranch,
// We explicitly want to check out the merged version from upstream
initialRef: '${upstream.name}/${state.framework.candidateBranch}',
upstreamRemote: upstream,
previousCheckoutLocation: state.framework.checkoutPath,
);
......
......@@ -403,6 +403,8 @@ class ConductorState extends $pb.GeneratedMessage {
enumValues: ReleasePhase.values)
..aOS(10, const $core.bool.fromEnvironment('protobuf.omit_field_names') ? '' : 'conductorVersion',
protoName: 'conductorVersion')
..aOS(11, const $core.bool.fromEnvironment('protobuf.omit_field_names') ? '' : 'incrementLevel',
protoName: 'incrementLevel')
..hasRequiredFields = false;
ConductorState._() : super();
......@@ -416,6 +418,7 @@ class ConductorState extends $pb.GeneratedMessage {
$core.Iterable<$core.String>? logs,
ReleasePhase? currentPhase,
$core.String? conductorVersion,
$core.String? incrementLevel,
}) {
final _result = create();
if (releaseChannel != null) {
......@@ -445,6 +448,9 @@ class ConductorState extends $pb.GeneratedMessage {
if (conductorVersion != null) {
_result.conductorVersion = conductorVersion;
}
if (incrementLevel != null) {
_result.incrementLevel = incrementLevel;
}
return _result;
}
factory ConductorState.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
......@@ -572,4 +578,16 @@ class ConductorState extends $pb.GeneratedMessage {
$core.bool hasConductorVersion() => $_has(8);
@$pb.TagNumber(10)
void clearConductorVersion() => clearField(10);
@$pb.TagNumber(11)
$core.String get incrementLevel => $_getSZ(9);
@$pb.TagNumber(11)
set incrementLevel($core.String v) {
$_setString(9, v);
}
@$pb.TagNumber(11)
$core.bool hasIncrementLevel() => $_has(9);
@$pb.TagNumber(11)
void clearIncrementLevel() => clearField(11);
}
......@@ -101,9 +101,10 @@ const ConductorState$json = const {
const {'1': 'logs', '3': 8, '4': 3, '5': 9, '10': 'logs'},
const {'1': 'currentPhase', '3': 9, '4': 1, '5': 14, '6': '.conductor_state.ReleasePhase', '10': 'currentPhase'},
const {'1': 'conductorVersion', '3': 10, '4': 1, '5': 9, '10': 'conductorVersion'},
const {'1': 'incrementLevel', '3': 11, '4': 1, '5': 9, '10': 'incrementLevel'},
],
};
/// Descriptor for `ConductorState`. Decode as a `google.protobuf.DescriptorProto`.
final $typed_data.Uint8List conductorStateDescriptor = $convert.base64Decode(
'Cg5Db25kdWN0b3JTdGF0ZRImCg5yZWxlYXNlQ2hhbm5lbBgBIAEoCVIOcmVsZWFzZUNoYW5uZWwSJgoOcmVsZWFzZVZlcnNpb24YAiABKAlSDnJlbGVhc2VWZXJzaW9uEjMKBmVuZ2luZRgEIAEoCzIbLmNvbmR1Y3Rvcl9zdGF0ZS5SZXBvc2l0b3J5UgZlbmdpbmUSOQoJZnJhbWV3b3JrGAUgASgLMhsuY29uZHVjdG9yX3N0YXRlLlJlcG9zaXRvcnlSCWZyYW1ld29yaxIgCgtjcmVhdGVkRGF0ZRgGIAEoA1ILY3JlYXRlZERhdGUSKAoPbGFzdFVwZGF0ZWREYXRlGAcgASgDUg9sYXN0VXBkYXRlZERhdGUSEgoEbG9ncxgIIAMoCVIEbG9ncxJBCgxjdXJyZW50UGhhc2UYCSABKA4yHS5jb25kdWN0b3Jfc3RhdGUuUmVsZWFzZVBoYXNlUgxjdXJyZW50UGhhc2USKgoQY29uZHVjdG9yVmVyc2lvbhgKIAEoCVIQY29uZHVjdG9yVmVyc2lvbg==');
'Cg5Db25kdWN0b3JTdGF0ZRImCg5yZWxlYXNlQ2hhbm5lbBgBIAEoCVIOcmVsZWFzZUNoYW5uZWwSJgoOcmVsZWFzZVZlcnNpb24YAiABKAlSDnJlbGVhc2VWZXJzaW9uEjMKBmVuZ2luZRgEIAEoCzIbLmNvbmR1Y3Rvcl9zdGF0ZS5SZXBvc2l0b3J5UgZlbmdpbmUSOQoJZnJhbWV3b3JrGAUgASgLMhsuY29uZHVjdG9yX3N0YXRlLlJlcG9zaXRvcnlSCWZyYW1ld29yaxIgCgtjcmVhdGVkRGF0ZRgGIAEoA1ILY3JlYXRlZERhdGUSKAoPbGFzdFVwZGF0ZWREYXRlGAcgASgDUg9sYXN0VXBkYXRlZERhdGUSEgoEbG9ncxgIIAMoCVIEbG9ncxJBCgxjdXJyZW50UGhhc2UYCSABKA4yHS5jb25kdWN0b3Jfc3RhdGUuUmVsZWFzZVBoYXNlUgxjdXJyZW50UGhhc2USKgoQY29uZHVjdG9yVmVyc2lvbhgKIAEoCVIQY29uZHVjdG9yVmVyc2lvbhImCg5pbmNyZW1lbnRMZXZlbBgLIAEoCVIOaW5jcmVtZW50TGV2ZWw=');
......@@ -111,4 +111,7 @@ message ConductorState {
// Commit hash of the Conductor tool.
string conductorVersion = 10;
// One of x, y, z, m, or n.
string incrementLevel = 11;
}
......@@ -74,8 +74,10 @@ abstract class Repository {
'Fetch ${upstreamRemote.name} to ensure we have latest refs',
workingDirectory: _checkoutDirectory!.path,
);
// Note: if [initialRef] is a remote ref the checkout will be left in a
// detached HEAD state.
git.run(
<String>['checkout', '${upstreamRemote.name}/$initialRef'],
<String>['checkout', initialRef!],
'Checking out initialRef $initialRef',
workingDirectory: _checkoutDirectory!.path,
);
......
......@@ -213,6 +213,7 @@ class StartCommand extends Command<void> {
state.releaseChannel = releaseChannel;
state.createdDate = unixDate;
state.lastUpdatedDate = unixDate;
state.incrementLevel = incrementLetter;
final EngineRepository engine = EngineRepository(
checkouts,
......
......@@ -43,10 +43,8 @@ String presentState(pb.ConductorState state) {
buffer.writeln('Release channel: ${state.releaseChannel}');
buffer.writeln('Release version: ${state.releaseVersion}');
buffer.writeln('');
buffer.writeln(
'Release started at: ${DateTime.fromMillisecondsSinceEpoch(state.createdDate.toInt())}');
buffer.writeln(
'Last updated at: ${DateTime.fromMillisecondsSinceEpoch(state.lastUpdatedDate.toInt())}');
buffer.writeln('Release started at: ${DateTime.fromMillisecondsSinceEpoch(state.createdDate.toInt())}');
buffer.writeln('Last updated at: ${DateTime.fromMillisecondsSinceEpoch(state.lastUpdatedDate.toInt())}');
buffer.writeln('');
buffer.writeln('Engine Repo');
buffer.writeln('\tCandidate branch: ${state.engine.candidateBranch}');
......@@ -128,15 +126,27 @@ String phaseInstructions(pb.ConductorState state) {
return <String>[
'You must now manually apply the following engine cherrypicks to the checkout',
'at ${state.engine.checkoutPath} in order:',
for (final pb.Cherrypick cherrypick in state.engine.cherrypicks)
'\t${cherrypick.trunkRevision}',
for (final pb.Cherrypick cherrypick in state.engine.cherrypicks) '\t${cherrypick.trunkRevision}',
'See $kReleaseDocumentationUrl for more information.',
].join('\n');
case ReleasePhase.CODESIGN_ENGINE_BINARIES:
if (!requiresEnginePR(state)) {
return 'You must now codesign the engine binaries for commit '
'${state.engine.startingGitHead}.';
}
// User's working branch was pushed to their mirror, but a PR needs to be
// opened on GitHub.
final String newPrLink = getNewPrLink(
userName: githubAccount(state.engine.mirror.url),
repoName: 'engine',
candidateBranch: state.engine.candidateBranch,
workingBranch: state.engine.workingBranch,
);
return <String>[
'You must verify pre-submit CI builds on your engine pull request are successful,',
'merge your pull request, validate post-submit CI, and then codesign the binaries ',
'on the merge commit.',
'Your working branch ${state.engine.workingBranch} was pushed to your mirror.',
'You must now open a pull request at $newPrLink, verify pre-submit CI',
'builds on your engine pull request are successful, merge your pull request,',
'validate post-submit CI, and then codesign the binaries on the merge commit.',
].join('\n');
case ReleasePhase.APPLY_FRAMEWORK_CHERRYPICKS:
final List<pb.Cherrypick> outstandingCherrypicks = state.framework.cherrypicks.where(
......@@ -144,17 +154,31 @@ String phaseInstructions(pb.ConductorState state) {
return cp.state == pb.CherrypickState.PENDING || cp.state == pb.CherrypickState.PENDING_WITH_CONFLICT;
},
).toList();
if (outstandingCherrypicks.isNotEmpty) {
return <String>[
'You must now manually apply the following framework cherrypicks to the checkout',
'at ${state.framework.checkoutPath} in order:',
for (final pb.Cherrypick cherrypick in outstandingCherrypicks)
'\t${cherrypick.trunkRevision}',
for (final pb.Cherrypick cherrypick in outstandingCherrypicks) '\t${cherrypick.trunkRevision}',
].join('\n');
}
return <String>['Either all cherrypicks have been auto-applied or there were none.'].join('\n');
case ReleasePhase.PUBLISH_VERSION:
if (!requiresFrameworkPR(state)) {
return 'Since there are no code changes in this release, no Framework '
'PR is necessary.';
}
final String newPrLink = getNewPrLink(
userName: githubAccount(state.framework.mirror.url),
repoName: 'framework',
candidateBranch: state.framework.candidateBranch,
workingBranch: state.framework.workingBranch,
);
return <String>[
'You must verify pre-submit CI builds on your framework pull request are successful,',
'merge your pull request, and validate post-submit CI. See $kReleaseDocumentationUrl,',
'for more information.',
'Your working branch ${state.framework.workingBranch} was pushed to your mirror.',
'You must now open a pull request at $newPrLink',
'verify pre-submit CI builds on your pull request are successful, merge your ',
'pull request, validate post-submit CI.',
].join('\n');
case ReleasePhase.PUBLISH_CHANNEL:
return 'Issue `conductor next` to publish your release to the release branch.';
......@@ -163,8 +187,36 @@ String phaseInstructions(pb.ConductorState state) {
case ReleasePhase.RELEASE_COMPLETED:
return 'This release has been completed.';
}
assert(false);
return ''; // For analyzer
// For analyzer
throw ConductorException('Unimplemented phase ${state.currentPhase}');
}
/// Regex pattern for git remote host URLs.
///
/// First group = git host (currently must be github.com)
/// Second group = account name
/// Third group = repo name
final RegExp githubRemotePattern =
RegExp(r'^(git@github\.com:|https?:\/\/github\.com\/)([a-zA-Z0-9_-]+)\/([a-zA-Z0-9_-]+)(\.git)?$');
/// Parses a Git remote URL and returns the account name.
///
/// Uses [githubRemotePattern].
String githubAccount(String remoteUrl) {
final String engineUrl = remoteUrl;
final RegExpMatch? match = githubRemotePattern.firstMatch(engineUrl);
if (match == null) {
throw ConductorException(
'Cannot determine the GitHub account from $engineUrl',
);
}
final String? accountName = match.group(2);
if (accountName == null || accountName.isEmpty) {
throw ConductorException(
'Cannot determine the GitHub account from $match',
);
}
return accountName;
}
/// Returns the next phase in the ReleasePhase enum.
......@@ -196,3 +248,38 @@ pb.ConductorState readStateFromFile(File file) {
);
return state;
}
/// This release will require a new Engine PR.
///
/// The logic is if there are engine cherrypicks that have not been abandoned OR
/// there is a new Dart revision, then return true, else false.
bool requiresEnginePR(pb.ConductorState state) {
final bool hasRequiredCherrypicks = state.engine.cherrypicks.any(
(pb.Cherrypick cp) => cp.state != pb.CherrypickState.ABANDONED,
);
if (hasRequiredCherrypicks) {
return true;
}
return state.engine.dartRevision.isNotEmpty;
}
/// This release will require a new Framework PR.
///
/// The logic is if there was an Engine PR OR there are framework cherrypicks
/// that have not been abandoned OR the increment level is 'm', then return
/// true, else false.
bool requiresFrameworkPR(pb.ConductorState state) {
if (requiresEnginePR(state)) {
return true;
}
final bool hasRequiredCherrypicks =
state.framework.cherrypicks.any((pb.Cherrypick cp) => cp.state != pb.CherrypickState.ABANDONED);
if (hasRequiredCherrypicks) {
return true;
}
if (state.incrementLevel == 'm') {
// requires an update to .ci.yaml
return true;
}
return false;
}
......@@ -89,6 +89,9 @@ void main() {
);
final pb.ConductorState state = pb.ConductorState(
currentPhase: ReleasePhase.APPLY_ENGINE_CHERRYPICKS,
engine: pb.Repository(
startingGitHead: revision1,
),
);
writeStateToFile(
fileSystem.file(stateFile),
......@@ -116,10 +119,13 @@ void main() {
expect(processManager, hasNoRemainingExpectations);
expect(finalState.currentPhase, ReleasePhase.CODESIGN_ENGINE_BINARIES);
expect(stdio.error, isEmpty);
expect(
stdio.stdout,
contains('You must now codesign the engine binaries for commit $revision1'));
});
test('confirms to stdout when all engine cherrypicks were auto-applied', () async {
const String remoteUrl = 'https://githost.com/org/repo.git';
const String remoteUrl = 'https://github.com/org/repo.git';
stdio.stdin.add('n');
final FakeProcessManager processManager = FakeProcessManager.empty();
final FakePlatform platform = FakePlatform(
......@@ -170,13 +176,14 @@ void main() {
});
test('updates lastPhase if user responds yes', () async {
const String remoteUrl = 'https://githost.com/org/repo.git';
const String remoteUrl = 'https://github.com/org/repo.git';
const String releaseChannel = 'dev';
stdio.stdin.add('y');
final FakeProcessManager processManager = FakeProcessManager.list(<FakeCommand>[
const FakeCommand(
command: <String>['git', 'fetch', 'upstream'],
),
const FakeCommand(command: <String>['git', 'checkout', 'upstream/$workingBranch']),
const FakeCommand(command: <String>['git', 'checkout', workingBranch]),
const FakeCommand(
command: <String>['git', 'rev-parse', 'HEAD'],
stdout: revision1,
......@@ -191,7 +198,9 @@ void main() {
pathSeparator: localPathSeparator,
);
final pb.ConductorState state = pb.ConductorState(
currentPhase: ReleasePhase.APPLY_ENGINE_CHERRYPICKS,
engine: pb.Repository(
candidateBranch: candidateBranch,
cherrypicks: <pb.Cherrypick>[
pb.Cherrypick(
trunkRevision: 'abc123',
......@@ -202,7 +211,7 @@ void main() {
upstream: pb.Remote(name: 'upstream', url: remoteUrl),
mirror: pb.Remote(name: 'mirror', url: remoteUrl),
),
currentPhase: ReleasePhase.APPLY_ENGINE_CHERRYPICKS,
releaseChannel: releaseChannel,
);
writeStateToFile(
fileSystem.file(stateFile),
......@@ -228,6 +237,9 @@ void main() {
);
expect(processManager, hasNoRemainingExpectations);
expect(
stdio.stdout,
contains('You must now open a pull request at https://github.com/flutter/engine/compare/flutter-1.2-candidate.3...org:cherrypicks-flutter-1.2-candidate.3?expand=1'));
expect(stdio.stdout, contains(
'Are you ready to push your engine branch to the repository $remoteUrl? (y/n) '));
expect(finalState.currentPhase, ReleasePhase.CODESIGN_ENGINE_BINARIES);
......@@ -332,9 +344,9 @@ void main() {
});
group('APPLY_FRAMEWORK_CHERRYPICKS to PUBLISH_VERSION', () {
const String mirrorRemoteUrl = 'https://githost.com/org/repo.git';
const String upstreamRemoteUrl = 'https://githost.com/mirror/repo.git';
const String engineUpstreamRemoteUrl = 'https://githost.com/mirror/engine.git';
const String mirrorRemoteUrl = 'https://github.com/org/repo.git';
const String upstreamRemoteUrl = 'https://github.com/mirror/repo.git';
const String engineUpstreamRemoteUrl = 'https://github.com/mirror/engine.git';
const String frameworkCheckoutPath = '$checkoutsParentDirectory/framework';
const String engineCheckoutPath = '$checkoutsParentDirectory/engine';
const String oldEngineVersion = '000000001';
......@@ -441,13 +453,14 @@ void main() {
stdio.stdin.add('n');
processManager.addCommands(const <FakeCommand>[
FakeCommand(command: <String>['git', 'fetch', 'upstream']),
// we want merged upstream commit, not local working commit
FakeCommand(command: <String>['git', 'checkout', 'upstream/$candidateBranch']),
FakeCommand(
command: <String>['git', 'rev-parse', 'HEAD'],
stdout: revision1,
),
FakeCommand(command: <String>['git', 'fetch', 'upstream']),
FakeCommand(command: <String>['git', 'checkout', 'upstream/$workingBranch']),
FakeCommand(command: <String>['git', 'checkout', workingBranch]),
FakeCommand(
command: <String>['git', 'rev-parse', 'HEAD'],
stdout: revision2,
......@@ -509,13 +522,14 @@ void main() {
stdio.stdin.add('n');
processManager.addCommands(const <FakeCommand>[
FakeCommand(command: <String>['git', 'fetch', 'upstream']),
// we want merged upstream commit, not local working commit
FakeCommand(command: <String>['git', 'checkout', 'upstream/$candidateBranch']),
FakeCommand(
command: <String>['git', 'rev-parse', 'HEAD'],
stdout: revision1,
),
FakeCommand(command: <String>['git', 'fetch', 'upstream']),
FakeCommand(command: <String>['git', 'checkout', 'upstream/$workingBranch']),
FakeCommand(command: <String>['git', 'checkout', workingBranch]),
FakeCommand(
command: <String>['git', 'rev-parse', 'HEAD'],
stdout: revision2,
......@@ -564,6 +578,7 @@ void main() {
processManager.addCommands(const <FakeCommand>[
// Engine repo
FakeCommand(command: <String>['git', 'fetch', 'upstream']),
// we want merged upstream commit, not local working commit
FakeCommand(command: <String>['git', 'checkout', 'upstream/$candidateBranch']),
FakeCommand(
command: <String>['git', 'rev-parse', 'HEAD'],
......@@ -571,7 +586,7 @@ void main() {
),
// Framework repo
FakeCommand(command: <String>['git', 'fetch', 'upstream']),
FakeCommand(command: <String>['git', 'checkout', 'upstream/$workingBranch']),
FakeCommand(command: <String>['git', 'checkout', workingBranch]),
FakeCommand(
command: <String>['git', 'rev-parse', 'HEAD'],
stdout: revision2,
......
......@@ -115,6 +115,7 @@ void main() {
const String nextDartRevision = 'f6c91128be6b77aef8351e1e3a9d07c85bc2e46e';
const String previousVersion = '1.2.0-1.0.pre';
const String nextVersion = '1.2.0-3.0.pre';
const String incrementLevel = 'm';
final Directory engine = fileSystem.directory(checkoutsParentDirectory)
.childDirectory('flutter_conductor_checkouts')
......@@ -261,7 +262,7 @@ void main() {
'--$kDartRevisionOption',
nextDartRevision,
'--$kIncrementOption',
'm',
incrementLevel,
]);
final File stateFile = fileSystem.file(stateFilePath);
......@@ -281,6 +282,7 @@ void main() {
expect(state.framework.startingGitHead, revision3);
expect(state.currentPhase, ReleasePhase.APPLY_ENGINE_CHERRYPICKS);
expect(state.conductorVersion, revision);
expect(state.incrementLevel, incrementLevel);
});
}, onPlatform: <String, dynamic>{
'windows': const Skip('Flutter Conductor only supported on macos/linux'),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment