Skip to content

F
Front-End
Unverified Commit fd0e223b authored by godofredoc's avatar godofredoc Committed by GitHub
Browse files
Options

Add Firebase tests to flutter dashboard. (#66677) 

This PR also enables firebase tests on presubmit and removes code that
won't be used anymore from test runner.

Bug:
  https://github.com/flutter/infra/pull/225
parent e6b9b5ef
Hide whitespace changes
Inline Side-by-side
Showing with 36 additions and 95 deletions
.cirrus.yml
View file @ fd0e223b
......@@ -4,17 +4,6 @@
# YAML anchors used to share fields between tasks.
# See https://confluence.atlassian.com/bitbucket/yaml-anchors-960154027.html
firebase_shard_template: &FIREBASE_SHARD_TEMPLATE
environment:
# Empirically, this shard runs in 20-25 minutes with just one CPU and 4G of RAM, as of
# October 2019. It does not seem to be sensitive to the number of CPUs or amount of RAM;
# doubling CPUs had no effect (mere seconds under 20 minutes), increasing RAM to 24G left it
# on the high end of the 20-25 minute range. (This makes sense, as it's just driving the
# Firebase test lab remotely.) Less than 4G of RAM made it go OOM.
CLOUDSDK_CORE_DISABLE_PROMPTS: 1
GCLOUD_FIREBASE_TESTLAB_KEY: ENCRYPTED[1c140257edc48f5578fa5a0e5038b84c8e53270c405efa5a8e35ea303a4e0d135853989f448f72136206de854d17fbec]
script:
- dart --enable-asserts ./dev/bots/test.dart
use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
......@@ -189,16 +178,6 @@ task:
- git clone https://github.com/flutter/tests.git bin/cache/pkg/tests
- dart --enable-asserts dev/customer_testing/run_tests.dart --skip-on-fetch-failure --skip-template bin/cache/pkg/tests/registry/*.test
# firebase_test_lab_tests are linux-only
- name: firebase_test_lab_tests-0-linux
<<: *FIREBASE_SHARD_TEMPLATE
- name: firebase_test_lab_tests-1-linux
<<: *FIREBASE_SHARD_TEMPLATE
- name: firebase_test_lab_tests-2_last-linux
<<: *FIREBASE_SHARD_TEMPLATE
- name: web_smoke_test
only_if: "changesInclude('.cirrus.yml', 'examples/hello_world/**' ,'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/**') && $CIRRUS_PR != ''"
environment:
......
dev/bots/test.dart
View file @ fd0e223b
......@@ -55,11 +55,6 @@ final List<String> flutterTestArgs = <String>[];
final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true';
/// The number of Cirrus jobs that run host-only devicelab tests in parallel.
///
/// WARNING: if you change this number, also change .cirrus.yml
/// and make sure it runs _all_ shards.
const int kDeviceLabShardCount = 2;
/// The number of Cirrus jobs that run build tests in parallel.
///
......@@ -121,10 +116,8 @@ Future<void> main(List<String> args) async {
await selectShard(const <String, ShardRunner>{
'add_to_app_life_cycle_tests': _runAddToAppLifeCycleTests,
'build_tests': _runBuildTests,
'firebase_test_lab_tests': _runFirebaseTestLabTests,
'framework_coverage': _runFrameworkCoverage,
'framework_tests': _runFrameworkTests,
'hostonly_devicelab_tests': _runHostOnlyDeviceLabTests,
'tool_coverage': _runToolCoverage,
'tool_tests': _runToolTests,
'web_tests': _runWebUnitTests,
......@@ -669,28 +662,6 @@ Future<void> _runFrameworkTests() async {
});
}
Future<void> _runFirebaseTestLabTests() async {
// Firebase Lab tests take ~20 minutes per integration test,
// so only one test is run per shard. Therefore, there are as
// many shards available as there are integration tests in this list.
// If you add a new test, add a corresponding firebase_test_lab-#-linux
// to .cirrus.yml
final List<String> integrationTests = <String>[
'release_smoke_test',
'abstract_method_smoke_test',
'android_embedding_v2_smoke_test',
];
final String firebaseScript = path.join(flutterRoot, 'dev', 'bots', 'firebase_testlab.sh');
final String integrationTestDirectory = path.join(flutterRoot, 'dev', 'integration_tests');
final List<ShardRunner> tests = integrationTests.map((String integrationTest) =>
() => runCommand(firebaseScript, <String>[ path.join(integrationTestDirectory, integrationTest) ])
).toList();
await _selectIndexedSubshard(tests, integrationTests.length);
}
Future<void> _runFrameworkCoverage() async {
final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
if (!coverageFile.existsSync()) {
......@@ -1215,51 +1186,6 @@ Map<String, String> _initGradleEnvironment() {
final Map<String, String> gradleEnvironment = _initGradleEnvironment();
Future<void> _runHostOnlyDeviceLabTests() async {
// Please don't add more tests here. We should not be using the devicelab
// logic to run tests outside devicelab, that's just confusing.
// Instead, create tests that are not devicelab tests, and run those.
// TODO(ianh): Move the tests that are not running on devicelab any more out
// of the device lab directory.
// List the tests to run.
// We split these into subshards. The tests are randomly distributed into
// those subshards so as to get a uniform distribution of costs, but the
// seed is fixed so that issues are reproducible.
final List<ShardRunner> tests = <ShardRunner>[
// Keep this in alphabetical order.
() => _runDevicelabTest('build_aar_module_test', environment: gradleEnvironment),
() => _runDevicelabTest('gradle_jetifier_test', environment: gradleEnvironment),
() => _runDevicelabTest('gradle_plugin_light_apk_test', environment: gradleEnvironment),
() => _runDevicelabTest('module_custom_host_app_name_test', environment: gradleEnvironment),
() => _runDevicelabTest('module_test', environment: gradleEnvironment, testEmbeddingV2: true),
() => _runDevicelabTest('plugin_dependencies_test', environment: gradleEnvironment),
]..shuffle(math.Random(0));
await _selectIndexedSubshard(tests, kDeviceLabShardCount);
}
Future<void> _runDevicelabTest(String testName, {
Map<String, String> environment,
// testEmbeddingV2 is only supported by certain specific devicelab tests.
// Don't use it unless you're sure the test actually supports it.
// You can check by looking to see if the test examines the environment
// for the ENABLE_ANDROID_EMBEDDING_V2 variable.
bool testEmbeddingV2 = false,
}) async {
await runCommand(
dart,
<String>['bin/run.dart', '-t', testName],
workingDirectory: path.join(flutterRoot, 'dev', 'devicelab'),
environment: <String, String>{
...?environment,
if (testEmbeddingV2)
'ENABLE_ANDROID_EMBEDDING_V2': 'true',
},
);
}
void deleteFile(String path) {
// This is technically a race condition but nobody else should be running
// while this script runs, so we should be ok. (Sadly recursive:true does not
......
dev/prod_builders.json
View file @ fd0e223b
......@@ -30,6 +30,24 @@
"task_name": "linux_build_tests",
"flaky": false
},
{
"name": "Linux firebase_abstract_method_smoke_test",
"repo": "flutter",
"task_name": "linux_firebase_abstract_method_smoke_test",
"flaky": false
},
{
"name": "Linux firebase_android_embedding_v2_smoke_test",
"repo": "flutter",
"task_name": "linux_firebase_android_embedding_v2_smoke_test",
"flaky": false
},
{
"name": "Linux firebase_release_smoke_test",
"repo": "flutter",
"task_name": "linux_firebase_release_smoke_test",
"flaky": false
},
{
"name": "Linux framework_tests",
"repo": "flutter",
......
dev/try_builders.json
View file @ fd0e223b
......@@ -39,6 +39,24 @@
"enabled":true,
"run_if":["dev/", "packages/flutter/", "packages/flutter_test/", "packages/flutter_goldens/", "packages/flutter_tools/lib/src/test/", "bin/"]
},
{
"name":"Linux firebase_abstract_method_smoke_test",
"repo":"flutter",
"task_name":"linux_firebase_abstract_method_smoke_test",
"enabled":true
},
{
"name":"Linux firebase_android_embedding_v2_smoke_test",
"repo":"flutter",
"task_name":"linux_firebase_android_embedding_v2_smoke_test",
"enabled":true
},
{
"name":"Linux firebase_release_smoke_test",
"repo":"flutter",
"task_name":"linux_firebase_release_smoke_test",
"enabled":true
},
{
"name":"Linux fuchsia_precache",
"repo":"flutter",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab