Add more flutter build ios-framework tests before the impending jonahpocalypse (#44308)

dev/devicelab/bin/tasks/build_aar_module_test.dart

@@ -152,7 +152,7 @@ Future<void> main() async {
 
       section('Check assets in release AAR');
 
-      checkItContains<String>(
+      checkCollectionContains<String>(
         <String>[
           ...flutterAssets,
           // AOT snapshots
@@ -278,7 +278,7 @@ Future<void> main() async {
         'flutter_debug-1.0.aar',
       ));
 
-      checkItContains<String>(<String>[
+      checkCollectionContains<String>(<String>[
         ...flutterAssets,
         ...debugAssets,
       ], debugAar);

dev/devicelab/bin/tasks/build_ios_framework_module_test.dart

@@ -5,8 +5,8 @@
 import 'dart:async';
 import 'dart:io';
 
-import 'package:flutter_devicelab/framework/apk_utils.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/ios.dart';
 import 'package:flutter_devicelab/framework/utils.dart';
 import 'package:path/path.dart' as path;
 
@@ -69,7 +69,29 @@ Future<void> main() async {
         'vm_snapshot_data',
       ));
 
-      section('Check profile, release builds has Dart dylib');
+      section('Check debug build has no Dart AOT');
+
+      // There's still an App.framework with a dylib, but it's empty.
+      checkFileExists(path.join(
+        outputPath,
+        'Debug',
+        'App.framework',
+        'App',
+      ));
+
+      final String aotSymbols = await dylibSymbols(path.join(
+        outputPath,
+        'Debug',
+        'App.framework',
+        'App',
+      ));
+
+      if (aotSymbols.contains('architecture') ||
+          aotSymbols.contains('_kDartVmSnapshot')) {
+        throw TaskResult.failure('Debug App.framework contains AOT');
+      }
+
+      section('Check profile, release builds has Dart AOT dylib');
 
       for (String mode in <String>['Profile', 'Release']) {
         checkFileExists(path.join(
@@ -79,6 +101,25 @@ Future<void> main() async {
           'App',
         ));
 
+        final String aotSymbols = await dylibSymbols(path.join(
+          outputPath,
+          mode,
+          'App.framework',
+          'App',
+        ));
+
+        if (!aotSymbols.contains('armv7')) {
+          throw TaskResult.failure('$mode App.framework armv7 architecture missing');
+        }
+
+        if (!aotSymbols.contains('arm64')) {
+          throw TaskResult.failure('$mode App.framework arm64 architecture missing');
+        }
+
+        if (!aotSymbols.contains('_kDartVmSnapshot')) {
+          throw TaskResult.failure('$mode App.framework missing Dart AOT');
+        }
+
         checkFileNotExists(path.join(
           outputPath,
           mode,

dev/devicelab/bin/tasks/gradle_plugin_bundle_test.dart

@@ -31,7 +31,7 @@ Future<void> main() async {
           'release',
           'app-release.aab',
         );
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...baseAabFiles,
           ...flutterAabAssets,
           'base/lib/arm64-v8a/libapp.so',
@@ -66,7 +66,7 @@ Future<void> main() async {
           'productionRelease',
           'app-production-release.aab',
         );
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...baseAabFiles,
           ...flutterAabAssets,
           'base/lib/arm64-v8a/libapp.so',
@@ -102,7 +102,7 @@ Future<void> main() async {
           'flavor_underscoreRelease',
           'app-flavor_underscore-release.aab',
         );
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...baseAabFiles,
           ...flutterAabAssets,
           'base/lib/arm64-v8a/libapp.so',
@@ -137,7 +137,7 @@ Future<void> main() async {
           'productionRelease',
           'app-production-release.aab',
         );
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...baseAabFiles,
           ...flutterAabAssets,
           'base/lib/arm64-v8a/libapp.so',
@@ -163,14 +163,14 @@ Future<void> main() async {
         );
 
         final Iterable<String> bundleFiles = await getFilesInAppBundle(releaseBundle);
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...baseAabFiles,
           ...flutterAabAssets,
           'base/lib/armeabi-v7a/libapp.so',
           'base/lib/armeabi-v7a/libflutter.so',
         ], bundleFiles);
 
-        checkItDoesNotContain<String>(<String>[
+        checkCollectionDoesNotContain<String>(<String>[
           'base/lib/arm64-v8a/libapp.so',
           'base/lib/arm64-v8a/libflutter.so',
         ], bundleFiles);

dev/devicelab/bin/tasks/gradle_plugin_fat_apk_test.dart

@@ -19,7 +19,7 @@ Future<void> main() async {
 
         final Iterable<String> apkFiles = await getFilesInApk(pluginProject.debugApkPath);
 
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...flutterAssets,
           ...debugAssets,
           ...baseApkFiles,
@@ -30,7 +30,7 @@ Future<void> main() async {
           'lib/x86_64/libflutter.so',
         ], apkFiles);
 
-        checkItDoesNotContain<String>(<String>[
+        checkCollectionDoesNotContain<String>(<String>[
           'lib/arm64-v8a/libapp.so',
           'lib/armeabi-v7a/libapp.so',
           'lib/x86/libapp.so',
@@ -44,7 +44,7 @@ Future<void> main() async {
 
         final Iterable<String> apkFiles = await getFilesInApk(pluginProject.releaseApkPath);
 
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...flutterAssets,
           ...baseApkFiles,
           'lib/armeabi-v7a/libflutter.so',
@@ -53,7 +53,7 @@ Future<void> main() async {
           'lib/arm64-v8a/libapp.so',
         ], apkFiles);
 
-        checkItDoesNotContain<String>(debugAssets, apkFiles);
+        checkCollectionDoesNotContain<String>(debugAssets, apkFiles);
       });
 
       await runPluginProjectTest((FlutterPluginProject pluginProject) async {
@@ -63,7 +63,7 @@ Future<void> main() async {
 
         final Iterable<String> apkFiles = await getFilesInApk(pluginProject.releaseApkPath);
 
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...flutterAssets,
           ...baseApkFiles,
           'lib/armeabi-v7a/libflutter.so',
@@ -72,7 +72,7 @@ Future<void> main() async {
           'lib/arm64-v8a/libapp.so',
         ], apkFiles);
 
-        checkItDoesNotContain<String>(debugAssets, apkFiles);
+        checkCollectionDoesNotContain<String>(debugAssets, apkFiles);
       });
 
       await runPluginProjectTest((FlutterPluginProject pluginProject) async {
@@ -83,25 +83,25 @@ Future<void> main() async {
 
         final Iterable<String> armApkFiles = await getFilesInApk(pluginProject.releaseArmApkPath);
 
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...flutterAssets,
           ...baseApkFiles,
           'lib/armeabi-v7a/libflutter.so',
           'lib/armeabi-v7a/libapp.so',
         ], armApkFiles);
 
-        checkItDoesNotContain<String>(debugAssets, armApkFiles);
+        checkCollectionDoesNotContain<String>(debugAssets, armApkFiles);
 
         final Iterable<String> arm64ApkFiles = await getFilesInApk(pluginProject.releaseArm64ApkPath);
 
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...flutterAssets,
           ...baseApkFiles,
           'lib/arm64-v8a/libflutter.so',
           'lib/arm64-v8a/libapp.so',
         ], arm64ApkFiles);
 
-        checkItDoesNotContain<String>(debugAssets, arm64ApkFiles);
+        checkCollectionDoesNotContain<String>(debugAssets, arm64ApkFiles);
       });
 
       await runProjectTest((FlutterProject project) async {

dev/devicelab/bin/tasks/gradle_plugin_light_apk_test.dart

@@ -19,7 +19,7 @@ Future<void> main() async {
 
         final Iterable<String> apkFiles = await getFilesInApk(pluginProject.debugApkPath);
 
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...flutterAssets,
           ...debugAssets,
           ...baseApkFiles,
@@ -29,7 +29,7 @@ Future<void> main() async {
           'lib/x86_64/libflutter.so',
         ], apkFiles);
 
-        checkItDoesNotContain<String>(<String>[
+        checkCollectionDoesNotContain<String>(<String>[
           'lib/arm64-v8a/libapp.so',
           'lib/armeabi-v7a/libapp.so',
           'lib/x86/libapp.so',
@@ -45,7 +45,7 @@ Future<void> main() async {
 
         final Iterable<String> apkFiles = await getFilesInApk(pluginProject.debugApkPath);
 
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...flutterAssets,
           ...debugAssets,
           ...baseApkFiles,
@@ -54,7 +54,7 @@ Future<void> main() async {
           'lib/x86_64/libflutter.so',
         ], apkFiles);
 
-        checkItDoesNotContain<String>(<String>[
+        checkCollectionDoesNotContain<String>(<String>[
           'lib/armeabi-v7a/libapp.so',
           'lib/x86/libapp.so',
           'lib/x86_64/libapp.so',
@@ -69,7 +69,7 @@ Future<void> main() async {
 
         final Iterable<String> apkFiles = await getFilesInApk(pluginProject.debugApkPath);
 
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...flutterAssets,
           ...debugAssets,
           ...baseApkFiles,
@@ -78,7 +78,7 @@ Future<void> main() async {
           'lib/x86_64/libflutter.so',
         ], apkFiles);
 
-        checkItDoesNotContain<String>(<String>[
+        checkCollectionDoesNotContain<String>(<String>[
           'lib/armeabi-v7a/libapp.so',
           'lib/x86/libapp.so',
           'lib/x86_64/libapp.so',
@@ -92,14 +92,14 @@ Future<void> main() async {
 
         final Iterable<String> apkFiles = await getFilesInApk(pluginProject.releaseApkPath);
 
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...flutterAssets,
           ...baseApkFiles,
           'lib/armeabi-v7a/libflutter.so',
           'lib/armeabi-v7a/libapp.so',
         ], apkFiles);
 
-        checkItDoesNotContain<String>(<String>[
+        checkCollectionDoesNotContain<String>(<String>[
           ...debugAssets,
           'lib/arm64-v8a/libflutter.so',
           'lib/arm64-v8a/libapp.so',
@@ -113,14 +113,14 @@ Future<void> main() async {
 
         final Iterable<String> apkFiles = await getFilesInApk(pluginProject.releaseApkPath);
 
-        checkItContains<String>(<String>[
+        checkCollectionContains<String>(<String>[
           ...flutterAssets,
           ...baseApkFiles,
           'lib/arm64-v8a/libflutter.so',
           'lib/arm64-v8a/libapp.so',
         ], apkFiles);
 
-        checkItDoesNotContain<String>(<String>[
+        checkCollectionDoesNotContain<String>(<String>[
           ...debugAssets,
           'lib/armeabi-v7a/libflutter.so',
           'lib/armeabi-v7a/libapp.so',

dev/devicelab/bin/tasks/module_host_with_custom_build_test.dart

@@ -123,7 +123,7 @@ Future<void> main() async {
 
       section('Verify snapshots in app-demo-debug.apk');
 
-      checkItContains<String>(<String>[
+      checkCollectionContains<String>(<String>[
         ...flutterAssets,
         ...debugAssets,
       ], await getFilesInApk(demoDebugApk));
@@ -166,7 +166,7 @@ Future<void> main() async {
 
       section('Verify snapshots in app-demo-debug.apk');
 
-      checkItContains<String>(<String>[
+      checkCollectionContains<String>(<String>[
         ...flutterAssets,
         ...debugAssets,
       ], await getFilesInApk(demoDebugApk2));
@@ -201,7 +201,7 @@ Future<void> main() async {
 
       section('Verify snapshots in app-demo-staging.apk');
 
-      checkItContains<String>(<String>[
+      checkCollectionContains<String>(<String>[
         ...flutterAssets,
         ...debugAssets,
       ], await getFilesInApk(demoStagingApk));
@@ -238,7 +238,7 @@ Future<void> main() async {
 
       section('Verify AOT blobs in app-demo-release-unsigned.apk');
 
-      checkItContains<String>(<String>[
+      checkCollectionContains<String>(<String>[
         ...flutterAssets,
         'lib/arm64-v8a/libflutter.so',
         'lib/arm64-v8a/libapp.so',
@@ -276,7 +276,7 @@ Future<void> main() async {
 
       section('Verify AOT blobs in app-demo-prod-unsigned.apk');
 
-      checkItContains<String>(<String>[
+      checkCollectionContains<String>(<String>[
         ...flutterAssets,
         'lib/arm64-v8a/libapp.so',
         'lib/arm64-v8a/libflutter.so',

dev/devicelab/bin/tasks/module_test.dart

@@ -198,7 +198,7 @@ Future<void> main() async {
 
       section('Check files in debug APK');
 
-      checkItContains<String>(<String>[
+      checkCollectionContains<String>(<String>[
         ...flutterAssets,
         ...debugAssets,
         ...baseApkFiles,
@@ -253,7 +253,7 @@ Future<void> main() async {
 
       section('Check files in release APK');
 
-      checkItContains<String>(<String>[
+      checkCollectionContains<String>(<String>[
         ...flutterAssets,
         ...baseApkFiles,
         'lib/arm64-v8a/libapp.so',

dev/devicelab/lib/framework/apk_utils.dart

@@ -76,36 +76,6 @@ Future<Iterable<String>> getFilesInAar(String aar) {
   return getFilesInApk(aar);
 }
 
-void checkItContains<T>(Iterable<T> values, Iterable<T> collection) {
-  for (T value in values) {
-    if (!collection.contains(value)) {
-      throw TaskResult.failure('Expected to find `$value` in `${collection.toString()}`.');
-    }
-  }
-}
-
-void checkItDoesNotContain<T>(Iterable<T> values, Iterable<T> collection) {
-  for (T value in values) {
-    if (collection.contains(value)) {
-      throw TaskResult.failure('Did not expect to find `$value` in `$collection`.');
-    }
-  }
-}
-
-///  Checks that [str] contains the specified [Pattern]s, otherwise throws
-/// a [TaskResult].
-void checkFileContains(List<Pattern> patterns, String filePath) {
-  final String fileContent = File(filePath).readAsStringSync();
-  for (Pattern pattern in patterns) {
-    if (!fileContent.contains(pattern)) {
-      throw TaskResult.failure(
-        'Expected to find `$pattern` in `$filePath` '
-        'instead it found:\n$fileContent'
-      );
-    }
-  }
-}
-
 TaskResult failure(String message, ProcessResult result) {
   print('Unexpected process result:');
   print('Exit code: ${result.exitCode}');

dev/devicelab/lib/framework/ios.dart

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 import 'dart:async';
+import 'dart:convert';
 import 'dart:io' as dart_io;
 
 import 'package:file/file.dart';
@@ -101,3 +102,50 @@ It is defined by environment variable $_kProvisioningConfigFileEnvironmentVariab
 
   return file.readAsString();
 }
+
+void _checkExitCode(int code) {
+  if (code != 0) {
+    throw Exception(
+      'Unexpected exit code = $code!',
+    );
+  }
+}
+
+Future<void> _execAndCheck(String executable, List<String> args) async {
+  _checkExitCode(await exec(executable, args));
+}
+
+// Measure the CPU/GPU percentage for [duration] while a Flutter app is running
+// on an iOS device (e.g., right after a Flutter driver test has finished, which
+// doesn't close the Flutter app, and the Flutter app has an indefinite
+// animation). The return should have a format like the following json
+// ```
+// {"gpu_percentage":12.6,"cpu_percentage":18.15}
+// ```
+Future<Map<String, dynamic>> measureIosCpuGpu({
+    Duration duration = const Duration(seconds: 10),
+    String deviceId,
+}) async {
+  await _execAndCheck('pub', <String>[
+    'global',
+    'activate',
+    'gauge',
+    '0.1.4',
+  ]);
+
+  await _execAndCheck('pub', <String>[
+    'global',
+    'run',
+    'gauge',
+    'ioscpugpu',
+    'new',
+    if (deviceId != null) ...<String>['-w', deviceId],
+    '-l',
+    '${duration.inMilliseconds}',
+  ]);
+  return json.decode(file('$cwd/result.json').readAsStringSync());
+}
+
+Future<String> dylibSymbols(String pathToDylib) {
+  return eval('nm', <String>['-g', pathToDylib]);
+}

dev/devicelab/lib/framework/utils.dart

@@ -13,6 +13,8 @@ import 'package:path/path.dart' as path;
 import 'package:process/process.dart';
 import 'package:stack_trace/stack_trace.dart';
 
+import 'framework.dart';
+
 /// Virtual current working directory, which affect functions, such as [exec].
 String cwd = Directory.current.path;
 
@@ -632,45 +634,34 @@ void checkFileNotExists(String file) {
   }
 }
 
-void _checkExitCode(int code) {
-  if (code != 0) {
-    throw Exception(
-      'Unexpected exit code = $code!',
-    );
+/// Check that `collection` contains all entries in `values`.
+void checkCollectionContains<T>(Iterable<T> values, Iterable<T> collection) {
+  for (T value in values) {
+    if (!collection.contains(value)) {
+      throw TaskResult.failure('Expected to find `$value` in `${collection.toString()}`.');
+    }
   }
 }
 
-Future<void> _execAndCheck(String executable, List<String> args) async {
-  _checkExitCode(await exec(executable, args));
+/// Check that `collection` does not contain any entries in `values`
+void checkCollectionDoesNotContain<T>(Iterable<T> values, Iterable<T> collection) {
+  for (T value in values) {
+    if (collection.contains(value)) {
+      throw TaskResult.failure('Did not expect to find `$value` in `$collection`.');
+    }
+  }
 }
 
-// Measure the CPU/GPU percentage for [duration] while a Flutter app is running
-// on an iOS device (e.g., right after a Flutter driver test has finished, which
-// doesn't close the Flutter app, and the Flutter app has an indefinite
-// animation). The return should have a format like the following json
-// ```
-// {"gpu_percentage":12.6,"cpu_percentage":18.15}
-// ```
-Future<Map<String, dynamic>> measureIosCpuGpu({
-    Duration duration = const Duration(seconds: 10),
-    String deviceId,
-}) async {
-  await _execAndCheck('pub', <String>[
-    'global',
-    'activate',
-    'gauge',
-    '0.1.4',
-  ]);
-
-  await _execAndCheck('pub', <String>[
-    'global',
-    'run',
-    'gauge',
-    'ioscpugpu',
-    'new',
-    if (deviceId != null) ...<String>['-w', deviceId],
-    '-l',
-    '${duration.inMilliseconds}',
-  ]);
-  return json.decode(file('$cwd/result.json').readAsStringSync());
+/// Checks that the contents of a [File] at `filePath` contains the specified
+/// [Pattern]s, otherwise throws a [TaskResult].
+void checkFileContains(List<Pattern> patterns, String filePath) {
+  final String fileContent = File(filePath).readAsStringSync();
+  for (Pattern pattern in patterns) {
+    if (!fileContent.contains(pattern)) {
+      throw TaskResult.failure(
+        'Expected to find `$pattern` in `$filePath` '
+        'instead it found:\n$fileContent'
+      );
+    }
+  }
 }