Scan assets from disk (#16413)

Scan folders when asset specified finishes with `/`

packages/flutter_tools/lib/src/asset.dart

@@ -9,6 +9,7 @@ import 'package:yaml/yaml.dart';
 
 import 'base/context.dart';
 import 'base/file_system.dart';
+import 'base/platform.dart';
 import 'build_info.dart';
 import 'cache.dart';
 import 'dart/package_map.dart';
@@ -365,9 +366,19 @@ Future<DevFSContent> _obtainLicenses(
   return new DevFSStringContent(combinedLicenses);
 }
 
+int _byBasename(_Asset a, _Asset b) {
+  return a.assetFile.basename.compareTo(b.assetFile.basename);
+}
+
 DevFSContent _createAssetManifest(Map<_Asset, List<_Asset>> assetVariants) {
   final Map<String, List<String>> jsonObject = <String, List<String>>{};
-  for (_Asset main in assetVariants.keys) {
+
+  // necessary for making unit tests deterministic
+  final List<_Asset> sortedKeys = assetVariants
+      .keys.toList()
+    ..sort(_byBasename);
+
+  for (_Asset main in sortedKeys) {
     final List<String> variants = <String>[];
     for (_Asset variant in assetVariants[main])
       variants.add(variant.entryUri.path);
@@ -485,6 +496,27 @@ class _AssetDirectoryCache {
 /// map of assets to asset variants.
 ///
 /// Returns null on missing assets.
+///
+/// Given package: 'test_package' and an assets directory like this:
+///
+/// assets/foo
+/// assets/var1/foo
+/// assets/var2/foo
+/// assets/bar
+///
+/// returns
+/// {
+///   asset: packages/test_package/assets/foo: [
+///     asset: packages/test_package/assets/foo,
+///     asset: packages/test_package/assets/var1/foo,
+///     asset: packages/test_package/assets/var2/foo,
+///   ],
+///   asset: packages/test_package/assets/bar: [
+///     asset: packages/test_package/assets/bar,
+///   ],
+/// }
+///
+
 Map<_Asset, List<_Asset>> _parseAssets(
   PackageMap packageMap,
   FlutterManifest flutterManifest,
@@ -496,27 +528,15 @@ Map<_Asset, List<_Asset>> _parseAssets(
 
   final _AssetDirectoryCache cache = new _AssetDirectoryCache(excludeDirs);
   for (Uri assetUri in flutterManifest.assets) {
-    final _Asset asset = _resolveAsset(
-      packageMap,
-      assetBase,
-      assetUri,
-      packageName,
-    );
-    final List<_Asset> variants = <_Asset>[];
-    for (String path in cache.variantsFor(asset.assetFile.path)) {
-      final String relativePath = fs.path.relative(path, from: asset.baseDir);
-      final Uri relativeUri = fs.path.toUri(relativePath);
-      final Uri entryUri = asset.symbolicPrefixUri == null
-          ? relativeUri
-          : asset.symbolicPrefixUri.resolveUri(relativeUri);
-      variants.add(new _Asset(
-        baseDir: asset.baseDir,
-        entryUri: entryUri,
-        relativeUri: relativeUri,
-      ));
+    if (assetUri.toString().endsWith('/')) {
+      _parseAssetsFromFolder(packageMap, flutterManifest, assetBase,
+          cache, result, assetUri,
+          excludeDirs: excludeDirs, packageName: packageName);
+    } else {
+      _parseAssetFromFile(packageMap, flutterManifest, assetBase,
+          cache, result, assetUri,
+          excludeDirs: excludeDirs, packageName: packageName);
     }
-
-    result[asset] = variants;
   }
 
   // Add assets referenced in the fonts section of the manifest.
@@ -540,6 +560,72 @@ Map<_Asset, List<_Asset>> _parseAssets(
   return result;
 }
 
+void _parseAssetsFromFolder(PackageMap packageMap,
+  FlutterManifest flutterManifest,
+  String assetBase,
+  _AssetDirectoryCache cache,
+  Map<_Asset, List<_Asset>> result,
+  Uri assetUri, {
+  List<String> excludeDirs: const <String>[],
+  String packageName
+}) {
+  final String directoryPath = fs.path.join(
+      assetBase, assetUri.toFilePath(windows: platform.isWindows));
+
+  if (!fs.directory(directoryPath).existsSync()) {
+    printError('Error: unable to find directory entry in pubspec.yaml: $directoryPath');
+    return;
+  }
+
+  final List<FileSystemEntity> lister = fs.directory(directoryPath).listSync();
+
+  for (FileSystemEntity entity in lister) {
+    if (entity is File) {
+      final String relativePath = fs.path.relative(entity.path, from: assetBase);
+
+      final Uri uri = new Uri.file(relativePath, windows: platform.isWindows);
+
+      _parseAssetFromFile(packageMap, flutterManifest, assetBase, cache, result,
+          uri, packageName: packageName);
+    }
+  }
+}
+
+void _parseAssetFromFile(PackageMap packageMap,
+  FlutterManifest flutterManifest,
+  String assetBase,
+  _AssetDirectoryCache cache,
+  Map<_Asset, List<_Asset>> result,
+  Uri assetUri, {
+  List<String> excludeDirs: const <String>[],
+  String packageName
+}) {
+  final _Asset asset = _resolveAsset(
+    packageMap,
+    assetBase,
+    assetUri,
+    packageName,
+  );
+  final List<_Asset> variants = <_Asset>[];
+  for (String path in cache.variantsFor(asset.assetFile.path)) {
+    final String relativePath = fs.path.relative(path, from: asset.baseDir);
+    final Uri relativeUri = fs.path.toUri(relativePath);
+    final Uri entryUri = asset.symbolicPrefixUri == null
+        ? relativeUri
+        : asset.symbolicPrefixUri.resolveUri(relativeUri);
+
+    variants.add(
+      new _Asset(
+        baseDir: asset.baseDir,
+        entryUri: entryUri,
+        relativeUri: relativeUri,
+        )
+    );
+  }
+
+  result[asset] = variants;
+}
+
 _Asset _resolveAsset(
   PackageMap packageMap,
   String assetsBaseDir,

packages/flutter_tools/lib/src/dart/package_map.dart

@@ -5,12 +5,14 @@
 import 'package:package_config/packages_file.dart' as packages_file;
 
 import '../base/file_system.dart';
+import '../base/platform.dart';
 
 const String kPackagesFileName = '.packages';
 
 Map<String, Uri> _parse(String packagesPath) {
   final List<int> source = fs.file(packagesPath).readAsBytesSync();
-  return packages_file.parse(source, new Uri.file(packagesPath));
+  return packages_file.parse(source,
+      new Uri.file(packagesPath, windows: platform.isWindows));
 }
 
 class PackageMap {

packages/flutter_tools/lib/src/flutter_manifest.dart

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 import 'dart:async';
+import 'dart:convert' as convert;
 
 import 'package:json_schema/json_schema.dart';
 import 'package:meta/meta.dart';
@@ -151,13 +152,27 @@ class FontAsset {
   String toString() => '$runtimeType(asset: ${assetUri.path}, weight; $weight, style: $style)';
 }
 
-Future<bool> _validate(Object manifest) async {
-  final String schemaPath = fs.path.join(
+@visibleForTesting
+String buildSchemaDir(FileSystem fs) {
+  return fs.path.join(
     fs.path.absolute(Cache.flutterRoot), 'packages', 'flutter_tools', 'schema',
+  );
+}
+
+@visibleForTesting
+String buildSchemaPath(FileSystem fs) {
+  return fs.path.join(
+    buildSchemaDir(fs),
     'pubspec_yaml.json',
   );
-  final Schema schema = await Schema.createSchemaFromUrl(fs.path.toUri(schemaPath).toString());
+}
+
+Future<bool> _validate(Object manifest) async {
+  final String schemaPath = buildSchemaPath(fs);
 
+  final String schemaData = fs.file(schemaPath).readAsStringSync();
+  final Schema schema = await Schema.createSchema(
+      convert.json.decode(schemaData));
   final Validator validator = new Validator(schema);
   if (validator.validate(manifest)) {
     return true;

packages/flutter_tools/test/asset_bundle_package_test.dart

@@ -6,11 +6,12 @@ import 'dart:async';
 import 'dart:convert';
 
 import 'package:file/file.dart';
-
+import 'package:file/memory.dart';
 import 'package:flutter_tools/src/asset.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
+import 'package:flutter_tools/src/base/platform.dart';
 import 'package:flutter_tools/src/cache.dart';
-
+import 'package:flutter_tools/src/flutter_manifest.dart';
 import 'package:test/test.dart';
 
 import 'src/common.dart';
@@ -36,7 +37,9 @@ flutter:
       assetsSection = buffer.toString();
     }
 
-    fs.file(path)
+    final Uri uri = new Uri.file(path, windows: platform.isWindows);
+
+    fs.file(uri)
       ..createSync(recursive: true)
       ..writeAsStringSync('''
 name: $name
@@ -48,8 +51,6 @@ $assetsSection
   }
 
   void establishFlutterRoot() {
-    // Setting flutterRoot here so that it picks up the MemoryFileSystem's
-    // path separator.
     Cache.flutterRoot = getFlutterRoot();
   }
 
@@ -70,7 +71,7 @@ $assetsSection
     for (String packageName in packages) {
       for (String asset in assets) {
         final String entryKey = Uri.encodeFull('packages/$packageName/$asset');
-        expect(bundle.entries.containsKey(entryKey), true);
+        expect(bundle.entries.containsKey(entryKey), true, reason: 'Cannot find key on bundle: $entryKey');
         expect(
           utf8.decode(await bundle.entries[entryKey].contentsAsBytes()),
           asset,
@@ -86,7 +87,14 @@ $assetsSection
 
   void writeAssets(String path, List<String> assets) {
     for (String asset in assets) {
-      fs.file('$path$asset')
+      final String fullPath = fs.path.join(path, asset); // posix compatible
+
+      final String normalizedFullPath = // posix and windows compatible over MemoryFileSystem
+      new Uri.file(
+          fullPath, windows: platform.isWindows)
+          .toFilePath(windows: platform.isWindows);
+
+      fs.file(normalizedFullPath)
         ..createSync(recursive: true)
         ..writeAsStringSync(asset);
     }
@@ -274,8 +282,8 @@ $assetsSection
 
       writeAssets('p/p/', assets);
       const String expectedAssetManifest =
-          '{"packages/test_package/a/foo":["packages/test_package/a/foo"],'
-          '"packages/test_package/a/bar":["packages/test_package/a/bar"]}';
+          '{"packages/test_package/a/bar":["packages/test_package/a/bar"],'
+          '"packages/test_package/a/foo":["packages/test_package/a/foo"]}';
 
       await buildAndVerifyAssets(
         assets,
@@ -306,8 +314,8 @@ $assetsSection
 
       writeAssets('p/p/lib/', assets);
       const String expectedAssetManifest =
-          '{"packages/test_package/a/foo":["packages/test_package/a/foo"],'
-          '"packages/test_package/a/bar":["packages/test_package/a/bar"]}';
+          '{"packages/test_package/a/bar":["packages/test_package/a/bar"],'
+          '"packages/test_package/a/foo":["packages/test_package/a/foo"]}';
 
       await buildAndVerifyAssets(
         assets,
@@ -447,4 +455,232 @@ $assetsSection
       expectedAssetManifest,
     );
   });
+
+  group('AssetBundle assets from scanned paths', () {
+    testUsingContext(
+        'Two assets are bundled when scanning their directory', () async {
+      establishFlutterRoot();
+
+      writePubspecFile('pubspec.yaml', 'test');
+      writePackagesFile('test_package:p/p/lib/');
+
+      final List<String> assetsOnDisk = <String>['a/foo', 'a/bar'];
+      final List<String> assetsOnManifest = <String>['a/'];
+
+      writePubspecFile(
+        'p/p/pubspec.yaml',
+        'test_package',
+        assets: assetsOnManifest,
+      );
+
+      writeAssets('p/p/', assetsOnDisk);
+      const String expectedAssetManifest =
+          '{"packages/test_package/a/bar":["packages/test_package/a/bar"],'
+          '"packages/test_package/a/foo":["packages/test_package/a/foo"]}';
+
+      await buildAndVerifyAssets(
+        assetsOnDisk,
+        <String>['test_package'],
+        expectedAssetManifest,
+      );
+    });
+
+    testUsingContext(
+        'Two assets are bundled when listing one and scanning second directory', () async {
+      establishFlutterRoot();
+
+      writePubspecFile('pubspec.yaml', 'test');
+      writePackagesFile('test_package:p/p/lib/');
+
+      final List<String> assetsOnDisk = <String>['a/foo', 'abc/bar'];
+      final List<String> assetOnManifest = <String>['a/foo', 'abc/'];
+
+      writePubspecFile(
+        'p/p/pubspec.yaml',
+        'test_package',
+        assets: assetOnManifest,
+      );
+
+      writeAssets('p/p/', assetsOnDisk);
+      const String expectedAssetManifest =
+          '{"packages/test_package/abc/bar":["packages/test_package/abc/bar"],'
+          '"packages/test_package/a/foo":["packages/test_package/a/foo"]}';
+
+      await buildAndVerifyAssets(
+        assetsOnDisk,
+        <String>['test_package'],
+        expectedAssetManifest,
+      );
+    });
+
+    testUsingContext(
+        'One asset is bundled with variant, scanning wrong directory', () async {
+      establishFlutterRoot();
+
+      writePubspecFile('pubspec.yaml', 'test');
+      writePackagesFile('test_package:p/p/lib/');
+
+      final List<String> assetsOnDisk = <String>['a/foo','a/b/foo','a/bar'];
+      final List<String> assetOnManifest = <String>['a','a/bar']; // can't list 'a' as asset, should be 'a/'
+
+      writePubspecFile(
+        'p/p/pubspec.yaml',
+        'test_package',
+        assets: assetOnManifest,
+      );
+
+      writeAssets('p/p/', assetsOnDisk);
+
+      final AssetBundle bundle = AssetBundleFactory.instance.createBundle();
+      await bundle.build(manifestPath: 'pubspec.yaml');
+      assert(bundle.entries['AssetManifest.json'] == null,'Invalid pubspec.yaml should not generate AssetManifest.json'  );
+    });
+  });
+
+  group('AssetBundle assets from scanned paths with MemoryFileSystem', () {
+    String readSchemaPath(FileSystem fs) {
+      final String schemaPath = buildSchemaPath(fs);
+      final File schemaFile = fs.file(schemaPath);
+
+      return schemaFile.readAsStringSync();
+    }
+
+    void writeSchema(String schema, FileSystem filesystem) {
+      final String schemaPath = buildSchemaPath(filesystem);
+      final File schemaFile = filesystem.file(schemaPath);
+
+      final Directory schemaDir = filesystem.directory(
+          buildSchemaDir(filesystem));
+
+      schemaDir.createSync(recursive: true);
+      schemaFile.writeAsStringSync(schema);
+    }
+
+    void testUsingContextAndFs(String description, dynamic testMethod(),) {
+      final FileSystem windowsFs = new MemoryFileSystem(
+          style: FileSystemStyle.windows);
+
+      final FileSystem posixFs = new MemoryFileSystem(
+          style: FileSystemStyle.posix);
+
+      const String _kFlutterRoot = '/flutter/flutter';
+      establishFlutterRoot();
+
+      final String schema = readSchemaPath(fs);
+
+      testUsingContext('$description - on windows FS', () async {
+        establishFlutterRoot();
+        writeSchema(schema, windowsFs);
+        await testMethod();
+      }, overrides: <Type, Generator>{
+        FileSystem: () => windowsFs,
+        Platform: () =>
+        new FakePlatform(
+            environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot,},
+            operatingSystem: 'windows')
+      });
+
+      testUsingContext('$description - on posix FS', () async {
+        establishFlutterRoot();
+        writeSchema(schema, posixFs);
+        await testMethod();
+      }, overrides: <Type, Generator>{
+        FileSystem: () => posixFs,
+        Platform: () =>
+        new FakePlatform(
+            environment: <String, String>{ 'FLUTTER_ROOT': _kFlutterRoot,},
+            operatingSystem: 'linux')
+      });
+
+      testUsingContext('$description - on original FS', () async {
+        establishFlutterRoot();
+        await testMethod();
+      });
+    }
+
+    testUsingContextAndFs(
+        'One asset is bundled with variant, scanning directory', () async {
+      establishFlutterRoot();
+
+      writePubspecFile('pubspec.yaml', 'test');
+      writePackagesFile('test_package:p/p/lib/');
+
+      final List<String> assetsOnDisk = <String>['a/foo','a/b/foo'];
+      final List<String> assetOnManifest = <String>['a/',];
+
+      writePubspecFile(
+        'p/p/pubspec.yaml',
+        'test_package',
+        assets: assetOnManifest,
+      );
+
+      writeAssets('p/p/', assetsOnDisk);
+      const String expectedAssetManifest =
+          '{"packages/test_package/a/foo":["packages/test_package/a/foo","packages/test_package/a/b/foo"]}';
+
+      await buildAndVerifyAssets(
+        assetsOnDisk,
+        <String>['test_package'],
+        expectedAssetManifest,
+      );
+    });
+
+    testUsingContextAndFs(
+        'No asset is bundled with variant, no assets or directories are listed', () async {
+      establishFlutterRoot();
+
+      writePubspecFile('pubspec.yaml', 'test');
+      writePackagesFile('test_package:p/p/lib/');
+
+      final List<String> assetsOnDisk = <String>['a/foo', 'a/b/foo'];
+      final List<String> assetOnManifest = <String>[];
+
+      writePubspecFile(
+        'p/p/pubspec.yaml',
+        'test_package',
+        assets: assetOnManifest,
+      );
+
+      writeAssets('p/p/', assetsOnDisk);
+      const String expectedAssetManifest = '{}';
+
+      await buildAndVerifyAssets(
+        assetOnManifest,
+        <String>['test_package'],
+        expectedAssetManifest,
+      );
+    });
+
+    testUsingContextAndFs(
+        'Expect error generating manifest, wrong non-existing directory is listed', () async {
+      establishFlutterRoot();
+
+      writePubspecFile('pubspec.yaml', 'test');
+      writePackagesFile('test_package:p/p/lib/');
+
+      final List<String> assetOnManifest = <String>['c/'];
+
+      writePubspecFile(
+        'p/p/pubspec.yaml',
+        'test_package',
+        assets: assetOnManifest,
+      );
+
+      try {
+        await buildAndVerifyAssets(
+          assetOnManifest,
+          <String>['test_package'],
+          null,
+        );
+
+        final Function watchdog = () async {
+          assert(false, 'Code failed to detect missing directory. Test failed.');
+        };
+        watchdog();
+      } catch (e) {
+        // Test successful
+      }
+    });
+
+  });
 }

packages/flutter_tools/test/flutter_manifest_test.dart

@@ -2,9 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'package:file/file.dart';
+import 'package:file/memory.dart';
+import 'package:flutter_tools/src/base/file_system.dart';
 import 'package:flutter_tools/src/cache.dart';
 import 'package:flutter_tools/src/flutter_manifest.dart';
-
 import 'package:test/test.dart';
 
 import 'src/common.dart';
@@ -358,4 +360,64 @@ flutter:
       expect(flutterManifest.isEmpty, false);
     });
   });
+
+  group('FlutterManifest with MemoryFileSystem', () {
+    void assertSchemaIsReadable() async {
+      const String manifest = '''
+name: test
+dependencies:
+  flutter:
+    sdk: flutter
+flutter:
+''';
+
+      final FlutterManifest flutterManifest = await FlutterManifest
+          .createFromString(manifest);
+      expect(flutterManifest.isEmpty, false);
+    }
+
+    void writeSchemaFile(FileSystem filesystem, String schemaData) {
+      final String schemaPath = buildSchemaPath(filesystem);
+      final File schemaFile = filesystem.file(schemaPath);
+
+      final String schemaDir = buildSchemaDir(filesystem);
+
+      filesystem.directory(schemaDir).createSync(recursive: true);
+      filesystem.file(schemaFile).writeAsStringSync(schemaData);
+    }
+
+    void testUsingContextAndFs(String description, FileSystem filesystem,
+        dynamic testMethod()) {
+      const String schemaData = '{}';
+
+      testUsingContext(description,
+              () async {
+            writeSchemaFile( filesystem, schemaData);
+            testMethod();
+      },
+          overrides: <Type, Generator>{
+            FileSystem: () => filesystem,
+          }
+      );
+    }
+
+    testUsingContext('Validate manifest on original fs', () async {
+      assertSchemaIsReadable();
+    });
+
+    testUsingContextAndFs('Validate manifest on Posix FS',
+        new MemoryFileSystem(style: FileSystemStyle.posix), () async {
+          assertSchemaIsReadable();
+        }
+    );
+
+    testUsingContextAndFs('Validate manifest on Windows FS',
+        new MemoryFileSystem(style: FileSystemStyle.windows), () async {
+          assertSchemaIsReadable();
+        }
+    );
+
+  });
+
 }
+