switch dart2js build to depfile, remove Source.function (#42977)

packages/flutter_tools/lib/src/build_system/depfile.dart

@@ -26,6 +26,28 @@ class Depfile {
     return Depfile(inputs, outputs);
   }
 
+  /// Parse the output of dart2js's used dependencies.
+  ///
+  /// The [file] contains a list of newline separated file URIs. The output
+  /// file must be manually specified.
+  factory Depfile.parseDart2js(File file, File output) {
+    final List<File> inputs = <File>[];
+    for (String rawUri in file.readAsLinesSync()) {
+      if (rawUri.trim().isEmpty) {
+        continue;
+      }
+      final Uri fileUri = Uri.tryParse(rawUri);
+      if (fileUri == null) {
+        continue;
+      }
+      if (fileUri.scheme != 'file') {
+        continue;
+      }
+      inputs.add(fs.file(fileUri));
+    }
+    return Depfile(inputs, <File>[output]);
+  }
+
   /// The input files for this depfile.
   final List<File> inputs;
 

packages/flutter_tools/lib/src/build_system/file_hash_store.dart

@@ -139,7 +139,8 @@ class FileHashStore {
     final List<File> dirty = <File>[];
     final Pool openFiles = Pool(kMaxOpenFiles);
     await Future.wait(<Future<void>>[
-      for (File file in files) _hashFile(file, dirty, openFiles)]);
+       for (File file in files) _hashFile(file, dirty, openFiles)
+    ]);
     return dirty;
   }
 
@@ -148,6 +149,13 @@ class FileHashStore {
     try {
       final String absolutePath = file.path;
       final String previousHash = previousHashes[absolutePath];
+      // If the file is missing it is assumed to be dirty.
+      if (!file.existsSync()) {
+        currentHashes.remove(absolutePath);
+        previousHashes.remove(absolutePath);
+        dirty.add(file);
+        return;
+      }
       final Digest digest = md5.convert(await file.readAsBytes());
       final String currentHash = digest.toString();
       if (currentHash != previousHash) {

packages/flutter_tools/lib/src/build_system/source.dart

@@ -9,10 +9,6 @@ import '../globals.dart';
 import 'build_system.dart';
 import 'exceptions.dart';
 
-/// An input function produces a list of additional input files for an
-/// [Environment].
-typedef InputFunction = List<File> Function(Environment environment);
-
 /// A set of source files.
 abstract class ResolvedFiles {
   /// Whether any of the sources we evaluated contained a missing depfile.
@@ -45,13 +41,6 @@ class SourceVisitor implements ResolvedFiles {
   bool get containsNewDepfile => _containsNewDepfile;
   bool _containsNewDepfile = false;
 
-  /// Visit a [Source] which contains a function.
-  ///
-  /// The function is expected to produce a list of [FileSystemEntities]s.
-  void visitFunction(InputFunction function) {
-    sources.addAll(function(environment));
-  }
-
   /// Visit a depfile which contains both input and output files.
   ///
   /// If the file is missing, this visitor is marked as [containsNewDepfile].
@@ -206,9 +195,6 @@ abstract class Source {
   /// environment variables.
   const factory Source.pattern(String pattern, { bool optional }) = _PatternSource;
 
-  /// This source is produced by invoking the provided function.
-  const factory Source.function(InputFunction function) = _FunctionSource;
-
   /// This source is produced by the [SourceBehavior] class.
   const factory Source.behavior(SourceBehavior behavior) = _SourceBehavior;
 
@@ -264,18 +250,6 @@ class _SourceBehavior implements Source {
   bool get implicit => true;
 }
 
-class _FunctionSource implements Source {
-  const _FunctionSource(this.value);
-
-  final InputFunction value;
-
-  @override
-  void accept(SourceVisitor visitor) => visitor.visitFunction(value);
-
-  @override
-  bool get implicit => true;
-}
-
 class _PatternSource implements Source {
   const _PatternSource(this.value, { this.optional = false });
 

packages/flutter_tools/lib/src/build_system/targets/web.dart

@@ -12,6 +12,7 @@ import '../../dart/package_map.dart';
 import '../../globals.dart';
 import '../../project.dart';
 import '../build_system.dart';
+import '../depfile.dart';
 import 'assets.dart';
 import 'dart.dart';
 
@@ -108,18 +109,17 @@ class Dart2JSTarget extends Target {
 
   @override
   List<Source> get inputs => const <Source>[
-    Source.pattern('{FLUTTER_ROOT}/packages/flutter_tools/lib/src/build_system/targets/web.dart'),
     Source.artifact(Artifact.flutterWebSdk),
     Source.artifact(Artifact.dart2jsSnapshot),
     Source.artifact(Artifact.engineDartBinary),
     Source.pattern('{BUILD_DIR}/main.dart'),
     Source.pattern('{PROJECT_DIR}/.packages'),
-    Source.function(listDartSources), // <- every dart file under {PROJECT_DIR}/lib and in .packages
+    Source.depfile('dart2js.d'),
   ];
 
   @override
   List<Source> get outputs => const <Source>[
-    Source.pattern('{BUILD_DIR}/main.dart.js'),
+    Source.depfile('dart2js.d'),
   ];
 
   @override
@@ -130,6 +130,7 @@ class Dart2JSTarget extends Target {
     final String packageFile = FlutterProject.fromDirectory(environment.projectDir).hasBuilders
       ? PackageMap.globalGeneratedPackagesPath
       : PackageMap.globalPackagesPath;
+    final File outputFile = environment.buildDir.childFile('main.dart.js');
     final ProcessResult result = await processManager.run(<String>[
       artifacts.getArtifactPath(Artifact.engineDartBinary),
       artifacts.getArtifactPath(Artifact.dart2jsSnapshot),
@@ -141,7 +142,7 @@ class Dart2JSTarget extends Target {
       else
         '-O4',
       '-o',
-      environment.buildDir.childFile('main.dart.js').path,
+      outputFile.path,
       '--packages=$packageFile',
       if (buildMode == BuildMode.profile)
         '-Ddart.vm.profile=true'
@@ -152,6 +153,18 @@ class Dart2JSTarget extends Target {
     if (result.exitCode != 0) {
       throw Exception(result.stdout + result.stderr);
     }
+    final File dart2jsDeps = environment.buildDir
+      .childFile('main.dart.js.deps');
+    if (!dart2jsDeps.existsSync()) {
+      printError('Warning: dart2js did not produced expected deps list at '
+        '${dart2jsDeps.path}');
+      return;
+    }
+    final Depfile depfile = Depfile.parseDart2js(
+      environment.buildDir.childFile('main.dart.js.deps'),
+      outputFile,
+    );
+    depfile.writeToFile(environment.buildDir.childFile('dart2js.d'));
   }
 }
 
@@ -170,7 +183,6 @@ class WebReleaseBundle extends Target {
   @override
   List<Source> get inputs => const <Source>[
     Source.pattern('{BUILD_DIR}/main.dart.js'),
-    Source.pattern('{FLUTTER_ROOT}/packages/flutter_tools/lib/src/build_system/targets/web.dart'),
     Source.behavior(AssetOutputBehavior('assets')),
     Source.pattern('{PROJECT_DIR}/web/index.html'),
   ];

packages/flutter_tools/lib/src/web/compile.dart

@@ -30,27 +30,32 @@ Future<void> buildWeb(FlutterProject flutterProject, String target, BuildInfo bu
   await injectPlugins(flutterProject, checkProjects: true);
   final Status status = logger.startProgress('Compiling $target for the Web...', timeout: null);
   final Stopwatch sw = Stopwatch()..start();
-  final BuildResult result = await buildSystem.build(const WebReleaseBundle(), Environment(
-    outputDir: fs.directory(getWebBuildDirectory()),
-    projectDir: fs.currentDirectory,
-    buildDir: flutterProject.directory
-      .childDirectory('.dart_tool')
-      .childDirectory('flutter_build'),
-    defines: <String, String>{
-      kBuildMode: getNameForBuildMode(buildInfo.mode),
-      kTargetFile: target,
-      kInitializePlatform: initializePlatform.toString(),
-      kHasWebPlugins: hasWebPlugins.toString(),
-    },
-  ));
-  if (!result.success) {
-    for (ExceptionMeasurement measurement in result.exceptions.values) {
-      printError(measurement.stackTrace.toString());
-      printError(measurement.exception.toString());
+  try {
+    final BuildResult result = await buildSystem.build(const WebReleaseBundle(), Environment(
+      outputDir: fs.directory(getWebBuildDirectory()),
+      projectDir: fs.currentDirectory,
+      buildDir: flutterProject.directory
+        .childDirectory('.dart_tool')
+        .childDirectory('flutter_build'),
+      defines: <String, String>{
+        kBuildMode: getNameForBuildMode(buildInfo.mode),
+        kTargetFile: target,
+        kInitializePlatform: initializePlatform.toString(),
+        kHasWebPlugins: hasWebPlugins.toString(),
+      },
+    ));
+    if (!result.success) {
+      for (ExceptionMeasurement measurement in result.exceptions.values) {
+        printError(measurement.stackTrace.toString());
+        printError(measurement.exception.toString());
+      }
+      throwToolExit('Failed to compile application for the Web.');
     }
-    throwToolExit('Failed to compile application for the Web.');
+  } catch (err) {
+    throwToolExit(err.toString());
+  } finally {
+    status.stop();
   }
-  status.stop();
   flutterUsage.sendTiming('build', 'dart2js', Duration(milliseconds: sw.elapsedMilliseconds));
 }
 

packages/flutter_tools/test/general.shard/build_system/build_system_test.dart

@@ -114,7 +114,7 @@ void main() {
     expect(buildResult.hasException, false);
   }));
 
-  test('Throws exception if it does not produce a specified output', () => testbed.run(() async {
+  test('Does not throw exception if it does not produce a specified output', () => testbed.run(() async {
     final Target badTarget = TestTarget((Environment environment) async {})
       ..inputs = const <Source>[
         Source.pattern('{PROJECT_DIR}/foo.dart'),
@@ -124,8 +124,7 @@ void main() {
       ];
     final BuildResult result = await buildSystem.build(badTarget, environment);
 
-    expect(result.hasException, true);
-    expect(result.exceptions.values.single.exception, isInstanceOf<FileSystemException>());
+    expect(result.hasException, false);
   }));
 
   test('Saves a stamp file with inputs and outputs', () => testbed.run(() async {

packages/flutter_tools/test/general.shard/build_system/depfile_test.dart

@@ -26,7 +26,6 @@ a.txt: b.txt
   }));
 
   test('Can parse depfile with multiple inputs', () => testbed.run(() {
-    final FileSystem fs = MemoryFileSystem();
     final File depfileSource = fs.file('example.d')..writeAsStringSync('''
 a.txt: b.txt c.txt d.txt
 ''');
@@ -41,7 +40,6 @@ a.txt: b.txt c.txt d.txt
   }));
 
   test('Can parse depfile with multiple outputs', () => testbed.run(() {
-    final FileSystem fs = MemoryFileSystem();
     final File depfileSource = fs.file('example.d')..writeAsStringSync('''
 a.txt c.txt d.txt: b.txt
 ''');
@@ -56,7 +54,6 @@ a.txt c.txt d.txt: b.txt
   }));
 
   test('Can parse depfile with windows file paths', () => testbed.run(() {
-    final FileSystem fs = MemoryFileSystem();
     final File depfileSource = fs.file('example.d')..writeAsStringSync(r'''
 C:\\a.txt: C:\\b.txt
 ''');
@@ -69,7 +66,6 @@ C:\\a.txt: C:\\b.txt
   }));
 
   test('Resillient to weird whitespace', () => testbed.run(() {
-    final FileSystem fs = MemoryFileSystem();
     final File depfileSource = fs.file('example.d')..writeAsStringSync(r'''
 a.txt
   : b.txt    c.txt
@@ -83,7 +79,6 @@ a.txt
   }));
 
   test('Resillient to duplicate files', () => testbed.run(() {
-    final FileSystem fs = MemoryFileSystem();
     final File depfileSource = fs.file('example.d')..writeAsStringSync(r'''
 a.txt: b.txt b.txt
 ''');
@@ -94,7 +89,6 @@ a.txt: b.txt b.txt
   }));
 
   test('Resillient to malformed file, missing :', () => testbed.run(() {
-    final FileSystem fs = MemoryFileSystem();
     final File depfileSource = fs.file('example.d')..writeAsStringSync(r'''
 a.text b.txt
 ''');
@@ -103,4 +97,42 @@ a.text b.txt
     expect(depfile.inputs, isEmpty);
     expect(depfile.outputs, isEmpty);
   }));
+
+  test('Can parse dart2js output format', () => testbed.run(() {
+    final File dart2jsDependencyFile = fs.file('main.dart.js.deps')..writeAsStringSync(r'''
+file:///Users/foo/collection.dart
+file:///Users/foo/algorithms.dart
+file:///Users/foo/canonicalized_map.dart
+''');
+
+    final Depfile depfile = Depfile.parseDart2js(dart2jsDependencyFile, fs.file('foo.dart.js'));
+
+    expect(depfile.inputs.map((File file) => file.path), <String>[
+      fs.path.absolute(fs.path.join('Users', 'foo', 'collection.dart')),
+      fs.path.absolute(fs.path.join('Users', 'foo', 'algorithms.dart')),
+      fs.path.absolute(fs.path.join('Users', 'foo', 'canonicalized_map.dart')),
+    ]);
+    expect(depfile.outputs.single.path, 'foo.dart.js');
+  }, overrides: <Type, Generator>{
+    FileSystem: () => MemoryFileSystem(style: FileSystemStyle.posix)
+  }));
+
+  test('Can parse handle invalid uri', () => testbed.run(() {
+    final File dart2jsDependencyFile = fs.file('main.dart.js.deps')..writeAsStringSync('''
+file:///Users/foo/collection.dart
+abcdevf
+file:///Users/foo/canonicalized_map.dart
+''');
+
+    final Depfile depfile = Depfile.parseDart2js(dart2jsDependencyFile, fs.file('foo.dart.js'));
+
+    expect(depfile.inputs.map((File file) => file.path), <String>[
+      fs.path.absolute(fs.path.join('Users', 'foo', 'collection.dart')),
+      fs.path.absolute(fs.path.join('Users', 'foo', 'canonicalized_map.dart')),
+    ]);
+    expect(depfile.outputs.single.path, 'foo.dart.js');
+  }, overrides: <Type, Generator>{
+    FileSystem: () => MemoryFileSystem(style: FileSystemStyle.posix)
+  }));
 }
+

packages/flutter_tools/test/general.shard/build_system/filecache_test.dart

@@ -81,4 +81,15 @@ void main() {
     // Does not throw.
     fileCache.persist();
   }));
+
+  test('handles hashing missing files', () => testbed.run(() async {
+    final FileHashStore fileCache = FileHashStore(environment);
+    fileCache.initialize();
+
+    final List<File> results = await fileCache.hashFiles(<File>[fs.file('hello.dart')]);
+
+    expect(results, hasLength(1));
+    expect(results.single.path, 'hello.dart');
+    expect(fileCache.currentHashes, isNot(contains(fs.path.absolute('hello.dart'))));
+  }));
 }

packages/flutter_tools/test/general.shard/build_system/source_test.dart

@@ -41,7 +41,6 @@ void main() {
   test('configures implicit vs explict correctly', () => testbed.run(() {
     expect(const Source.pattern('{PROJECT_DIR}/foo').implicit, false);
     expect(const Source.pattern('{PROJECT_DIR}/*foo').implicit, true);
-    expect(Source.function((Environment environment) => <File>[]).implicit, true);
     expect(Source.behavior(TestBehavior()).implicit, true);
   }));
 

packages/flutter_tools/test/general.shard/build_system/targets/web_test.dart

@@ -6,6 +6,7 @@ import 'package:flutter_tools/src/base/file_system.dart';
 import 'package:flutter_tools/src/base/platform.dart';
 import 'package:flutter_tools/src/base/process_manager.dart';
 import 'package:flutter_tools/src/build_system/build_system.dart';
+import 'package:flutter_tools/src/build_system/depfile.dart';
 import 'package:flutter_tools/src/build_system/targets/dart.dart';
 import 'package:flutter_tools/src/build_system/targets/web.dart';
 import 'package:mockito/mockito.dart';
@@ -215,6 +216,25 @@ void main() {
   }, overrides: <Type, Generator>{
     ProcessManager: () => MockProcessManager(),
   }));
+
+  test('Dart2JSTarget produces expected depfile', () => testbed.run(() async {
+    environment.defines[kBuildMode] = 'release';
+    when(processManager.run(any)).thenAnswer((Invocation invocation) async {
+      environment.buildDir.childFile('main.dart.js.deps')
+        ..writeAsStringSync('file:///a.dart');
+      return FakeProcessResult(exitCode: 0);
+    });
+    await const Dart2JSTarget().build(environment);
+
+    expect(environment.buildDir.childFile('dart2js.d').existsSync(), true);
+    final Depfile depfile = Depfile.parse(environment.buildDir.childFile('dart2js.d'));
+
+    expect(depfile.inputs.single.path, fs.path.absolute('a.dart'));
+    expect(depfile.outputs.single.path,
+      environment.buildDir.childFile('main.dart.js').absolute.path);
+  }, overrides: <Type, Generator>{
+    ProcessManager: () => MockProcessManager(),
+  }));
 }
 
 class MockProcessManager extends Mock implements ProcessManager {}