Support URL tunnelling (pass dwds UrlEncoder through to editors via daemon) (#44271)

* Prposal for supporting URL tunnelling

* Update daemon.md

* Add the ability for daemon to call clients to expose URLs

* Fix dwds mock in web_fs tests

* Fix type error

* Remove build_runner import from run

* Move appStartedTime back to after the app has started

* Remove nested DI scope and pass urlTunneller down

* Fix import

* Tweak TODO

* Fix existing tests

* Fix spec to use result instead of params for response object

* Fix exposeUrl to use a url field, as spec'd

* Test that the daemon's exposeUrl sends a request and handles the response

packages/flutter_tools/doc/daemon.md

@@ -166,6 +166,22 @@ This is sent when an app is stopped or detached from. The `params` field will be
 
 This is sent once a web application is being served and available for the user to access. The `params` field will be a map with a string `url` field and a boolean `launched` indicating whether the application has already been launched in a browser (this will generally be true for a browser device unless `--no-web-browser-launch` was used, and false for the headless `web-server` device).
 
+### Daemon-to-Editor Requests
+
+These requests come _from_ the Flutter daemon and should be responded to by the client/editor.
+
+#### app.exposeUrl
+
+This request is enabled only if `flutter run` is run with the `--web-allow-expose-url` flag.
+
+This request is sent by the server when it has a local URL that needs to be exposed to the end user. This is to support running on a remote machine where a URL (for example `http://localhost:1234`) may not be directly accessible to the end user. With this URL clients can perform tunnelling and then provide the tunneled URL back to Flutter so that it can be used in code that will be executed on the end users machine (for example wehen a web application needs to be able to connect back to a service like the DWDS debugging service).
+
+This request will only be sent if a web application was run in a mode that requires mapped URLs (such as using `--no-web-browser-launch` for browser devices or the headless `web-server` device when debugging).
+
+The request will contain an `id` field and a `params` field that is a map containing a string `url` field.
+
+The response should be sent using the same `id` as the request with a `result` map containing the mapped `url` (or the same URL in the case where the client does not need to perform any mapping).
+
 ### device domain
 
 #### device.getDevices

packages/flutter_tools/lib/src/base/net.dart

@@ -16,6 +16,8 @@ const int kNetworkProblemExitCode = 50;
 
 typedef HttpClientFactory = HttpClient Function();
 
+typedef UrlTunneller = Future<String> Function(String url);
+
 /// Download a file from the given URL.
 ///
 /// If a destination file is not provided, returns the bytes.

packages/flutter_tools/lib/src/build_runner/resident_web_runner.dart

@@ -17,6 +17,7 @@ import '../base/common.dart';
 import '../base/file_system.dart';
 import '../base/io.dart';
 import '../base/logger.dart';
+import '../base/net.dart';
 import '../base/os.dart';
 import '../base/terminal.dart';
 import '../base/utils.dart';
@@ -47,6 +48,7 @@ class DwdsWebRunnerFactory extends WebRunnerFactory {
     @required bool ipv6,
     @required DebuggingOptions debuggingOptions,
     @required List<String> dartDefines,
+    @required UrlTunneller urlTunneller,
   }) {
     if (featureFlags.isWebIncrementalCompilerEnabled && debuggingOptions.buildInfo.isDebug) {
       return _ExperimentalResidentWebRunner(
@@ -57,6 +59,7 @@ class DwdsWebRunnerFactory extends WebRunnerFactory {
         ipv6: ipv6,
         stayResident: stayResident,
         dartDefines: dartDefines,
+        // TODO(dantup): If this becomes default it may need to urlTunneller.
       );
     }
     return _DwdsResidentWebRunner(
@@ -67,6 +70,7 @@ class DwdsWebRunnerFactory extends WebRunnerFactory {
       ipv6: ipv6,
       stayResident: stayResident,
       dartDefines: dartDefines,
+      urlTunneller: urlTunneller,
     );
   }
 }
@@ -550,6 +554,7 @@ class _DwdsResidentWebRunner extends ResidentWebRunner {
     @required FlutterProject flutterProject,
     @required bool ipv6,
     @required DebuggingOptions debuggingOptions,
+    @required this.urlTunneller,
     bool stayResident = true,
     @required List<String> dartDefines,
   }) : super(
@@ -562,6 +567,8 @@ class _DwdsResidentWebRunner extends ResidentWebRunner {
           dartDefines: dartDefines,
         );
 
+  UrlTunneller urlTunneller;
+
   @override
   Future<int> run({
     Completer<DebugConnectionInfo> connectionInfoCompleter,
@@ -606,6 +613,7 @@ class _DwdsResidentWebRunner extends ResidentWebRunner {
           initializePlatform: debuggingOptions.initializePlatform,
           hostname: debuggingOptions.hostname,
           port: debuggingOptions.port,
+          urlTunneller: urlTunneller,
           skipDwds: !_enableDwds,
           dartDefines: dartDefines,
         );

packages/flutter_tools/lib/src/build_runner/web_fs.dart

@@ -26,6 +26,7 @@ import '../base/common.dart';
 import '../base/context.dart';
 import '../base/file_system.dart';
 import '../base/io.dart';
+import '../base/net.dart';
 import '../base/os.dart';
 import '../base/platform.dart';
 import '../build_info.dart';
@@ -69,6 +70,7 @@ typedef DwdsFactory = Future<Dwds> Function({
   LogWriter logWriter,
   bool verbose,
   bool enableDebugExtension,
+  UrlEncoder urlEncoder,
 });
 
 /// A function with the same signature as [WebFs.start].
@@ -80,6 +82,7 @@ typedef WebFsFactory = Future<WebFs> Function({
   @required bool initializePlatform,
   @required String hostname,
   @required String port,
+  @required UrlTunneller urlTunneller,
   @required List<String> dartDefines,
 });
 
@@ -175,6 +178,7 @@ class WebFs {
     @required bool initializePlatform,
     @required String hostname,
     @required String port,
+    @required UrlTunneller urlTunneller,
     @required List<String> dartDefines,
   }) async {
     // workaround for https://github.com/flutter/flutter/issues/38290
@@ -298,6 +302,7 @@ class WebFs {
           serveDevTools: false,
           verbose: false,
           enableDebugExtension: true,
+          urlEncoder: urlTunneller,
           logWriter: (dynamic level, String message) => printTrace(message),
         );
         handler = pipeline.addHandler(dwds.handler);

packages/flutter_tools/lib/src/commands/daemon.dart

@@ -119,6 +119,8 @@ class Daemon {
   DeviceDomain deviceDomain;
   EmulatorDomain emulatorDomain;
   StreamSubscription<Map<String, dynamic>> _commandSubscription;
+  int _outgoingRequestId = 1;
+  final Map<String, Completer<dynamic>> _outgoingRequestCompleters = <String, Completer<dynamic>>{};
 
   final DispatchCommand sendCommand;
   final NotifyingLogger notifyingLogger;
@@ -147,17 +149,32 @@ class Daemon {
 
     try {
       final String method = request['method'] as String;
-      if (!method.contains('.')) {
-        throw 'method not understood: $method';
-      }
+      if (method != null) {
+        if (!method.contains('.')) {
+          throw 'method not understood: $method';
+        }
 
-      final String prefix = method.substring(0, method.indexOf('.'));
-      final String name = method.substring(method.indexOf('.') + 1);
-      if (_domainMap[prefix] == null) {
-        throw 'no domain for method: $method';
-      }
+        final String prefix = method.substring(0, method.indexOf('.'));
+        final String name = method.substring(method.indexOf('.') + 1);
+        if (_domainMap[prefix] == null) {
+          throw 'no domain for method: $method';
+        }
+
+        _domainMap[prefix].handleCommand(name, id, castStringKeyedMap(request['params']) ?? const <String, dynamic>{});
+      } else {
+        // If there was no 'method' field then it's a response to a daemon-to-editor request.
+        final Completer<dynamic> completer = _outgoingRequestCompleters[id.toString()];
+        if (completer == null) {
+          throw 'unexpected response with id: $id';
+        }
+        _outgoingRequestCompleters.remove(id.toString());
 
-      _domainMap[prefix].handleCommand(name, id, castStringKeyedMap(request['params']) ?? const <String, dynamic>{});
+        if (request['error'] != null) {
+          completer.completeError(request['error']);
+        } else {
+          completer.complete(request['result']);
+        }
+      }
     } catch (error, trace) {
       _send(<String, dynamic>{
         'id': id,
@@ -167,6 +184,22 @@ class Daemon {
     }
   }
 
+  Future<dynamic> sendRequest(String method, [ dynamic args ]) {
+    final Map<String, dynamic> map = <String, dynamic>{'method': method};
+    if (args != null) {
+      map['params'] = _toJsonable(args);
+    }
+
+    final int id = _outgoingRequestId++;
+    final Completer<dynamic> completer = Completer<dynamic>();
+
+    map['id'] = id.toString();
+    _outgoingRequestCompleters[id.toString()] = completer;
+
+    _send(map);
+    return completer.future;
+  }
+
   void _send(Map<String, dynamic> map) => sendCommand(map);
 
   void shutdown({ dynamic error }) {
@@ -187,6 +220,7 @@ class Daemon {
 abstract class Domain {
   Domain(this.daemon, this.name);
 
+
   final Daemon daemon;
   final String name;
   final Map<String, CommandHandler> _handlers = <String, CommandHandler>{};
@@ -317,6 +351,21 @@ class DaemonDomain extends Domain {
     return Future<String>.value(protocolVersion);
   }
 
+  /// Sends a request back to the client asking it to expose/tunnel a URL.
+  ///
+  /// This method should only be called if the client opted-in with the
+  /// --web-allow-expose-url switch. The client may return the same URL back if
+  /// tunnelling is not required for a given URL.
+  Future<String> exposeUrl(String url) async {
+    final dynamic res = await daemon.sendRequest('app.exposeUrl', <String, String>{'url': url});
+    if (res is Map<String, dynamic> && res['url'] is String) {
+      return res['url'] as String;
+    } else {
+      printError('Invalid response to exposeUrl - params should include a String url field');
+      return url;
+    }
+  }
+
   Future<void> shutdown(Map<String, dynamic> args) {
     Timer.run(daemon.shutdown);
     return Future<void>.value();
@@ -448,6 +497,7 @@ class AppDomain extends Domain {
         ipv6: ipv6,
         stayResident: true,
         dartDefines: daemon.dartDefines,
+        urlTunneller: options.webEnableExposeUrl ? daemon.daemonDomain.exposeUrl : null,
       );
     } else if (enableHotReload) {
       runner = HotRunner(

packages/flutter_tools/lib/src/commands/run.dart

@@ -314,6 +314,7 @@ class RunCommand extends RunCommandBase {
         initializePlatform: boolArg('web-initialize-platform'),
         hostname: featureFlags.isWebEnabled ? stringArg('web-hostname') : '',
         port: featureFlags.isWebEnabled ? stringArg('web-port') : '',
+        webEnableExposeUrl: featureFlags.isWebEnabled && boolArg('web-allow-expose-url'),
       );
     } else {
       return DebuggingOptions.enabled(
@@ -334,6 +335,7 @@ class RunCommand extends RunCommandBase {
         initializePlatform: boolArg('web-initialize-platform'),
         hostname: featureFlags.isWebEnabled ? stringArg('web-hostname') : '',
         port: featureFlags.isWebEnabled ? stringArg('web-port') : '',
+        webEnableExposeUrl: featureFlags.isWebEnabled && boolArg('web-allow-expose-url'),
         vmserviceOutFile: stringArg('vmservice-out-file'),
         // Allow forcing fast-start to off to prevent doing more work on devices that
         // don't support it.
@@ -489,6 +491,7 @@ class RunCommand extends RunCommandBase {
         debuggingOptions: _createDebuggingOptions(),
         stayResident: stayResident,
         dartDefines: dartDefines,
+        urlTunneller: null,
       );
     } else {
       runner = ColdRunner(

packages/flutter_tools/lib/src/device.dart

@@ -536,12 +536,18 @@ class DebuggingOptions {
     this.initializePlatform = true,
     this.hostname,
     this.port,
+    this.webEnableExposeUrl,
     this.vmserviceOutFile,
     this.fastStart = false,
    }) : debuggingEnabled = true;
 
-  DebuggingOptions.disabled(this.buildInfo, { this.initializePlatform = true, this.port, this.hostname, this.cacheSkSL = false, })
-    : debuggingEnabled = false,
+  DebuggingOptions.disabled(this.buildInfo, {
+      this.initializePlatform = true,
+      this.port,
+      this.hostname,
+      this.webEnableExposeUrl,
+      this.cacheSkSL = false,
+    }) : debuggingEnabled = false,
       useTestFonts = false,
       startPaused = false,
       dartFlags = '',
@@ -577,6 +583,7 @@ class DebuggingOptions {
   final int deviceVmServicePort;
   final String port;
   final String hostname;
+  final bool webEnableExposeUrl;
   /// A file where the vmservice URL should be written after the application is started.
   final String vmserviceOutFile;
   final bool fastStart;

packages/flutter_tools/lib/src/runner/flutter_command.dart

@@ -150,6 +150,12 @@ abstract class FlutterCommand extends Command<void> {
         'will select a random open port on the host.',
       hide: hide,
     );
+    argParser.addFlag('web-allow-expose-url',
+      defaultsTo: false,
+      help: 'Enables daemon-to-editor requests (app.exposeUrl) for exposing URLs '
+        'when running on remote machines.',
+      hide: hide,
+    );
   }
 
   void usesTargetOption() {

packages/flutter_tools/lib/src/web/web_runner.dart

@@ -5,6 +5,7 @@
 import 'package:meta/meta.dart';
 
 import '../base/context.dart';
+import '../base/net.dart';
 import '../device.dart';
 import '../project.dart';
 import '../resident_runner.dart';
@@ -24,5 +25,6 @@ abstract class WebRunnerFactory {
     @required bool ipv6,
     @required DebuggingOptions debuggingOptions,
     @required List<String> dartDefines,
+    @required UrlTunneller urlTunneller,
   });
 }

packages/flutter_tools/test/commands.shard/hermetic/build_web_test.dart

@@ -71,6 +71,7 @@ void main() {
       debuggingOptions: DebuggingOptions.enabled(BuildInfo.debug),
       stayResident: true,
       dartDefines: const <String>[],
+      urlTunneller: null,
     ) as ResidentWebRunner;
     expect(await runner.run(), 1);
   }));

packages/flutter_tools/test/commands.shard/hermetic/daemon_test.dart

@@ -12,6 +12,7 @@ import 'package:flutter_tools/src/fuchsia/fuchsia_workflow.dart';
 import 'package:flutter_tools/src/globals.dart';
 import 'package:flutter_tools/src/ios/ios_workflow.dart';
 import 'package:flutter_tools/src/resident_runner.dart';
+import 'package:pedantic/pedantic.dart';
 
 import '../../src/common.dart';
 import '../../src/context.dart';
@@ -279,6 +280,35 @@ void main() {
       await responses.close();
       await commands.close();
     });
+
+    testUsingContext('daemon can send exposeUrl requests to the client', () async {
+      const String originalUrl = 'http://localhost:1234/';
+      const String mappedUrl = 'https://publichost:4321/';
+      final StreamController<Map<String, dynamic>> input = StreamController<Map<String, dynamic>>();
+      final StreamController<Map<String, dynamic>> output = StreamController<Map<String, dynamic>>();
+
+      daemon = Daemon(
+        input.stream,
+        output.add,
+        notifyingLogger: notifyingLogger,
+        dartDefines: const <String>[],
+      );
+
+      // Respond to any requests from the daemon to expose a URL.
+      unawaited(output.stream
+        .firstWhere((Map<String, dynamic> request) => request['method'] == 'app.exposeUrl')
+        .then((Map<String, dynamic> request) {
+          expect(request['params']['url'], equals(originalUrl));
+          input.add(<String, dynamic>{'id': request['id'], 'result': <String, dynamic>{'url': mappedUrl}});
+        })
+      );
+
+      final String exposedUrl = await daemon.daemonDomain.exposeUrl(originalUrl);
+      expect(exposedUrl, equals(mappedUrl));
+
+      await output.close();
+      await input.close();
+    });
   });
 
   group('daemon serialization', () {

packages/flutter_tools/test/commands.shard/hermetic/run_test.dart

@@ -14,6 +14,7 @@ import 'package:flutter_tools/src/base/context.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
 import 'package:flutter_tools/src/base/logger.dart';
 import 'package:flutter_tools/src/base/user_messages.dart';
+import 'package:flutter_tools/src/base/net.dart';
 import 'package:flutter_tools/src/build_info.dart';
 import 'package:flutter_tools/src/cache.dart';
 import 'package:flutter_tools/src/commands/run.dart';
@@ -629,6 +630,7 @@ class MockWebRunnerFactory extends Mock implements WebRunnerFactory {
     bool ipv6,
     DebuggingOptions debuggingOptions,
     List<String> dartDefines,
+    UrlTunneller urlTunneller,
   }) {
     _dartDefines = dartDefines;
     return MockWebRunner();

packages/flutter_tools/test/general.shard/resident_web_runner_cold_test.dart

@@ -8,6 +8,7 @@ import 'package:dwds/dwds.dart';
 import 'package:flutter_tools/src/base/common.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
 import 'package:flutter_tools/src/base/logger.dart';
+import 'package:flutter_tools/src/base/net.dart';
 import 'package:flutter_tools/src/build_info.dart';
 import 'package:flutter_tools/src/device.dart';
 import 'package:flutter_tools/src/globals.dart';
@@ -35,13 +36,14 @@ void main() {
     when(mockFlutterDevice.device).thenReturn(mockWebDevice);
     testbed = Testbed(
       setup: () {
-        residentWebRunner =  residentWebRunner = DwdsWebRunnerFactory().createWebRunner(
+        residentWebRunner = residentWebRunner = DwdsWebRunnerFactory().createWebRunner(
           mockFlutterDevice,
           flutterProject: FlutterProject.current(),
           debuggingOptions: DebuggingOptions.disabled(BuildInfo.release),
           ipv6: true,
           stayResident: true,
           dartDefines: const <String>[],
+          urlTunneller: null,
         ) as ResidentWebRunner;
       },
       overrides: <Type, Generator>{
@@ -53,6 +55,7 @@ void main() {
           @required bool initializePlatform,
           @required String hostname,
           @required String port,
+          @required UrlTunneller urlTunneller,
           @required List<String> dartDefines,
         }) async {
           return mockWebFs;

packages/flutter_tools/test/general.shard/resident_web_runner_test.dart

@@ -11,6 +11,7 @@ import 'package:flutter_tools/src/base/common.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
 import 'package:flutter_tools/src/base/io.dart';
 import 'package:flutter_tools/src/base/logger.dart';
+import 'package:flutter_tools/src/base/net.dart';
 import 'package:flutter_tools/src/build_info.dart';
 import 'package:flutter_tools/src/build_runner/resident_web_runner.dart';
 import 'package:flutter_tools/src/build_runner/web_fs.dart';
@@ -78,6 +79,7 @@ void main() {
           ipv6: true,
           stayResident: true,
           dartDefines: const <String>[],
+          urlTunneller: null,
         ) as ResidentWebRunner;
       },
       overrides: <Type, Generator>{
@@ -90,6 +92,7 @@ void main() {
           @required String hostname,
           @required String port,
           @required List<String> dartDefines,
+          @required UrlTunneller urlTunneller,
         }) async {
           didSkipDwds = skipDwds;
           return mockWebFs;
@@ -142,6 +145,7 @@ void main() {
       ipv6: true,
       stayResident: true,
       dartDefines: const <String>[],
+      urlTunneller: null,
     ) as ResidentWebRunner;
 
     expect(profileResidentWebRunner.debuggingEnabled, false);
@@ -172,6 +176,7 @@ void main() {
       ipv6: true,
       stayResident: true,
       dartDefines: <String>[],
+      urlTunneller: null,
     );
 
     expect(profileResidentWebRunner.debuggingEnabled, true);
@@ -187,6 +192,7 @@ void main() {
       ipv6: true,
       stayResident: true,
       dartDefines: <String>[],
+      urlTunneller: null,
     ) as ResidentWebRunner;
 
     final Completer<DebugConnectionInfo> connectionInfoCompleter = Completer<DebugConnectionInfo>();
@@ -210,6 +216,7 @@ void main() {
       ipv6: true,
       stayResident: true,
       dartDefines: const <String>[],
+      urlTunneller: null,
     );
 
     expect(profileResidentWebRunner.supportsServiceProtocol, false);
@@ -263,6 +270,7 @@ void main() {
       ipv6: true,
       stayResident: false,
       dartDefines: const <String>[],
+      urlTunneller: null,
     ) as ResidentWebRunner;
 
     expect(await residentWebRunner.run(), 0);
@@ -299,6 +307,7 @@ void main() {
       ipv6: true,
       stayResident: true,
       dartDefines: const <String>[],
+      urlTunneller: null,
     ) as ResidentWebRunner;
     _setupMocks();
     final Completer<DebugConnectionInfo> connectionInfoCompleter = Completer<DebugConnectionInfo>();
@@ -513,6 +522,7 @@ void main() {
       ipv6: true,
       stayResident: true,
       dartDefines: const <String>[],
+      urlTunneller: null,
     ) as ResidentWebRunner;
 
     expect(residentWebRunner.runtimeType.toString(), '_DwdsResidentWebRunner');
@@ -830,6 +840,7 @@ void main() {
       ipv6: true,
       stayResident: true,
       dartDefines: const <String>[],
+      urlTunneller: null,
     ) as ResidentWebRunner;
 
     final Completer<DebugConnectionInfo> connectionInfoCompleter = Completer<DebugConnectionInfo>();
@@ -865,6 +876,7 @@ void main() {
       ipv6: true,
       stayResident: true,
       dartDefines: const <String>[],
+      urlTunneller: null,
     ) as ResidentWebRunner;
 
     final Completer<DebugConnectionInfo> connectionInfoCompleter = Completer<DebugConnectionInfo>();

packages/flutter_tools/test/general.shard/web/web_fs_test.dart

@@ -109,6 +109,7 @@ void main() {
           LogWriter logWriter,
           bool verbose,
           bool enableDebugExtension,
+          UrlEncoder urlEncoder,
         }) async {
           return mockDwds;
         },
@@ -126,6 +127,7 @@ void main() {
       initializePlatform: true,
       hostname: null,
       port: null,
+      urlTunneller: null,
       dartDefines: const <String>[],
     );
     // Since the .packages file is missing in the memory filesystem, this should
@@ -156,6 +158,7 @@ void main() {
       initializePlatform: false,
       hostname: null,
       port: null,
+      urlTunneller: null,
       dartDefines: const <String>[],
     );
 
@@ -177,6 +180,7 @@ void main() {
       initializePlatform: false,
       hostname: 'foo',
       port: '1234',
+      urlTunneller: null,
       dartDefines: const <String>[],
     );
 
@@ -210,6 +214,7 @@ void main() {
       initializePlatform: false,
       hostname: 'foo',
       port: '1234',
+      urlTunneller: null,
       dartDefines: const <String>[],
     ), throwsA(isInstanceOf<Exception>()));
   }));