Use the proper charset for decoding HTTP responses (#3182)

Previously we always used Latin-1.

packages/flutter/lib/src/http/mojo_client.dart

@@ -7,12 +7,12 @@ import 'dart:convert';
 import 'dart:typed_data';
 
 import 'package:flutter/services.dart';
-import 'package:mojo_services/mojo/network_service.mojom.dart' as mojo;
-import 'package:mojo_services/mojo/url_loader.mojom.dart' as mojo;
+import 'package:mojo_services/mojo/network_service.mojom.dart' as mojom;
+import 'package:mojo_services/mojo/url_loader.mojom.dart' as mojom;
 import 'package:mojo/core.dart' as mojo;
-import 'package:mojo/mojo/url_request.mojom.dart' as mojo;
-import 'package:mojo/mojo/url_response.mojom.dart' as mojo;
-import 'package:mojo/mojo/http_header.mojom.dart' as mojo;
+import 'package:mojo/mojo/url_request.mojom.dart' as mojom;
+import 'package:mojo/mojo/url_response.mojom.dart' as mojom;
+import 'package:mojo/mojo/http_header.mojom.dart' as mojom;
 
 import 'response.dart';
 
@@ -124,15 +124,15 @@ class MojoClient {
   }
 
   Future<Response> _send(String method, dynamic url, Map<String, String> headers, [dynamic body, Encoding encoding = UTF8]) async {
-    mojo.UrlLoaderProxy loader = new mojo.UrlLoaderProxy.unbound();
-    List<mojo.HttpHeader> mojoHeaders = <mojo.HttpHeader>[];
+    mojom.UrlLoaderProxy loader = new mojom.UrlLoaderProxy.unbound();
+    List<mojom.HttpHeader> mojoHeaders = <mojom.HttpHeader>[];
     headers?.forEach((String name, String value) {
-      mojo.HttpHeader header = new mojo.HttpHeader()
+      mojom.HttpHeader header = new mojom.HttpHeader()
         ..name = name
         ..value = value;
       mojoHeaders.add(header);
     });
-    mojo.UrlRequest request = new mojo.UrlRequest()
+    mojom.UrlRequest request = new mojom.UrlRequest()
       ..url = url.toString()
       ..headers = mojoHeaders
       ..method = method;
@@ -145,10 +145,16 @@ class MojoClient {
     }
     try {
       networkService.ptr.createUrlLoader(loader);
-      mojo.UrlResponse response = (await loader.ptr.start(request)).response;
+      mojom.UrlResponse response = (await loader.ptr.start(request)).response;
       ByteData data = await mojo.DataPipeDrainer.drainHandle(response.body);
       Uint8List bodyBytes = new Uint8List.view(data.buffer);
-      return new Response(bodyBytes: bodyBytes, statusCode: response.statusCode);
+      Map<String, String> headers = <String, String>{};
+      for (mojom.HttpHeader header in response.headers) {
+        String headerName = header.name.toLowerCase();
+        String existingValue = headers[headerName];
+        headers[headerName] = existingValue != null ? '$existingValue, ${header.value}' : header.value;
+      }
+      return new Response.bytes(bodyBytes, response.statusCode, headers: headers);
     } catch (exception, stack) {
       FlutterError.reportError(new FlutterErrorDetails(
         exception: exception,
@@ -157,7 +163,7 @@ class MojoClient {
         context: 'while sending bytes to the Mojo network library',
         silent: true
       ));
-      return new Response(statusCode: 500);
+      return new Response.bytes(null, 500);
     } finally {
       loader.close();
     }
@@ -169,12 +175,12 @@ class MojoClient {
     throw new Exception("Request to $url failed with status ${response.statusCode}.");
   }
 
-  static mojo.NetworkServiceProxy _initNetworkService() {
-    mojo.NetworkServiceProxy proxy = new mojo.NetworkServiceProxy.unbound();
+  static mojom.NetworkServiceProxy _initNetworkService() {
+    mojom.NetworkServiceProxy proxy = new mojom.NetworkServiceProxy.unbound();
     shell.connectToService("mojo:authenticated_network_service", proxy);
     return proxy;
   }
 
   /// A handle to the [NetworkService] object used by [MojoClient].
-  static final mojo.NetworkServiceProxy networkService = _initNetworkService();
+  static final mojom.NetworkServiceProxy networkService = _initNetworkService();
 }

packages/flutter/lib/src/http/response.dart

@@ -2,6 +2,7 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:convert';
 import 'dart:typed_data';
 
 /// An HTTP response where the entire response body is known in advance.
@@ -9,16 +10,12 @@ class Response {
   /// Creates a [Response] object with the given fields.
   ///
   /// If [bodyBytes] is non-null, it is used to populate [body].
-  Response({
-    Uint8List bodyBytes,
-    this.statusCode
-  }) : body = bodyBytes != null ? new String.fromCharCodes(bodyBytes) : null,
-       bodyBytes = bodyBytes;
+  Response.bytes(this.bodyBytes, this.statusCode, { this.headers: const {} });
 
   /// The result of decoding [bodyBytes] using ISO-8859-1.
   ///
   /// If [bodyBytes] is null, this will also be null.
-  final String body;
+  String get body => _encodingForHeaders(headers).decode(bodyBytes);
 
   /// The raw byte stream.
   final Uint8List bodyBytes;
@@ -27,4 +24,67 @@ class Response {
   ///
   /// The code 500 is used when no status code could be obtained from the host.
   final int statusCode;
+
+  /// The headers for this response.
+  final Map<String, String> headers;
+}
+
+bool _isSpace(String c) {
+  return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f';
+}
+
+int _skipSpaces(String string, int index) {
+  while (index < string.length && _isSpace(string[index]))
+    index += 1;
+  return index;
+}
+
+// https://html.spec.whatwg.org/#algorithm-for-extracting-a-character-encoding-from-a-meta-element
+String _getCharset(String contentType) {
+  int index = 0;
+  while (index < contentType.length) {
+    index = contentType.indexOf(new RegExp(r'charset', caseSensitive: false), index);
+    if (index == -1)
+      return null;
+    index += 7;
+    index = _skipSpaces(contentType, index);
+    if (index >= contentType.length)
+      return null;
+    if (contentType[index] != '=')
+      continue;
+    index += 1;
+    index = _skipSpaces(contentType, index);
+    if (index >= contentType.length)
+      return null;
+    String delimiter = contentType[index];
+    if (delimiter == '"' || delimiter == '\'') {
+      index += 1;
+      if (index >= contentType.length)
+        return null;
+      int start = index;
+      int end = contentType.indexOf(delimiter, start);
+      if (end == -1)
+        return null;
+      return contentType.substring(start, end);
+    }
+    int start = index;
+    while (index < contentType.length) {
+      String c = contentType[index];
+      if (c == ' ' || c == ';')
+        break;
+      index += 1;
+    }
+    return contentType.substring(start, index);
+  }
+  return null;
+}
+
+Encoding _encodingForHeaders(Map<String, String> headers) {
+  String contentType = headers['content-type'];
+  if (contentType == null)
+    return LATIN1;
+  String charset = _getCharset(contentType);
+  if (charset == null)
+    return LATIN1;
+  return Encoding.getByName(charset) ?? LATIN1;
 }