Wrap dart:convert to track utf8 decode failures (#26650)

packages/flutter_tools/lib/src/android/android_device.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:meta/meta.dart';
 
@@ -18,6 +17,7 @@ import '../base/logger.dart';
 import '../base/process.dart';
 import '../base/process_manager.dart';
 import '../build_info.dart';
+import '../convert.dart';
 import '../device.dart';
 import '../globals.dart';
 import '../project.dart';
@@ -681,7 +681,9 @@ class _AdbLogReader extends DeviceLogReader {
       _timeOrigin = null;
     runCommand(device.adbCommandForDevice(args)).then<void>((Process process) {
       _process = process;
-      const Utf8Decoder decoder = Utf8Decoder(allowMalformed: true);
+      // We expect logcat streams to occasionally contain invalid utf-8,
+      // see: https://github.com/flutter/flutter/pull/8864.
+      const Utf8Decoder decoder = Utf8Decoder(reportErrors: false);
       _process.stdout.transform<String>(decoder).transform<String>(const LineSplitter()).listen(_onLine);
       _process.stderr.transform<String>(decoder).transform<String>(const LineSplitter()).listen(_onLine);
       _process.exitCode.whenComplete(() {

packages/flutter_tools/lib/src/android/android_sdk.dart

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:convert';
-
 import 'package:meta/meta.dart';
 
 import '../base/common.dart';
@@ -15,6 +13,7 @@ import '../base/platform.dart';
 import '../base/process.dart';
 import '../base/process_manager.dart';
 import '../base/version.dart';
+import '../convert.dart';
 import '../globals.dart';
 import 'android_studio.dart' as android_studio;
 

packages/flutter_tools/lib/src/android/android_workflow.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import '../base/common.dart';
 import '../base/context.dart';
@@ -14,6 +13,7 @@ import '../base/process_manager.dart';
 import '../base/user_messages.dart';
 import '../base/utils.dart';
 import '../base/version.dart';
+import '../convert.dart';
 import '../doctor.dart';
 import '../globals.dart';
 import 'android_sdk.dart';
@@ -256,13 +256,15 @@ class AndroidLicenseValidator extends DoctorValidator {
       environment: androidSdk.sdkManagerEnv,
     );
     process.stdin.write('n\n');
+    // We expect logcat streams to occasionally contain invalid utf-8,
+    // see: https://github.com/flutter/flutter/pull/8864.
     final Future<void> output = process.stdout
-      .transform<String>(const Utf8Decoder(allowMalformed: true))
+      .transform<String>(const Utf8Decoder(reportErrors: false))
       .transform<String>(const LineSplitter())
       .listen(_handleLine)
       .asFuture<void>(null);
     final Future<void> errors = process.stderr
-      .transform<String>(const Utf8Decoder(allowMalformed: true))
+      .transform<String>(const Utf8Decoder(reportErrors: false))
       .transform<String>(const LineSplitter())
       .listen(_handleLine)
       .asFuture<void>(null);

packages/flutter_tools/lib/src/android/gradle.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:archive/archive.dart';
 import 'package:meta/meta.dart';
@@ -20,6 +19,7 @@ import '../base/process.dart';
 import '../base/utils.dart';
 import '../build_info.dart';
 import '../cache.dart';
+import '../convert.dart';
 import '../flutter_manifest.dart';
 import '../globals.dart';
 import '../project.dart';

packages/flutter_tools/lib/src/asset.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:yaml/yaml.dart';
 
@@ -13,6 +12,7 @@ import 'base/platform.dart';
 import 'base/utils.dart';
 import 'build_info.dart';
 import 'cache.dart';
+import 'convert.dart';
 import 'dart/package_map.dart';
 import 'devfs.dart';
 import 'flutter_manifest.dart';

packages/flutter_tools/lib/src/base/config.dart

@@ -2,8 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:convert';
-
+import '../convert.dart';
 import 'context.dart';
 import 'file_system.dart';
 import 'platform.dart';

packages/flutter_tools/lib/src/base/fingerprint.dart

@@ -3,12 +3,12 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert' show json;
 
 import 'package:crypto/crypto.dart' show md5;
 import 'package:meta/meta.dart';
 import 'package:quiver/core.dart' show hash2;
 
+import '../convert.dart' show json;
 import '../globals.dart';
 import '../version.dart';
 import 'file_system.dart';

packages/flutter_tools/lib/src/base/process.dart

@@ -3,8 +3,8 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
+import '../convert.dart';
 import '../globals.dart';
 import 'file_system.dart';
 import 'io.dart';

packages/flutter_tools/lib/src/base/terminal.dart

@@ -3,8 +3,8 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert' show AsciiDecoder;
 
+import '../convert.dart';
 import '../globals.dart';
 import 'context.dart';
 import 'io.dart' as io;

packages/flutter_tools/lib/src/base/utils.dart

@@ -3,12 +3,12 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 import 'dart:math' show Random, max;
 
 import 'package:crypto/crypto.dart';
 import 'package:intl/intl.dart';
 
+import '../convert.dart';
 import '../globals.dart';
 import 'context.dart';
 import 'file_system.dart';

packages/flutter_tools/lib/src/build_runner/build_runner.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:meta/meta.dart';
 
@@ -14,6 +13,7 @@ import '../base/io.dart';
 import '../base/platform.dart';
 import '../base/process_manager.dart';
 import '../cache.dart';
+import '../convert.dart';
 import '../dart/package_map.dart';
 import '../globals.dart';
 import '../project.dart';

packages/flutter_tools/lib/src/commands/config.dart

@@ -3,10 +3,10 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import '../android/android_sdk.dart';
 import '../android/android_studio.dart';
+import '../convert.dart';
 import '../globals.dart';
 import '../runner/flutter_command.dart';
 import '../usage.dart';

packages/flutter_tools/lib/src/commands/create.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:linter/src/rules/pub/package_names.dart' as package_names; // ignore: implementation_imports
 import 'package:linter/src/utils.dart' as linter_utils; // ignore: implementation_imports
@@ -18,6 +17,7 @@ import '../base/net.dart';
 import '../base/os.dart';
 import '../base/utils.dart';
 import '../cache.dart';
+import '../convert.dart';
 import '../dart/pub.dart';
 import '../doctor.dart';
 import '../globals.dart';

packages/flutter_tools/lib/src/commands/daemon.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:meta/meta.dart';
 
@@ -17,6 +16,7 @@ import '../base/terminal.dart';
 import '../base/utils.dart';
 import '../build_info.dart';
 import '../cache.dart';
+import '../convert.dart';
 import '../device.dart';
 import '../emulator.dart';
 import '../fuchsia/fuchsia_device.dart';

packages/flutter_tools/lib/src/commands/screenshot.dart

@@ -3,11 +3,11 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import '../base/common.dart';
 import '../base/file_system.dart';
 import '../base/utils.dart';
+import '../convert.dart';
 import '../device.dart';
 import '../globals.dart';
 import '../runner/flutter_command.dart';

packages/flutter_tools/lib/src/commands/trace.dart

@@ -3,12 +3,12 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import '../base/common.dart';
 import '../base/file_system.dart';
 import '../base/utils.dart';
 import '../cache.dart';
+import '../convert.dart';
 import '../globals.dart';
 import '../runner/flutter_command.dart';
 import '../tracing.dart';

packages/flutter_tools/lib/src/compile.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:meta/meta.dart';
 import 'package:usage/uuid/uuid.dart';
@@ -17,6 +16,7 @@ import 'base/io.dart';
 import 'base/platform.dart';
 import 'base/process_manager.dart';
 import 'base/terminal.dart';
+import 'convert.dart';
 import 'dart/package_map.dart';
 import 'globals.dart';
 

packages/flutter_tools/lib/src/convert.dart

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Hide the original utf8 [Codec] so that we can export our own implementation
+// which adds additional error handling.
+import 'dart:convert' hide utf8;
+import 'dart:convert' as cnv show utf8, Utf8Decoder;
+
+import 'base/common.dart';
+export 'dart:convert' hide utf8, Utf8Codec, Utf8Decoder;
+
+/// A [Codec] which reports malformed bytes when decoding.
+// Created to solve https://github.com/flutter/flutter/issues/15646.
+class Utf8Codec extends Encoding {
+  const Utf8Codec();
+
+  @override
+  Converter<List<int>, String> get decoder => const Utf8Decoder();
+
+  @override
+  Converter<String, List<int>> get encoder => cnv.utf8.encoder;
+
+  @override
+  String get name => cnv.utf8.name;
+}
+
+Encoding get utf8 => const Utf8Codec();
+
+class Utf8Decoder extends cnv.Utf8Decoder {
+  const Utf8Decoder({this.reportErrors = true}) : super(allowMalformed: true);
+
+  final bool reportErrors;
+
+  @override
+  String convert(List<int> codeUnits, [int start = 0, int end]) {
+    final String result = super.convert(codeUnits, start, end);
+    // Finding a unicode replacement character indicates that the input
+    // was malformed.
+    if (reportErrors && result.contains('\u{FFFD}')) {
+      throwToolExit(
+        'Bad UTF-8 encoding found while decoding string: $result. '
+        'The Flutter team would greatly appreciate if you could file a bug or leave a'
+        'comment on the issue https://github.com/flutter/flutter/issues/15646.\n'
+        'The source bytes were:\n$codeUnits\n\n');
+    }
+    return result;
+  }
+}
\ No newline at end of file

packages/flutter_tools/lib/src/dart/analysis.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 import 'dart:math' as math;
 
 import '../base/file_system.dart' hide IOSink;
@@ -13,6 +12,7 @@ import '../base/platform.dart';
 import '../base/process_manager.dart';
 import '../base/terminal.dart';
 import '../base/utils.dart';
+import '../convert.dart';
 import '../globals.dart';
 
 class AnalysisServer {

packages/flutter_tools/lib/src/devfs.dart

@@ -3,8 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert' show base64, utf8;
-
 import 'package:json_rpc_2/json_rpc_2.dart' as rpc;
 import 'package:meta/meta.dart';
 
@@ -15,6 +13,7 @@ import 'base/io.dart';
 import 'build_info.dart';
 import 'bundle.dart';
 import 'compile.dart';
+import 'convert.dart' show base64, utf8;
 import 'dart/package_map.dart';
 import 'globals.dart';
 import 'vmservice.dart';

packages/flutter_tools/lib/src/flutter_manifest.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert' as convert;
 
 import 'package:json_schema/json_schema.dart';
 import 'package:meta/meta.dart';
@@ -12,6 +11,7 @@ import 'package:yaml/yaml.dart';
 import 'base/file_system.dart';
 import 'base/utils.dart';
 import 'cache.dart';
+import 'convert.dart' as convert;
 import 'globals.dart';
 
 final RegExp _versionPattern = RegExp(r'^(\d+)(\.(\d+)(\.(\d+))?)?(\+(\d+))?$');

packages/flutter_tools/lib/src/fuchsia/fuchsia_sdk.dart

@@ -3,13 +3,13 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import '../base/context.dart';
 import '../base/file_system.dart';
 import '../base/io.dart';
 import '../base/process.dart';
 import '../base/process_manager.dart';
+import '../convert.dart';
 import '../globals.dart';
 
 /// The [FuchsiaSdk] instance.

packages/flutter_tools/lib/src/intellij/intellij.dart

@@ -2,12 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:convert';
-
 import 'package:archive/archive.dart';
 
 import '../base/file_system.dart';
 import '../base/version.dart';
+import '../convert.dart';
 import '../doctor.dart';
 
 class IntelliJPlugins {

packages/flutter_tools/lib/src/ios/code_signing.dart

@@ -2,7 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 import 'dart:async';
-import 'dart:convert' show utf8;
 
 import 'package:quiver/strings.dart';
 
@@ -11,6 +10,7 @@ import '../base/common.dart';
 import '../base/io.dart';
 import '../base/process.dart';
 import '../base/terminal.dart';
+import '../convert.dart' show utf8;
 import '../globals.dart';
 
 /// User message when no development certificates are found in the keychain.

packages/flutter_tools/lib/src/ios/devices.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:meta/meta.dart';
 
@@ -15,6 +14,7 @@ import '../base/platform.dart';
 import '../base/process.dart';
 import '../base/process_manager.dart';
 import '../build_info.dart';
+import '../convert.dart';
 import '../device.dart';
 import '../globals.dart';
 import '../protocol_discovery.dart';

packages/flutter_tools/lib/src/ios/mac.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert' show json;
 
 import 'package:meta/meta.dart';
 
@@ -20,6 +19,7 @@ import '../base/process.dart';
 import '../base/process_manager.dart';
 import '../base/utils.dart';
 import '../build_info.dart';
+import '../convert.dart';
 import '../globals.dart';
 import '../plugins.dart';
 import '../project.dart';

packages/flutter_tools/lib/src/ios/simulators.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 import 'dart:math' as math;
 
 import '../application_package.dart';
@@ -17,6 +16,7 @@ import '../base/process_manager.dart';
 import '../base/utils.dart';
 import '../build_info.dart';
 import '../bundle.dart' as bundle;
+import '../convert.dart';
 import '../device.dart';
 import '../globals.dart';
 import '../protocol_discovery.dart';

packages/flutter_tools/lib/src/macos/macos_device.dart

@@ -2,14 +2,13 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:convert';
-
 import '../application_package.dart';
 import '../base/io.dart';
 import '../base/os.dart';
 import '../base/platform.dart';
 import '../base/process_manager.dart';
 import '../build_info.dart';
+import '../convert.dart';
 import '../device.dart';
 import '../globals.dart';
 import '../macos/application_package.dart';

packages/flutter_tools/lib/src/run_hot.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:json_rpc_2/error_code.dart' as rpc_error_code;
 import 'package:json_rpc_2/json_rpc_2.dart' as rpc;
@@ -17,6 +16,7 @@ import 'base/terminal.dart';
 import 'base/utils.dart';
 import 'build_info.dart';
 import 'compile.dart';
+import 'convert.dart';
 import 'dart/dependencies.dart';
 import 'dart/pub.dart';
 import 'devfs.dart';

packages/flutter_tools/lib/src/runner/flutter_command_runner.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:args/args.dart';
 import 'package:args/command_runner.dart';
@@ -27,6 +26,7 @@ import '../base/terminal.dart';
 import '../base/user_messages.dart';
 import '../base/utils.dart';
 import '../cache.dart';
+import '../convert.dart';
 import '../dart/package_map.dart';
 import '../device.dart';
 import '../globals.dart';

packages/flutter_tools/lib/src/services.dart

@@ -3,12 +3,12 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:yaml/yaml.dart';
 
 import 'android/android_sdk.dart';
 import 'base/file_system.dart';
+import 'convert.dart';
 import 'dart/package_map.dart';
 import 'globals.dart';
 

packages/flutter_tools/lib/src/test/event_printer.dart

@@ -2,9 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:convert' show json;
-
 import '../base/io.dart' show stdout;
+import '../convert.dart';
 import 'watcher.dart';
 
 /// Prints JSON events when running a test in --machine mode.

packages/flutter_tools/lib/src/test/flutter_platform.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:meta/meta.dart';
 import 'package:stream_channel/stream_channel.dart';
@@ -26,6 +25,7 @@ import '../base/terminal.dart';
 import '../build_info.dart';
 import '../bundle.dart';
 import '../compile.dart';
+import '../convert.dart';
 import '../dart/package_map.dart';
 import '../globals.dart';
 import '../vmservice.dart';
@@ -144,7 +144,7 @@ String generateTestBootstrap({
   final StringBuffer buffer = StringBuffer();
   buffer.write('''
 import 'dart:async';
-import 'dart:convert';
+import 'dart:convert';  // ignore: dart_convert_import
 import 'dart:io';  // ignore: dart_io_import
 import 'dart:isolate';
 

packages/flutter_tools/lib/src/tester/flutter_tester.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:meta/meta.dart';
 
@@ -15,6 +14,7 @@ import '../base/io.dart';
 import '../base/process_manager.dart';
 import '../build_info.dart';
 import '../bundle.dart' as bundle;
+import '../convert.dart';
 import '../dart/package_map.dart';
 import '../device.dart';
 import '../globals.dart';

packages/flutter_tools/lib/src/version.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:meta/meta.dart';
 
@@ -15,6 +14,7 @@ import 'base/process.dart';
 import 'base/process_manager.dart';
 import 'base/time.dart';
 import 'cache.dart';
+import 'convert.dart';
 import 'globals.dart';
 
 class FlutterVersion {

packages/flutter_tools/lib/src/vmservice.dart

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert' show base64;
 import 'dart:math' as math;
 
 import 'package:file/file.dart';
@@ -19,6 +18,7 @@ import 'base/context.dart';
 import 'base/file_system.dart';
 import 'base/io.dart' as io;
 import 'base/utils.dart';
+import 'convert.dart' show base64;
 import 'globals.dart';
 import 'vmservice_record_replay.dart';
 

packages/flutter_tools/lib/src/vmservice_record_replay.dart

@@ -3,13 +3,13 @@
 // found in the LICENSE file.
 
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:file/file.dart';
 import 'package:stream_channel/stream_channel.dart';
 
 import 'base/io.dart';
 import 'base/process.dart';
+import 'convert.dart';
 import 'globals.dart';
 
 const String _kManifest = 'MANIFEST.txt';

packages/flutter_tools/lib/src/vscode/vscode.dart

@@ -2,12 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:convert';
-
 import '../base/common.dart';
 import '../base/file_system.dart';
 import '../base/platform.dart';
 import '../base/version.dart';
+import '../convert.dart';
 import '../doctor.dart';
 
 // Include VS Code insiders (useful for debugging).

packages/flutter_tools/test/forbidden_imports_test.dart

@@ -47,6 +47,28 @@ void main() {
       }
     }
   });
+
+  test('no unauthorized imports of dart:convert', () {
+    final String whitelistedPath = fs.path.join(flutterTools, 'lib', 'src', 'convert.dart');
+    bool _isNotWhitelisted(FileSystemEntity entity) => entity.path != whitelistedPath;
+
+    for (String dirName in <String>['lib']) {
+      final Iterable<File> files = fs.directory(fs.path.join(flutterTools, dirName))
+        .listSync(recursive: true)
+        .where(_isDartFile)
+        .where(_isNotWhitelisted)
+        .map(_asFile);
+      for (File file in files) {
+        for (String line in file.readAsLinesSync()) {
+          if (line.startsWith(RegExp(r'import.*dart:convert')) &&
+              !line.contains('ignore: dart_convert_import')) {
+            final String relativePath = fs.path.relative(file.path, from:flutterTools);
+            fail("$relativePath imports 'dart:convert'; import 'lib/src/convert.dart' instead");
+          }
+        }
+      }
+    }
+  });
 }
 
 bool _isDartFile(FileSystemEntity entity) => entity is File && entity.path.endsWith('.dart');