[gen_l10n] Fallback feature for untranslated messages (#53374)

* Generate methods using template resources if they do not exist in other locales

* Added a flag to either output of messages that have not been translated with detail into a file, or display a summary on the terminal.

* Add integration test for fallback message usage

dev/tools/localization/bin/gen_l10n.dart

@@ -40,6 +40,15 @@ void main(List<String> arguments) {
     help: 'The filename for the output localization and localizations '
       'delegate classes.',
   );
+  parser.addOption(
+    'untranslated-messages-file',
+    help: 'The location of a file that describes the localization\n'
+      'messages have not been translated yet. Using this option will create\n'
+      'a JSON file at the target location, in the following format:\n\n'
+      '"locale": ["message_1", "message_2" ... "message_n"]\n\n'
+      'If this option is not specified, a summary of the messages that\n'
+      'have not been translated will be printed on the command line.'
+  );
   parser.addOption(
     'output-class',
     defaultsTo: 'AppLocalizations',
@@ -84,6 +93,7 @@ void main(List<String> arguments) {
   final String arbPathString = results['arb-dir'] as String;
   final String outputFileString = results['output-localization-file'] as String;
   final String templateArbFileName = results['template-arb-file'] as String;
+  final String untranslatedMessagesFile = results['untranslated-messages-file'] as String;
   final String classNameString = results['output-class'] as String;
   final String preferredSupportedLocaleString = results['preferred-supported-locales'] as String;
   final String headerString = results['header'] as String;
@@ -104,7 +114,8 @@ void main(List<String> arguments) {
         headerFile: headerFile,
       )
       ..loadResources()
-      ..writeOutputFile();
+      ..writeOutputFile()
+      ..outputUnimplementedMessages(untranslatedMessagesFile);
   } on FileSystemException catch (e) {
     exitWithError(e.message);
   } on FormatException catch (e) {

dev/tools/localization/gen_l10n.dart

@@ -193,48 +193,6 @@ String generateMethod(Message message, AppResourceBundle bundle) {
     .replaceAll('@(message)', generateMessage());
 }
 
-String generateBaseClassFile(
-  String className,
-  String fileName,
-  String header,
-  AppResourceBundle bundle,
-  Iterable<Message> messages,
-) {
-  final LocaleInfo locale = bundle.locale;
-  final Iterable<String> methods = messages
-    .where((Message message) => bundle.translationFor(message) != null)
-    .map((Message message) => generateMethod(message, bundle));
-
-  return classFileTemplate
-    .replaceAll('@(header)', header)
-    .replaceAll('@(language)', describeLocale(locale.toString()))
-    .replaceAll('@(baseClass)', className)
-    .replaceAll('@(fileName)', fileName)
-    .replaceAll('@(class)', '$className${locale.camelCase()}')
-    .replaceAll('@(localeName)', locale.toString())
-    .replaceAll('@(methods)', methods.join('\n\n'));
-}
-
-String generateSubclass({
-  String className,
-  AppResourceBundle bundle,
-  Iterable<Message> messages,
-}) {
-  final LocaleInfo locale = bundle.locale;
-  final String baseClassName = '$className${LocaleInfo.fromString(locale.languageCode).camelCase()}';
-
-  final Iterable<String> methods = messages
-    .where((Message message) => bundle.translationFor(message) != null)
-    .map((Message message) => generateMethod(message, bundle));
-
-  return subclassTemplate
-    .replaceAll('@(language)', describeLocale(locale.toString()))
-    .replaceAll('@(baseLanguageClassName)', baseClassName)
-    .replaceAll('@(class)', '$className${locale.camelCase()}')
-    .replaceAll('@(localeName)', locale.toString())
-    .replaceAll('@(methods)', methods.join('\n\n'));
-}
-
 String generateBaseClassMethod(Message message) {
   final String comment = message.description ?? 'No description provided in @${message.resourceId}';
   if (message.placeholders.isNotEmpty) {
@@ -373,6 +331,7 @@ class LocalizationsGenerator {
   final file.FileSystem _fs;
   Iterable<Message> _allMessages;
   AppResourceBundleCollection _allBundles;
+  LocaleInfo _templateArbLocale;
 
   /// The reference to the project's l10n directory.
   ///
@@ -436,6 +395,8 @@ class LocalizationsGenerator {
   /// The header to be prepended to the generated Dart localization file.
   String header = '';
 
+  final Map<LocaleInfo, List<String>> _unimplementedMessages = <LocaleInfo, List<String>>{};
+
   /// Initializes [l10nDirectory], [templateArbFile], [outputFile] and [className].
   ///
   /// Throws an [L10nException] when a provided configuration is not allowed
@@ -609,6 +570,7 @@ class LocalizationsGenerator {
   // files in l10nDirectory. Also initialized: supportedLocales.
   void loadResources() {
     final AppResourceBundle templateBundle = AppResourceBundle(templateArbFile);
+    _templateArbLocale = templateBundle.locale;
     _allMessages = templateBundle.resourceIds.map((String id) => Message(templateBundle.resources, id));
     for (final String resourceId in templateBundle.resourceIds)
       if (!_isValidGetterAndMethodName(resourceId)) {
@@ -639,6 +601,71 @@ class LocalizationsGenerator {
     supportedLocales.addAll(allLocales);
   }
 
+  void _addUnimplementedMessage(LocaleInfo locale, String message) {
+    if (_unimplementedMessages.containsKey(locale)) {
+      _unimplementedMessages[locale].add(message);
+    } else {
+      _unimplementedMessages.putIfAbsent(locale, () => <String>[message]);
+    }
+  }
+
+  String _generateBaseClassFile(
+    String className,
+    String fileName,
+    String header,
+    AppResourceBundle bundle,
+    AppResourceBundle templateBundle,
+    Iterable<Message> messages,
+  ) {
+    final LocaleInfo locale = bundle.locale;
+
+    final Iterable<String> methods = messages.map((Message message) {
+      if (bundle.translationFor(message) == null) {
+        _addUnimplementedMessage(locale, message.resourceId);
+      }
+
+      return generateMethod(
+        message,
+        bundle.translationFor(message) == null ? templateBundle : bundle,
+      );
+    });
+
+    return classFileTemplate
+      .replaceAll('@(header)', header)
+      .replaceAll('@(language)', describeLocale(locale.toString()))
+      .replaceAll('@(baseClass)', className)
+      .replaceAll('@(fileName)', fileName)
+      .replaceAll('@(class)', '$className${locale.camelCase()}')
+      .replaceAll('@(localeName)', locale.toString())
+      .replaceAll('@(methods)', methods.join('\n\n'));
+  }
+
+  String _generateSubclass(
+    String className,
+    AppResourceBundle bundle,
+    Iterable<Message> messages,
+  ) {
+    final LocaleInfo locale = bundle.locale;
+    final String baseClassName = '$className${LocaleInfo.fromString(locale.languageCode).camelCase()}';
+
+    messages
+      .where((Message message) => bundle.translationFor(message) == null)
+      .forEach((Message message) {
+        _addUnimplementedMessage(locale, message.resourceId);
+      });
+
+    final Iterable<String> methods = messages
+      .where((Message message) => bundle.translationFor(message) != null)
+      .map((Message message) => generateMethod(message, bundle));
+
+    return subclassTemplate
+      .replaceAll('@(language)', describeLocale(locale.toString()))
+      .replaceAll('@(baseLanguageClassName)', baseClassName)
+      .replaceAll('@(class)', '$className${locale.camelCase()}')
+      .replaceAll('@(localeName)', locale.toString())
+      .replaceAll('@(methods)', methods.join('\n\n'));
+  }
+
   // Generate the AppLocalizations class, its LocalizationsDelegate subclass,
   // and all AppLocalizations subclasses for every locale.
   String generateCode() {
@@ -692,11 +719,12 @@ class LocalizationsGenerator {
         // Generate the template for the base class file. Further string
         // interpolation will be done to determine if there are
         // subclasses that extend the base class.
-        final String languageBaseClassFile = generateBaseClassFile(
+        final String languageBaseClassFile = _generateBaseClassFile(
           className,
           outputFileName,
           header,
           _allBundles.bundleFor(locale),
+          _allBundles.bundleFor(_templateArbLocale),
           _allMessages,
         );
 
@@ -705,10 +733,10 @@ class LocalizationsGenerator {
 
         // Generate every subclass that is needed for the particular language
         final Iterable<String> subclasses = localesForLanguage.map<String>((LocaleInfo locale) {
-          return generateSubclass(
-            className: className,
-            bundle: _allBundles.bundleFor(locale),
-            messages: _allMessages,
+          return _generateSubclass(
+            className,
+            _allBundles.bundleFor(locale),
+            _allMessages,
           );
         });
 
@@ -741,4 +769,52 @@ class LocalizationsGenerator {
   void writeOutputFile() {
     outputFile.writeAsStringSync(generateCode());
   }
+
+  void outputUnimplementedMessages(String untranslatedMessagesFile) {
+    if (untranslatedMessagesFile == null || untranslatedMessagesFile == '') {
+      _unimplementedMessages.forEach((LocaleInfo locale, List<String> messages) {
+        stdout.writeln('"$locale": ${messages.length} untranslated message(s).');
+      });
+      stdout.writeln(
+        'To see a detailed report, use the --unimplemented-messages-file \n'
+        'option in the tool to generate a JSON format file containing \n'
+        'all messages that need to be translated.'
+      );
+    } else {
+      _writeUnimplementedMessagesFile(untranslatedMessagesFile);
+    }
+  }
+
+  void _writeUnimplementedMessagesFile(String untranslatedMessagesFile) {
+    if (_unimplementedMessages.isEmpty) {
+      return;
+    }
+
+    final File unimplementedMessageTranslationsFile = _fs.file(untranslatedMessagesFile);
+
+    String resultingFile = '{\n';
+    int count = 0;
+    final int numberOfLocales = _unimplementedMessages.length;
+    _unimplementedMessages.forEach((LocaleInfo locale, List<String> messages) {
+      resultingFile += '  "$locale": [\n';
+
+      for (int i = 0; i < messages.length; i += 1) {
+        resultingFile += '    "${messages[i]}"';
+        if (i != messages.length - 1) {
+          resultingFile += ',';
+        }
+        resultingFile += '\n';
+      }
+
+      resultingFile += '  ]';
+      count += 1;
+      if (count < numberOfLocales) {
+        resultingFile += ',\n';
+      }
+      resultingFile += '\n';
+    });
+
+    resultingFile += '}\n';
+    unimplementedMessageTranslationsFile.writeAsStringSync(resultingFile);
+  }
 }

dev/tools/test/localization/gen_l10n_test.dart

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:convert';
 import 'dart:io';
 
 import 'package:file/file.dart';
@@ -25,6 +26,17 @@ const String singleMessageArbFileString = '''
     "description": "Title for the application"
   }
 }''';
+const String twoMessageArbFileString = '''
+{
+  "title": "Title",
+  "@title": {
+    "description": "Title for the application"
+  },
+  "subtitle": "Subtitle",
+  "@subtitle": {
+    "description": "Subtitle for the application"
+  }
+}''';
 const String esArbFileName = 'app_es.arb';
 const String singleEsMessageArbFileString = '''
 {
@@ -277,6 +289,43 @@ void main() {
     expect(generator.header, '/// Sample header in a text file');
   });
 
+  test('correctly creates an unimplemented messages file', () {
+    fs.currentDirectory.childDirectory('lib').childDirectory('l10n')
+      ..createSync(recursive: true)
+      ..childFile(defaultTemplateArbFileName).writeAsStringSync(twoMessageArbFileString)
+      ..childFile(esArbFileName).writeAsStringSync(singleEsMessageArbFileString);
+
+    LocalizationsGenerator generator;
+    try {
+      generator = LocalizationsGenerator(fs);
+      generator
+        ..initialize(
+          l10nDirectoryPath: defaultArbPathString,
+          templateArbFileName: defaultTemplateArbFileName,
+          outputFileString: defaultOutputFileString,
+          classNameString: defaultClassNameString,
+        )
+        ..loadResources()
+        ..generateCode()
+        ..outputUnimplementedMessages(path.join('lib', 'l10n', 'unimplemented_message_translations.json'));
+    } on L10nException catch (e) {
+      fail('Generating output should not fail: \n${e.message}');
+    }
+
+    final File unimplementedOutputFile = fs.file(
+      path.join('lib', 'l10n', 'unimplemented_message_translations.json'),
+    );
+    final String unimplementedOutputString = unimplementedOutputFile.readAsStringSync();
+    try {
+      // Since ARB file is essentially JSON, decoding it should not fail.
+      json.decode(unimplementedOutputString);
+    } on Exception {
+      fail('Parsing arb file should not fail');
+    }
+    expect(unimplementedOutputString, contains('es'));
+    expect(unimplementedOutputString, contains('subtitle'));
+  });
+
   test('setting both a headerString and a headerFile should fail', () {
     fs.currentDirectory.childDirectory('lib').childDirectory('l10n')
       ..createSync(recursive: true)

packages/flutter_tools/test/integration.shard/gen_l10n_test.dart

@@ -95,41 +95,42 @@ void main() {
       '#l10n 16 (你好)\n'
       '#l10n 17 (你好世界)\n'
       '#l10n 18 (你好2个其他世界)\n'
-      '#l10n 19 (--- scriptCode: zh_Hans ---)\n'
-      '#l10n 20 (简体你好世界)\n'
-      '#l10n 21 (--- scriptCode - zh_Hant ---)\n'
-      '#l10n 22 (繁體你好世界)\n'
-      '#l10n 23 (--- scriptCode - zh_Hant_TW ---)\n'
-      '#l10n 24 (台灣繁體你好世界)\n'
-      '#l10n 25 (--- General formatting tests ---)\n'
-      '#l10n 26 (Hello World)\n'
-      '#l10n 27 (Hello _NEWLINE_ World)\n'
-      '#l10n 28 (Hello World)\n'
+      '#l10n 19 (Hello 世界)\n'
+      '#l10n 20 (--- scriptCode: zh_Hans ---)\n'
+      '#l10n 21 (简体你好世界)\n'
+      '#l10n 22 (--- scriptCode - zh_Hant ---)\n'
+      '#l10n 23 (繁體你好世界)\n'
+      '#l10n 24 (--- scriptCode - zh_Hant_TW ---)\n'
+      '#l10n 25 (台灣繁體你好世界)\n'
+      '#l10n 26 (--- General formatting tests ---)\n'
+      '#l10n 27 (Hello World)\n'
+      '#l10n 28 (Hello _NEWLINE_ World)\n'
       '#l10n 29 (Hello World)\n'
-      '#l10n 30 (Hello World on Friday, January 1, 1960)\n'
-      '#l10n 31 (Hello world argument on 1/1/1960 at 00:00)\n'
-      '#l10n 32 (Hello World from 1960 to 2020)\n'
-      '#l10n 33 (Hello for 123)\n'
-      '#l10n 34 (Hello for price USD123.00)\n'
-      '#l10n 35 (Hello)\n'
-      '#l10n 36 (Hello World)\n'
-      '#l10n 37 (Hello two worlds)\n'
-      '#l10n 38 (Hello)\n'
-      '#l10n 39 (Hello new World)\n'
-      '#l10n 40 (Hello two new worlds)\n'
-      '#l10n 41 (Hello on Friday, January 1, 1960)\n'
-      '#l10n 42 (Hello World, on Friday, January 1, 1960)\n'
-      '#l10n 43 (Hello two worlds, on Friday, January 1, 1960)\n'
-      '#l10n 44 (Hello other 0 worlds, with a total of 100 citizens)\n'
-      '#l10n 45 (Hello World of 101 citizens)\n'
-      '#l10n 46 (Hello two worlds with 102 total citizens)\n'
-      '#l10n 47 ([Hello] -World- #123#)\n'
-      '#l10n 48 (\$!)\n'
-      '#l10n 49 (One \$)\n'
-      '#l10n 50 (Flutter\'s amazing!)\n'
-      '#l10n 51 (Flutter\'s amazing, times 2!)\n'
-      '#l10n 52 (Flutter is "amazing"!)\n'
-      '#l10n 53 (Flutter is "amazing", times 2!)\n'
+      '#l10n 30 (Hello World)\n'
+      '#l10n 31 (Hello World on Friday, January 1, 1960)\n'
+      '#l10n 32 (Hello world argument on 1/1/1960 at 00:00)\n'
+      '#l10n 33 (Hello World from 1960 to 2020)\n'
+      '#l10n 34 (Hello for 123)\n'
+      '#l10n 35 (Hello for price USD123.00)\n'
+      '#l10n 36 (Hello)\n'
+      '#l10n 37 (Hello World)\n'
+      '#l10n 38 (Hello two worlds)\n'
+      '#l10n 39 (Hello)\n'
+      '#l10n 40 (Hello new World)\n'
+      '#l10n 41 (Hello two new worlds)\n'
+      '#l10n 42 (Hello on Friday, January 1, 1960)\n'
+      '#l10n 43 (Hello World, on Friday, January 1, 1960)\n'
+      '#l10n 44 (Hello two worlds, on Friday, January 1, 1960)\n'
+      '#l10n 45 (Hello other 0 worlds, with a total of 100 citizens)\n'
+      '#l10n 46 (Hello World of 101 citizens)\n'
+      '#l10n 47 (Hello two worlds with 102 total citizens)\n'
+      '#l10n 48 ([Hello] -World- #123#)\n'
+      '#l10n 49 (\$!)\n'
+      '#l10n 50 (One \$)\n'
+      '#l10n 51 (Flutter\'s amazing!)\n'
+      '#l10n 52 (Flutter\'s amazing, times 2!)\n'
+      '#l10n 53 (Flutter is "amazing"!)\n'
+      '#l10n 54 (Flutter is "amazing", times 2!)\n'
       '#l10n END\n'
     );
   });

packages/flutter_tools/test/integration.shard/test_data/gen_l10n_project.dart

@@ -129,6 +129,9 @@ class Home extends StatelessWidget {
             results.add(AppLocalizations.of(context).helloWorlds(0));
             results.add(AppLocalizations.of(context).helloWorlds(1));
             results.add(AppLocalizations.of(context).helloWorlds(2));
+            // Should use the fallback language, in this case,
+            // "Hello 世界" should be displayed.
+            results.add(AppLocalizations.of(context).hello("世界"));
           },
         ),
         LocaleBuilder(
@@ -438,33 +441,11 @@ void main() {
 }
 ''';
 
-  // Only tests `helloWorld` and `helloWorlds`. The rest of the messages
-  // are added out of necessity since every base class requires an
-  // override for every message.
   final String appZh = r'''
 {
   "@@locale": "zh",
   "helloWorld": "你好世界",
-  "helloWorlds": "{count,plural, =0{你好} =1{你好世界} other{你好{count}个其他世界}}",
-  "helloNewlineWorld": "Hello \n World",
-  "hello": "Hello {world}",
-  "greeting": "{hello} {world}",
-  "helloWorldOn": "Hello World on {date}",
-  "helloWorldDuring": "Hello World from {startDate} to {endDate}",
-  "helloOn": "Hello {world} on {date} at {time}",
-  "helloFor": "Hello for {value}",
-  "helloCost": "Hello for {price} {value}",
-  "helloAdjectiveWorlds": "{count,plural, =0{Hello} =1{Hello {adjective} World} =2{Hello two {adjective} worlds} other{Hello other {count} {adjective} worlds}}",
-  "helloWorldsOn": "{count,plural, =0{Hello on {date}} =1{Hello World, on {date}} =2{Hello two worlds, on {date}} other{Hello other {count} worlds, on {date}}}",
-  "helloWorldPopulation": "{count,plural, =1{Hello World of {population} citizens} =2{Hello two worlds with {population} total citizens} many{Hello all {count} worlds, with a total of {population} citizens} other{Hello other {count} worlds, with a total of {population} citizens}}",
-  "helloWorldInterpolation": "[{hello}] #{world}#",
-  "helloWorldsInterpolation": "{count,plural, other {[{hello}] -{world}- #{count}#}}",
-  "dollarSign": "$!",
-  "dollarSignPlural": "{count,plural, =1{One $} other{Many $}}",
-  "singleQuote": "Flutter's amazing!",
-  "singleQuotePlural": "{count,plural, =1{Flutter's amazing, times 1!} other{Flutter's amazing, times {count}!}",
-  "doubleQuote": "Flutter is \"amazing\"!",
-  "doubleQuotePlural": "{count,plural, =1{Flutter is \"amazing\", times 1!} other{Flutter is \"amazing\", times {count}!"
+  "helloWorlds": "{count,plural, =0{你好} =1{你好世界} other{你好{count}个其他世界}}"
 }
 ''';