ICU Message Syntax Parser (#112390)

* init

* code generation

* improve syntax error, add tests

* add tests and fix bugs

* code generation fix

* fix all tests :)

* fix bug

* init

* fix all code gen issues

* FIXED ALL TESTS :D

* add license

* remove trailing spaces

* remove print

* tests fix

* specify type annotation

* fix test

* lint

* fix todos

* fix subclass issues

* final fix; flutter gallery runs

* escaping for later pr

* fix comment

* address PR comments

* more

* more descriptive errors

* last fixes

packages/flutter_tools/lib/src/localizations/gen_l10n_templates.dart

@@ -135,70 +135,37 @@ const String getterTemplate = '''
   String get @(name) => @(message);''';
 
 const String methodTemplate = '''
-  @override
-  String @(name)(@(parameters)) {
-    return @(message);
-  }''';
-
-const String formatMethodTemplate = '''
   @override
   String @(name)(@(parameters)) {
 @(dateFormatting)
 @(numberFormatting)
+@(helperMethods)
     return @(message);
   }''';
 
-const String pluralMethodTemplate = '''
-  @override
-  String @(name)(@(parameters)) {
-@(dateFormatting)
-@(numberFormatting)
-    return intl.Intl.pluralLogic(
-      @(count),
-      locale: localeName,
-@(pluralLogicArgs),
-    );
-  }''';
-
-const String pluralMethodTemplateInString = '''
-  @override
-  String @(name)(@(parameters)) {
-@(dateFormatting)
-@(numberFormatting)
-    final String @(variable) = intl.Intl.pluralLogic(
-      @(count),
-      locale: localeName,
-@(pluralLogicArgs),
-    );
-
-    return @(string);
-  }''';
-
-const String selectMethodTemplate = '''
-  @override
-  String @(name)(@(parameters)) {
-    return intl.Intl.select(
-      @(choice),
-      {
-        @(cases)
-      },
-      desc: '@(description)'
-    );
-  }''';
-
-const String selectMethodTemplateInString = '''
-  @override
-  String @(name)(@(parameters)) {
-    final String @(variable) = intl.Intl.select(
-      @(choice),
-      {
-        @(cases)
-      },
-      desc: '@(description)'
-    );
-
-    return @(string);
-  }''';
+const String messageHelperTemplate = '''
+    String @(name)(@(parameters)) {
+      return @(message);
+    }''';
+
+const String pluralHelperTemplate = '''
+    String @(name)(@(parameters)) {
+      return intl.Intl.pluralLogic(
+        @(count),
+        locale: localeName,
+@(pluralLogicArgs)
+      );
+    }''';
+
+const String selectHelperTemplate = '''
+    String @(name)(@(parameters)) {
+      return intl.Intl.selectLogic(
+        @(choice),
+        {
+@(selectCases)
+        },
+      );
+    }''';
 
 const String classFileTemplate = '''
 @(header)@(requiresIntlImport)import '@(fileName)';

packages/flutter_tools/lib/src/localizations/gen_l10n_types.dart

@@ -129,6 +129,25 @@ class L10nException implements Exception {
   String toString() => message;
 }
 
+class L10nParserException extends L10nException {
+  L10nParserException(
+    this.error,
+    this.fileName,
+    this.messageId,
+    this.messageString,
+    this.charNumber
+  ): super('''
+$error
+[$fileName:$messageId] $messageString
+${List<String>.filled(4 + fileName.length + messageId.length + charNumber, ' ').join()}^''');
+
+  final String error;
+  final String fileName;
+  final String messageId;
+  final String messageString;
+  final int charNumber;
+}
+
 // One optional named parameter to be used by a NumberFormat.
 //
 // Some of the NumberFormat factory constructors have optional named parameters.
@@ -202,16 +221,16 @@ class Placeholder {
   final String resourceId;
   final String name;
   final String? example;
-  final String? type;
+  String? type;
   final String? format;
   final List<OptionalParameter> optionalParameters;
   final bool? isCustomDateFormat;
 
-  bool get requiresFormatting => <String>['DateTime', 'double', 'num'].contains(type) || (type == 'int' && format != null);
-  bool get isNumber => <String>['double', 'int', 'num'].contains(type);
+  bool get requiresFormatting => requiresDateFormatting || requiresNumFormatting;
+  bool get requiresDateFormatting => type == 'DateTime';
+  bool get requiresNumFormatting => <String>['int', 'num', 'double'].contains(type) && format != null;
   bool get hasValidNumberFormat => _validNumberFormats.contains(format);
   bool get hasNumberFormatWithParameters => _numberFormatsWithNamedParameters.contains(format);
-  bool get isDate => 'DateTime' == type;
   bool get hasValidDateFormat => _validDateFormats.contains(format);
 
   static String? _stringAttribute(
@@ -290,6 +309,8 @@ class Placeholder {
 // The value of this Message is "Hello World". The Message's value is the
 // localized string to be shown for the template ARB file's locale.
 // The docs for the Placeholder explain how placeholder entries are defined.
+// TODO(thkim1011): We need to refactor this Message class to own all the messages in each language.
+// See https://github.com/flutter/flutter/issues/112709.
 class Message {
   Message(Map<String, Object?> bundle, this.resourceId, bool isResourceAttributeRequired)
     : assert(bundle != null),
@@ -298,7 +319,12 @@ class Message {
       description = _description(bundle, resourceId, isResourceAttributeRequired),
       placeholders = _placeholders(bundle, resourceId, isResourceAttributeRequired),
       _pluralMatch = _pluralRE.firstMatch(_value(bundle, resourceId)),
-      _selectMatch = _selectRE.firstMatch(_value(bundle, resourceId));
+      _selectMatch = _selectRE.firstMatch(_value(bundle, resourceId)) {
+        if (isPlural) {
+          final Placeholder placeholder = getCountPlaceholder();
+          placeholder.type = 'num';
+        }
+      }
 
   static final RegExp _pluralRE = RegExp(r'\s*\{([\w\s,]*),\s*plural\s*,');
   static final RegExp _selectRE = RegExp(r'\s*\{([\w\s,]*),\s*select\s*,');
@@ -769,3 +795,50 @@ final Set<String> _iso639Languages = <String>{
   'zh',
   'zu',
 };
+
+// Used in LocalizationsGenerator._generateMethod.generateHelperMethod.
+class HelperMethod {
+  HelperMethod(this.dependentPlaceholders, {this.helper, this.placeholder, this.string }):
+    assert((() {
+      // At least one of helper, placeholder, string must be nonnull.
+      final bool a = helper == null;
+      final bool b = placeholder == null;
+      final bool c = string == null;
+      return (!a && b && c) || (a && !b && c) || (a && b && !c);
+    })());
+
+  Set<Placeholder> dependentPlaceholders;
+  String? helper;
+  Placeholder? placeholder;
+  String? string;
+
+  String get helperOrPlaceholder {
+    if (helper != null) {
+      return '$helper($methodArguments)';
+    } else if (string != null) {
+      return '$string';
+    } else {
+      if (placeholder!.requiresFormatting) {
+        return '${placeholder!.name}String';
+      } else {
+        return placeholder!.name;
+      }
+    }
+  }
+
+  String get methodParameters {
+    assert(helper != null);
+    return dependentPlaceholders.map((Placeholder placeholder) =>
+      (placeholder.requiresFormatting)
+        ? 'String ${placeholder.name}String'
+        : '${placeholder.type} ${placeholder.name}').join(', ');
+  }
+
+  String get methodArguments {
+    assert(helper != null);
+    return dependentPlaceholders.map((Placeholder placeholder) =>
+      (placeholder.requiresFormatting)
+        ? '${placeholder.name}String'
+        : placeholder.name).join(', ');
+  }
+}

packages/flutter_tools/lib/src/localizations/localizations_utils.dart

@@ -292,7 +292,34 @@ String generateString(String value) {
     // Reintroduce escaped backslashes into generated Dart string.
     .replaceAll(backslash, r'\\');
 
-  return "'$value'";
+  return value;
+}
+
+/// Given a list of strings, placeholders, or helper function calls, concatenate
+/// them into one expression to be returned.
+String generateReturnExpr(List<HelperMethod> helpers) {
+  if (helpers.isEmpty) {
+    return "''";
+  } else if (
+    helpers.length == 1
+      && helpers[0].string == null
+      && (helpers[0].placeholder?.type == 'String' || helpers[0].helper != null)
+  ) {
+    return helpers[0].helperOrPlaceholder;
+  } else {
+    final String string = helpers.reversed.fold<String>('', (String string, HelperMethod helper) {
+      if (helper.string != null) {
+        return generateString(helper.string!) + string;
+      }
+      final RegExp alphanumeric = RegExp(r'^([0-9a-zA-Z]|_)+$');
+      if (alphanumeric.hasMatch(helper.helperOrPlaceholder) && !(string.isNotEmpty && alphanumeric.hasMatch(string[0]))) {
+        return '\$${helper.helperOrPlaceholder}$string';
+      } else {
+        return '\${${helper.helperOrPlaceholder}}$string';
+      }
+    });
+    return "'$string'";
+  }
 }
 
 /// Typed configuration from the localizations config file.

packages/flutter_tools/test/integration.shard/gen_l10n_test.dart

@@ -123,45 +123,48 @@ void main() {
       '#l10n 70 (Indeed, they like Flutter!)\n'
       '#l10n 71 (Indeed, he likes ice cream!)\n'
       '#l10n 72 (Indeed, she likes chocolate!)\n'
-      '#l10n 73 (--- es ---)\n'
-      '#l10n 74 (ES - Hello world)\n'
-      '#l10n 75 (ES - Hello _NEWLINE_ World)\n'
-      '#l10n 76 (ES - Hola \$ Mundo)\n'
-      '#l10n 77 (ES - Hello Mundo)\n'
-      '#l10n 78 (ES - Hola Mundo)\n'
-      '#l10n 79 (ES - Hello World on viernes, 1 de enero de 1960)\n'
-      '#l10n 80 (ES - Hello world argument on 1/1/1960 at 0:00)\n'
-      '#l10n 81 (ES - Hello World from 1960 to 2020)\n'
-      '#l10n 82 (ES - Hello for 123)\n'
-      '#l10n 83 (ES - Hello)\n'
-      '#l10n 84 (ES - Hello World)\n'
-      '#l10n 85 (ES - Hello two worlds)\n'
+      '#l10n 73 (he)\n'
+      '#l10n 74 (they)\n'
+      '#l10n 75 (she)\n'
+      '#l10n 76 (--- es ---)\n'
+      '#l10n 77 (ES - Hello world)\n'
+      '#l10n 78 (ES - Hello _NEWLINE_ World)\n'
+      '#l10n 79 (ES - Hola \$ Mundo)\n'
+      '#l10n 80 (ES - Hello Mundo)\n'
+      '#l10n 81 (ES - Hola Mundo)\n'
+      '#l10n 82 (ES - Hello World on viernes, 1 de enero de 1960)\n'
+      '#l10n 83 (ES - Hello world argument on 1/1/1960 at 0:00)\n'
+      '#l10n 84 (ES - Hello World from 1960 to 2020)\n'
+      '#l10n 85 (ES - Hello for 123)\n'
       '#l10n 86 (ES - Hello)\n'
-      '#l10n 87 (ES - Hello nuevo World)\n'
-      '#l10n 88 (ES - Hello two nuevo worlds)\n'
-      '#l10n 89 (ES - Hello on viernes, 1 de enero de 1960)\n'
-      '#l10n 90 (ES - Hello World, on viernes, 1 de enero de 1960)\n'
-      '#l10n 91 (ES - Hello two worlds, on viernes, 1 de enero de 1960)\n'
-      '#l10n 92 (ES - Hello other 0 worlds, with a total of 100 citizens)\n'
-      '#l10n 93 (ES - Hello World of 101 citizens)\n'
-      '#l10n 94 (ES - Hello two worlds with 102 total citizens)\n'
-      '#l10n 95 (ES - [Hola] -Mundo- #123#)\n'
-      '#l10n 96 (ES - \$!)\n'
-      '#l10n 97 (ES - One \$)\n'
-      "#l10n 98 (ES - Flutter's amazing!)\n"
-      "#l10n 99 (ES - Flutter's amazing, times 2!)\n"
-      '#l10n 100 (ES - Flutter is "amazing"!)\n'
-      '#l10n 101 (ES - Flutter is "amazing", times 2!)\n'
-      '#l10n 102 (ES - 16 wheel truck)\n'
-      "#l10n 103 (ES - Sedan's elegance)\n"
-      '#l10n 104 (ES - Cabriolet has "acceleration")\n'
-      '#l10n 105 (ES - Oh, she found ES - 1 itemES - !)\n'
-      '#l10n 106 (ES - Indeed, ES - they like ES - Flutter!)\n'
-      '#l10n 107 (--- es_419 ---)\n'
-      '#l10n 108 (ES 419 - Hello World)\n'
-      '#l10n 109 (ES 419 - Hello)\n'
-      '#l10n 110 (ES 419 - Hello World)\n'
-      '#l10n 111 (ES 419 - Hello two worlds)\n'
+      '#l10n 87 (ES - Hello World)\n'
+      '#l10n 88 (ES - Hello two worlds)\n'
+      '#l10n 89 (ES - Hello)\n'
+      '#l10n 90 (ES - Hello nuevo World)\n'
+      '#l10n 91 (ES - Hello two nuevo worlds)\n'
+      '#l10n 92 (ES - Hello on viernes, 1 de enero de 1960)\n'
+      '#l10n 93 (ES - Hello World, on viernes, 1 de enero de 1960)\n'
+      '#l10n 94 (ES - Hello two worlds, on viernes, 1 de enero de 1960)\n'
+      '#l10n 95 (ES - Hello other 0 worlds, with a total of 100 citizens)\n'
+      '#l10n 96 (ES - Hello World of 101 citizens)\n'
+      '#l10n 97 (ES - Hello two worlds with 102 total citizens)\n'
+      '#l10n 98 (ES - [Hola] -Mundo- #123#)\n'
+      '#l10n 99 (ES - \$!)\n'
+      '#l10n 100 (ES - One \$)\n'
+      "#l10n 101 (ES - Flutter's amazing!)\n"
+      "#l10n 102 (ES - Flutter's amazing, times 2!)\n"
+      '#l10n 103 (ES - Flutter is "amazing"!)\n'
+      '#l10n 104 (ES - Flutter is "amazing", times 2!)\n'
+      '#l10n 105 (ES - 16 wheel truck)\n'
+      "#l10n 106 (ES - Sedan's elegance)\n"
+      '#l10n 107 (ES - Cabriolet has "acceleration")\n'
+      '#l10n 108 (ES - Oh, she found ES - 1 itemES - !)\n'
+      '#l10n 109 (ES - Indeed, ES - they like ES - Flutter!)\n'
+      '#l10n 110 (--- es_419 ---)\n'
+      '#l10n 111 (ES 419 - Hello World)\n'
+      '#l10n 112 (ES 419 - Hello)\n'
+      '#l10n 113 (ES 419 - Hello World)\n'
+      '#l10n 114 (ES 419 - Hello two worlds)\n'
       '#l10n END\n'
     );
   }

packages/flutter_tools/test/integration.shard/test_data/gen_l10n_project.dart

@@ -229,6 +229,9 @@ class Home extends StatelessWidget {
               "${localizations.selectInString('he')}",
               "${localizations.selectWithPlaceholder('male', 'ice cream')}",
               "${localizations.selectWithPlaceholder('female', 'chocolate')}",
+              "${localizations.selectInPlural('male', 1)}",
+              "${localizations.selectInPlural('male', 2)}",
+              "${localizations.selectInPlural('female', 1)}",
             ]);
           },
         ),
@@ -627,7 +630,7 @@ void main() {
     }
   },
 
-  "singleQuoteSelect": "{vehicleType, select, sedan{Sedan's elegance} cabriolet{Cabriolet' acceleration} truck{truck's heavy duty} other{Other's mirrors!}}",
+  "singleQuoteSelect": "{vehicleType, select, sedan{Sedan's elegance} cabriolet{Cabriolet's acceleration} truck{truck's heavy duty} other{Other's mirrors!}}",
   "@singleQuoteSelect": {
     "description": "A select message with a single quote.",
     "placeholders": {
@@ -666,6 +669,19 @@ void main() {
       "gender": {},
       "preference": {}
     }
+  },
+
+  "selectInPlural": "{count, plural, =1{{gender, select, male{he} female{she} other{they}}} other{they}}",
+  "@selectInPlural": {
+    "description": "Pronoun dependent on the count and gender.",
+    "placeholders": {
+      "gender": {
+        "type": "String"
+      },
+      "count": {
+        "type": "num"
+      }
+    }
   }
 }
 ''';
@@ -702,7 +718,7 @@ void main() {
   "helloFor": "ES - Hello for {value}",
   "helloAdjectiveWorlds": "{count,plural, =0{ES - Hello} =1{ES - Hello {adjective} World} =2{ES - Hello two {adjective} worlds} other{ES - Hello other {count} {adjective} worlds}}",
   "helloWorldsOn": "{count,plural, =0{ES - Hello on {date}} =1{ES - Hello World, on {date}} =2{ES - Hello two worlds, on {date}} other{ES - Hello other {count} worlds, on {date}}}",
-  "helloWorldPopulation": "{ES - count,plural, =1{ES - Hello World of {population} citizens} =2{ES - Hello two worlds with {population} total citizens} many{ES - Hello all {count} worlds, with a total of {population} citizens} other{ES - Hello other {count} worlds, with a total of {population} citizens}}",
+  "helloWorldPopulation": "{count,plural, =1{ES - Hello World of {population} citizens} =2{ES - Hello two worlds with {population} total citizens} many{ES - Hello all {count} worlds, with a total of {population} citizens} other{ES - Hello other {count} worlds, with a total of {population} citizens}}",
   "helloWorldInterpolation": "ES - [{hello}] #{world}#",
   "helloWorldsInterpolation": "{count,plural, other {ES - [{hello}] -{world}- #{count}#}}",
   "dollarSign": "ES - $!",