misc .arb fixes; localizations validator (#12197)

* misc .arb fixes; localizations validator

* regenerate localizations

* address comments

* do not treat plural variations as invalid keys

dev/bots/test.dart

@@ -52,8 +52,42 @@ Future<Null> _generateDocs() async {
   print('${bold}DONE: test.dart does nothing in the docs shard.$reset');
 }
 
+Future<Null> _verifyInternationalizations() async {
+  final EvalResult genResult = await _evalCommand(
+    dart,
+    <String>[
+      path.join('dev', 'tools', 'gen_localizations.dart'),
+      path.join('packages', 'flutter', 'lib', 'src', 'material', 'i18n'),
+      'material'
+    ],
+    workingDirectory: flutterRoot,
+  );
+
+  final String localizationsFile = path.join('packages', 'flutter', 'lib', 'src', 'material', 'i18n', 'localizations.dart');
+
+  final EvalResult sourceContents = await _evalCommand(
+    'cat',
+    <String>[localizationsFile],
+    workingDirectory: flutterRoot,
+  );
+
+  if (genResult.stdout.trim() != sourceContents.stdout.trim()) {
+    stderr
+      ..writeln('<<<<<<< $localizationsFile')
+      ..writeln(sourceContents.stdout.trim())
+      ..writeln('=======')
+      ..writeln(genResult.stdout.trim())
+      ..writeln('>>>>>>> gen_localizations')
+      ..writeln('The contents of $localizationsFile are different from that produced by gen_localizations.')
+      ..writeln()
+      ..writeln('Did you forget to run gen_localizations.dart after updating a .arb file?');
+    exit(1);
+  }
+}
+
 Future<Null> _analyzeRepo() async {
   await _verifyNoBadImports(flutterRoot);
+  await _verifyInternationalizations();
 
   // Analyze all the Dart code in the repo.
   await _runFlutterAnalyze(flutterRoot,
@@ -176,6 +210,57 @@ Future<Null> _pubRunTest(
   return _runCommand(pub, args, workingDirectory: workingDirectory);
 }
 
+class EvalResult {
+  EvalResult({
+    this.stdout,
+    this.stderr,
+  });
+
+  final String stdout;
+  final String stderr;
+}
+
+Future<EvalResult> _evalCommand(String executable, List<String> arguments, {
+  String workingDirectory,
+  Map<String, String> environment,
+  bool skip: false,
+}) async {
+  final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}';
+  final String relativeWorkingDir = path.relative(workingDirectory);
+  if (skip) {
+    _printProgress('SKIPPING', relativeWorkingDir, commandDescription);
+    return null;
+  }
+  _printProgress('RUNNING', relativeWorkingDir, commandDescription);
+
+  final Process process = await Process.start(executable, arguments,
+    workingDirectory: workingDirectory,
+    environment: environment,
+  );
+
+  final Future<List<List<int>>> savedStdout = process.stdout.toList();
+  final Future<List<List<int>>> savedStderr = process.stderr.toList();
+  final int exitCode = await process.exitCode;
+  final EvalResult result = new EvalResult(
+    stdout: UTF8.decode((await savedStdout).expand((List<int> ints) => ints).toList()),
+    stderr: UTF8.decode((await savedStderr).expand((List<int> ints) => ints).toList()),
+  );
+
+  if (exitCode != 0) {
+    stderr.write(result.stderr);
+    print(
+      '$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset\n'
+      '${bold}ERROR:$red Last command exited with $exitCode.$reset\n'
+      '${bold}Command:$red $commandDescription$reset\n'
+      '${bold}Relative working directory:$red $relativeWorkingDir$reset\n'
+      '$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset'
+    );
+    exit(1);
+  }
+
+  return result;
+}
+
 Future<Null> _runCommand(String executable, List<String> arguments, {
   String workingDirectory,
   Map<String, String> environment,

dev/tools/gen_localizations.dart

@@ -26,6 +26,8 @@
 import 'dart:convert' show JSON;
 import 'dart:io';
 
+import 'localizations_validator.dart';
+
 const String outputHeader = '''
 // Copyright 2017 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -36,8 +38,14 @@ const String outputHeader = '''
 // @(regenerate)
 ''';
 
+/// Maps locales to resource key/value pairs.
 final Map<String, Map<String, String>> localeToResources = <String, Map<String, String>>{};
 
+/// Maps locales to resource attributes.
+/// 
+/// See also https://github.com/googlei18n/app-resource-bundle/wiki/ApplicationResourceBundleSpecification#resource-attributes
+final Map<String, Map<String, dynamic>> localeToResourceAttributes = <String, Map<String, dynamic>>{};
+
 // Return s as a Dart-parseable raw string in double quotes. Expand double quotes:
 // foo => r"foo"
 // foo "bar" => r"foo " '"' r"bar" '"'
@@ -92,12 +100,15 @@ const Map<String, Map<String, String>> localizations = const <String, Map<String
 
 void processBundle(File file, String locale) {
   localeToResources[locale] ??= <String, String>{};
+  localeToResourceAttributes[locale] ??= <String, dynamic>{};
   final Map<String, String> resources = localeToResources[locale];
+  final Map<String, dynamic> attributes = localeToResourceAttributes[locale];
   final Map<String, dynamic> bundle = JSON.decode(file.readAsStringSync());
   for (String key in bundle.keys) {
     // The ARB file resource "attributes" for foo are called @foo.
     if (key.startsWith('@'))
-      continue;
+      attributes[key.substring(1)] = bundle[key];
+    else
       resources[key] = bundle[key];
   }
 }
@@ -121,6 +132,7 @@ void main(List<String> args) {
       processBundle(new File(path), locale);
     }
   }
+  validateLocalizations(localeToResources, localeToResourceAttributes);
 
   final String regenerate = 'dart dev/tools/gen_localizations.dart ${directory.path} ${args[1]}';
   print(outputHeader.replaceFirst('@(regenerate)', regenerate));

dev/tools/localizations_validator.dart

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+/// Enforces the following invariants in our localizations:
+/// 
+/// - Resource keys are valid, i.e. they appear in the canonical list.
+/// - Resource keys are complete for language-level locales, e.g. "es", "he".
+/// 
+/// Uses "en" localizations as the canonical source of locale keys that other
+/// locales are compared against.
+/// 
+/// If validation fails, print an error message to STDERR and quit with exit
+/// code 1.
+void validateLocalizations(
+  Map<String, Map<String, String>> localeToResources,
+  Map<String, Map<String, dynamic>> localeToAttributes,
+) {
+  final Map<String, String> canonicalLocalizations = localeToResources['en'];
+  final Set<String> canonicalKeys = new Set<String>.from(canonicalLocalizations.keys);
+  final StringBuffer errorMessages = new StringBuffer();
+  bool explainMissingKeys = false;
+  for (final String locale in localeToResources.keys) {
+    final Map<String, String> resources = localeToResources[locale];
+
+    // Whether `key` corresponds to one of the plural variations of a key with
+    // the same prefix and suffix "Other".
+    //
+    // Many languages require only a subset of these variations, so we do not
+    // require them so long as the "Other" variation exists.
+    bool isPluralVariation(String key) {
+      final RegExp pluralRegexp = new RegExp(r'(\w*)(Zero|One|Two|Few|Many)$');
+      final Match pluralMatch = pluralRegexp.firstMatch(key);
+      
+      if (pluralMatch == null)
+        return false;
+
+      final String prefix = pluralMatch[1];
+      return resources.containsKey('${prefix}Other');
+    }
+
+    final Set<String> keys = new Set<String>.from(
+      resources.keys.where((String key) => !isPluralVariation(key))
+    );
+
+    // Make sure keys are valid (i.e. they also exist in the canonical
+    // localizations)
+    final Set<String> invalidKeys = keys.difference(canonicalKeys);
+    if (invalidKeys.isNotEmpty)
+      errorMessages.writeln('Locale "$locale" contains invalid resource keys: ${invalidKeys.join(', ')}');
+
+    // For language-level locales only, check that they have a complete list of
+    // keys, or opted out of using certain ones.
+    if (locale.length == 2) {
+      final Map<String, dynamic> attributes = localeToAttributes[locale];
+      final List<String> missingKeys = <String>[];
+
+      for (final String missingKey in canonicalKeys.difference(keys)) {
+        final dynamic attribute = attributes[missingKey];
+        final bool intentionallyOmitted = attribute is Map && attribute.containsKey('notUsed');
+        if (!intentionallyOmitted && !isPluralVariation(missingKey))
+          missingKeys.add(missingKey);
+      }
+      if (missingKeys.isNotEmpty) {
+        explainMissingKeys = true;
+        errorMessages.writeln('Locale "$locale" is missing the following resource keys: ${missingKeys.join(', ')}');
+      }
+    }
+  }
+
+  if (errorMessages.isNotEmpty) {
+    if (explainMissingKeys) {
+        errorMessages
+          ..writeln()
+          ..writeln(
+            'If a resource key is intentionally omitted, add an attribute corresponding '
+            'to the key name with a "notUsed" property explaining why. Example:'
+          )
+          ..writeln()
+          ..writeln('"@anteMeridiemAbbreviation": {')
+          ..writeln('  "notUsed": "Sindhi time format does not use a.m. indicator"')
+          ..writeln('}');
+    }
+
+    stderr.writeln('ERROR:');
+    stderr.writeln(errorMessages);
+    exit(1);
+  }
+}

packages/flutter/lib/src/material/i18n/localizations.dart

@@ -4,7 +4,7 @@
 
 // This file has been automatically generated.  Please do not edit it manually.
 // To regenerate the file, use:
-// dart dev/tools/gen_localizations.dart lib/src/material/i18n material
+// dart dev/tools/gen_localizations.dart packages/flutter/lib/src/material/i18n material
 
 /// Maps from [Locale.languageCode] to a map that contains the localized strings
 /// for that locale.
@@ -136,6 +136,8 @@ const Map<String, Map<String, String>> localizations = const <String, Map<String
   },
   "es_US": const <String, String>{
     "timeOfDayFormat": r"h:mm a",
+    "anteMeridiemAbbreviation": r"AM",
+    "postMeridiemAbbreviation": r"PM",
   },
   "fa": const <String, String>{
     "timeOfDayFormat": r"H:mm",
@@ -339,7 +341,9 @@ const Map<String, Map<String, String>> localizations = const <String, Map<String
     "rowsPerPageTitle": r"Строки на страницу:",
     "aboutListTileTitle": r"O $applicationName",
     "licensesPageTitle": r"Лицензии",
-    "selectedRowCountTitleOther": r"Выбранно $selectedRowCount строк",
+    "selectedRowCountTitleZero": r"Строки не выбраны",
+    "selectedRowCountTitleOne": r"Выбрана 1 строка",
+    "selectedRowCountTitleOther": r"Выбрано $selectedRowCount строк",
     "cancelButtonLabel": r"ОТМЕНИТЬ",
     "closeButtonLabel": r"ЗАКРЫТЬ",
     "continueButtonLabel": r"ПРОДОЛЖИТЬ",

packages/flutter/lib/src/material/i18n/material_de.arb

 {
   "timeOfDayFormat": "HH:mm",
+  "@anteMeridiemAbbreviation": { "notUsed": "German time format does not use a.m. indicator" },
+  "@postMeridiemAbbreviation": { "notUsed": "German time format does not use p.m. indicator" },
   "openAppDrawerTooltip": "Navigationsmenü öffnen",
   "backButtonTooltip": "Zurück",
   "closeButtonTooltip": "Schließen",

packages/flutter/lib/src/material/i18n/material_en.arb

@@ -67,7 +67,7 @@
 
   "pageRowsInfoTitle": "$firstRow–$lastRow of $rowCount",
   "pageRowsInfoTitleApproximate": "$firstRow–$lastRow of about $rowCount",
-  "@pageRowInfoTitle": {
+  "@pageRowsInfoTitle": {
     "description": "Title for the [PaginatedDataTable]'s row info footer",
     "type": "text"
   },

packages/flutter/lib/src/material/i18n/material_es.arb

 {
   "timeOfDayFormat": "H:mm",
+  "@anteMeridiemAbbreviation": { "notUsed": "Standard Spanish time format does not use a.m. indicator" },
+  "@postMeridiemAbbreviation": { "notUsed": "Standard Spanish time format does not use p.m. indicator" },
   "openAppDrawerTooltip": "Abrir el menú de navegación",
   "backButtonTooltip": "Espalda",
   "closeButtonTooltip": "Cerrar",

packages/flutter/lib/src/material/i18n/material_es_US.arb

 {
-  "timeOfDayFormat": "h:mm a"
+  "timeOfDayFormat": "h:mm a",
+  "anteMeridiemAbbreviation": "AM",
+  "postMeridiemAbbreviation": "PM"
 }

packages/flutter/lib/src/material/i18n/material_fa.arb

 {
   "timeOfDayFormat": "H:mm",
+  "@anteMeridiemAbbreviation": { "notUsed": "Farsi time format does not use a.m. indicator" },
+  "@postMeridiemAbbreviation": { "notUsed": "Farsi time format does not use p.m. indicator" },
   "openAppDrawerTooltip": "منوی ناوبری را باز کنید",
   "backButtonTooltip": "بازگشت",
   "closeButtonTooltip": "بستن",

packages/flutter/lib/src/material/i18n/material_fr.arb

 {
   "timeOfDayFormat": "HH:mm",
+  "@anteMeridiemAbbreviation": { "notUsed": "French time format does not use a.m. indicator" },
+  "@postMeridiemAbbreviation": { "notUsed": "French time format does not use p.m. indicator" },
   "openAppDrawerTooltip": "Ouvrir le menu de navigation",
   "backButtonTooltip": "Retour",
   "closeButtonTooltip": "Fermer",

packages/flutter/lib/src/material/i18n/material_he.arb

 {
   "timeOfDayFormat": "H:mm",
+  "@anteMeridiemAbbreviation": { "notUsed": "Hebrew time format does not use a.m. indicator" },
+  "@postMeridiemAbbreviation": { "notUsed": "Hebrew time format does not use p.m. indicator" },
   "openAppDrawerTooltip": "פתח תפריט ניווט",
   "backButtonTooltip": "אחורה",
   "closeButtonTooltip": "סגור",

packages/flutter/lib/src/material/i18n/material_it.arb

 {
   "timeOfDayFormat": "HH:mm",
+  "@anteMeridiemAbbreviation": { "notUsed": "Italian time format does not use a.m. indicator" },
+  "@postMeridiemAbbreviation": { "notUsed": "Italian time format does not use p.m. indicator" },
   "openAppDrawerTooltip": "Apri il menu di navigazione",
   "backButtonTooltip": "Indietro",
   "closeButtonTooltip": "Chiudi",

packages/flutter/lib/src/material/i18n/material_ja.arb

 {
   "timeOfDayFormat": "H:mm",
+  "@anteMeridiemAbbreviation": { "notUsed": "Japanese time format does not use a.m. indicator" },
+  "@postMeridiemAbbreviation": { "notUsed": "Japanese time format does not use p.m. indicator" },
   "openAppDrawerTooltip": "ナビゲーションメニューを開く",
   "backButtonTooltip": "戻る",
   "closeButtonTooltip": "閉じる",

packages/flutter/lib/src/material/i18n/material_ps.arb

 {
   "timeOfDayFormat": "HH:mm",
+  "@anteMeridiemAbbreviation": { "notUsed": "Pashto time format does not use a.m. indicator" },
+  "@postMeridiemAbbreviation": { "notUsed": "Pashto time format does not use p.m. indicator" },
   "openAppDrawerTooltip": "د پرانیستی نیینګ مینو",
   "backButtonTooltip": "شاته",
   "closeButtonTooltip": "بنده",

packages/flutter/lib/src/material/i18n/material_pt.arb

 {
   "timeOfDayFormat": "HH:mm",
+  "@anteMeridiemAbbreviation": { "notUsed": "Portuguese time format does not use a.m. indicator" },
+  "@postMeridiemAbbreviation": { "notUsed": "Portuguese time format does not use p.m. indicator" },
   "openAppDrawerTooltip": "Abrir menu de navegação",
   "backButtonTooltip": "Costas",
   "closeButtonTooltip": "Fechar",

packages/flutter/lib/src/material/i18n/material_ru.arb

 {
   "timeOfDayFormat": "H:mm",
+  "@anteMeridiemAbbreviation": { "notUsed": "Russian time format does not use a.m. indicator" },
+  "@postMeridiemAbbreviation": { "notUsed": "Russian time format does not use p.m. indicator" },
   "openAppDrawerTooltip": "Открыть меню навигации",
   "backButtonTooltip": "Назад",
   "closeButtonTooltip": "Закрыть",
@@ -13,7 +15,9 @@
   "rowsPerPageTitle": "Строки на страницу:",
   "aboutListTileTitle": "O $applicationName",
   "licensesPageTitle": "Лицензии",
-  "selectedRowCountTitleOther": "Выбранно $selectedRowCount строк",
+  "selectedRowCountTitleZero": "Строки не выбраны",
+  "selectedRowCountTitleOne": "Выбрана 1 строка",
+  "selectedRowCountTitleOther": "Выбрано $selectedRowCount строк",
   "cancelButtonLabel": "ОТМЕНИТЬ",
   "closeButtonLabel": "ЗАКРЫТЬ",
   "continueButtonLabel": "ПРОДОЛЖИТЬ",

packages/flutter/lib/src/material/i18n/material_sd.arb

 {
   "timeOfDayFormat": "HH:mm",
+  "@anteMeridiemAbbreviation": { "notUsed": "Sindhi time format does not use a.m. indicator" },
+  "@postMeridiemAbbreviation": { "notUsed": "Sindhi time format does not use p.m. indicator" },
   "openAppDrawerTooltip": "اوپن جي مينڊيٽ مينيو",
   "backButtonTooltip": "پوئتي",
   "closeButtonTooltip": "بند ڪريو",