localizations_utils.dart 16.1 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
import 'dart:convert';
6 7 8 9 10
import 'dart:io';

import 'package:args/args.dart' as argslib;
import 'package:meta/meta.dart';

11 12
import 'language_subtag_registry.dart';

13 14 15
typedef HeaderGenerator = String Function(String regenerateInstructions);
typedef ConstructorGenerator = String Function(LocaleInfo locale);

16 17 18 19
int sortFilesByPath (FileSystemEntity a, FileSystemEntity b) {
  return a.path.compareTo(b.path);
}

20
/// Simple data class to hold parsed locale. Does not promise validity of any data.
21
@immutable
22
class LocaleInfo implements Comparable<LocaleInfo> {
23
  const LocaleInfo({
24
    required this.languageCode,
25 26
    this.scriptCode,
    this.countryCode,
27 28
    required this.length,
    required this.originalString,
29 30 31
  });

  /// Simple parser. Expects the locale string to be in the form of 'language_script_COUNTRY'
Chris Bracken's avatar
Chris Bracken committed
32
  /// where the language is 2 characters, script is 4 characters with the first uppercase,
33 34 35
  /// and country is 2-3 characters and all uppercase.
  ///
  /// 'language_COUNTRY' or 'language_script' are also valid. Missing fields will be null.
36 37 38 39
  ///
  /// When `deriveScriptCode` is true, if [scriptCode] was unspecified, it will
  /// be derived from the [languageCode] and [countryCode] if possible.
  factory LocaleInfo.fromString(String locale, { bool deriveScriptCode = false }) {
40 41 42
    final List<String> codes = locale.split('_'); // [language, script, country]
    assert(codes.isNotEmpty && codes.length < 4);
    final String languageCode = codes[0];
43 44
    String? scriptCode;
    String? countryCode;
45 46 47 48 49 50 51 52 53
    int length = codes.length;
    String originalString = locale;
    if (codes.length == 2) {
      scriptCode = codes[1].length >= 4 ? codes[1] : null;
      countryCode = codes[1].length < 4 ? codes[1] : null;
    } else if (codes.length == 3) {
      scriptCode = codes[1].length > codes[2].length ? codes[1] : codes[2];
      countryCode = codes[1].length < codes[2].length ? codes[1] : codes[2];
    }
54
    assert(codes[0].isNotEmpty);
55 56 57 58 59 60 61 62 63
    assert(countryCode == null || countryCode.isNotEmpty);
    assert(scriptCode == null || scriptCode.isNotEmpty);

    /// Adds scriptCodes to locales where we are able to assume it to provide
    /// finer granularity when resolving locales.
    ///
    /// The basis of the assumptions here are based off of known usage of scripts
    /// across various countries. For example, we know Taiwan uses traditional (Hant)
    /// script, so it is safe to apply (Hant) to Taiwanese languages.
64
    if (deriveScriptCode && scriptCode == null) {
65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
      switch (languageCode) {
        case 'zh': {
          if (countryCode == null) {
            scriptCode = 'Hans';
          }
          switch (countryCode) {
            case 'CN':
            case 'SG':
              scriptCode = 'Hans';
            case 'TW':
            case 'HK':
            case 'MO':
              scriptCode = 'Hant';
          }
          break;
        }
        case 'sr': {
          if (countryCode == null) {
            scriptCode = 'Cyrl';
          }
          break;
        }
      }
      // Increment length if we were able to assume a scriptCode.
      if (scriptCode != null) {
        length += 1;
      }
      // Update the base string to reflect assumed scriptCodes.
      originalString = languageCode;
94
      if (scriptCode != null) {
95
        originalString += '_$scriptCode';
96 97
      }
      if (countryCode != null) {
98
        originalString += '_$countryCode';
99
      }
100 101 102 103 104 105 106 107 108 109 110 111
    }

    return LocaleInfo(
      languageCode: languageCode,
      scriptCode: scriptCode,
      countryCode: countryCode,
      length: length,
      originalString: originalString,
    );
  }

  final String languageCode;
112 113
  final String? scriptCode;
  final String? countryCode;
114 115 116
  final int length;             // The number of fields. Ranges from 1-3.
  final String originalString;  // Original un-parsed locale string.

117 118 119 120
  String camelCase() {
    return originalString
      .split('_')
      .map<String>((String part) => part.substring(0, 1).toUpperCase() + part.substring(1).toLowerCase())
121
      .join();
122 123
  }

124 125
  @override
  bool operator ==(Object other) {
126 127
    return other is LocaleInfo
        && other.originalString == originalString;
128 129 130
  }

  @override
131
  int get hashCode => originalString.hashCode;
132 133 134 135 136 137 138 139 140 141 142 143

  @override
  String toString() {
    return originalString;
  }

  @override
  int compareTo(LocaleInfo other) {
    return originalString.compareTo(other.originalString);
  }
}

144 145 146
/// Parse the data for a locale from a file, and store it in the [attributes]
/// and [resources] keys.
void loadMatchingArbsIntoBundleMaps({
147 148 149 150
  required Directory directory,
  required RegExp filenamePattern,
  required Map<LocaleInfo, Map<String, String>> localeToResources,
  required Map<LocaleInfo, Map<String, dynamic>> localeToResourceAttributes,
151 152 153 154 155 156 157 158 159 160
}) {

  /// Set that holds the locales that were assumed from the existing locales.
  ///
  /// For example, when the data lacks data for zh_Hant, we will use the data of
  /// the first Hant Chinese locale as a default by repeating the data. If an
  /// explicit match is later found, we can reference this set to see if we should
  /// overwrite the existing assumed data.
  final Set<LocaleInfo> assumedLocales = <LocaleInfo>{};

161
  for (final FileSystemEntity entity in directory.listSync().toList()..sort(sortFilesByPath)) {
162 163
    final String entityPath = entity.path;
    if (FileSystemEntity.isFileSync(entityPath) && filenamePattern.hasMatch(entityPath)) {
164
      final String localeString = filenamePattern.firstMatch(entityPath)![1]!;
165 166 167 168
      final File arbFile = File(entityPath);

      // Helper method to fill the maps with the correct data from file.
      void populateResources(LocaleInfo locale, File file) {
169 170
        final Map<String, String> resources = localeToResources[locale]!;
        final Map<String, dynamic> attributes = localeToResourceAttributes[locale]!;
171
        final Map<String, dynamic> bundle = json.decode(file.readAsStringSync()) as Map<String, dynamic>;
172
        for (final String key in bundle.keys) {
173
          // The ARB file resource "attributes" for foo are called @foo.
174
          if (key.startsWith('@')) {
175
            attributes[key.substring(1)] = bundle[key];
176
          } else {
177
            resources[key] = bundle[key] as String;
178
          }
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
        }
      }
      // Only pre-assume scriptCode if there is a country or script code to assume off of.
      // When we assume scriptCode based on languageCode-only, we want this initial pass
      // to use the un-assumed version as a base class.
      LocaleInfo locale = LocaleInfo.fromString(localeString, deriveScriptCode: localeString.split('_').length > 1);
      // Allow overwrite if the existing data is assumed.
      if (assumedLocales.contains(locale)) {
        localeToResources[locale] = <String, String>{};
        localeToResourceAttributes[locale] = <String, dynamic>{};
        assumedLocales.remove(locale);
      } else {
        localeToResources[locale] ??= <String, String>{};
        localeToResourceAttributes[locale] ??= <String, dynamic>{};
      }
      populateResources(locale, arbFile);
      // Add an assumed locale to default to when there is no info on scriptOnly locales.
      locale = LocaleInfo.fromString(localeString, deriveScriptCode: true);
      if (locale.scriptCode != null) {
198
        final LocaleInfo scriptLocale = LocaleInfo.fromString('${locale.languageCode}_${locale.scriptCode}');
199 200 201 202 203 204 205 206 207 208 209
        if (!localeToResources.containsKey(scriptLocale)) {
          assumedLocales.add(scriptLocale);
          localeToResources[scriptLocale] ??= <String, String>{};
          localeToResourceAttributes[scriptLocale] ??= <String, dynamic>{};
          populateResources(scriptLocale, arbFile);
        }
      }
    }
  }
}

210
void exitWithError(String errorMessage) {
211
  stderr.writeln('fatal: $errorMessage');
212 213 214 215
  exit(1);
}

void checkCwdIsRepoRoot(String commandName) {
216
  final bool isRepoRoot = Directory('.git').existsSync();
217 218

  if (!isRepoRoot) {
219
    exitWithError(
220 221 222 223 224 225 226
      '$commandName must be run from the root of the Flutter repository. The '
      'current working directory is: ${Directory.current.path}'
    );
  }
}

GeneratorOptions parseArgs(List<String> rawArgs) {
227
  final argslib.ArgParser argParser = argslib.ArgParser()
228 229 230 231 232
    ..addFlag(
      'help',
      abbr: 'h',
      help: 'Print the usage message for this command',
    )
233 234 235
    ..addFlag(
      'overwrite',
      abbr: 'w',
236 237 238 239 240
      help: 'Overwrite existing localizations',
    )
    ..addFlag(
      'remove-undefined',
      help: 'Remove any localizations that are not defined in the canonical locale.',
241
    )
242 243 244 245
    ..addFlag(
      'widgets',
      help: 'Whether to print the generated classes for the Widgets package only. Ignored when --overwrite is passed.',
    )
246 247 248 249 250 251 252
    ..addFlag(
      'material',
      help: 'Whether to print the generated classes for the Material package only. Ignored when --overwrite is passed.',
    )
    ..addFlag(
      'cupertino',
      help: 'Whether to print the generated classes for the Cupertino package only. Ignored when --overwrite is passed.',
253 254
    );
  final argslib.ArgResults args = argParser.parse(rawArgs);
255 256 257 258
  if (args.wasParsed('help') && args['help'] == true) {
    stderr.writeln(argParser.usage);
    exit(0);
  }
259
  final bool writeToFile = args['overwrite'] as bool;
260
  final bool removeUndefined = args['remove-undefined'] as bool;
261
  final bool widgetsOnly = args['widgets'] as bool;
262 263
  final bool materialOnly = args['material'] as bool;
  final bool cupertinoOnly = args['cupertino'] as bool;
264

265 266 267 268
  return GeneratorOptions(
    writeToFile: writeToFile,
    materialOnly: materialOnly,
    cupertinoOnly: cupertinoOnly,
269
    widgetsOnly: widgetsOnly,
270 271
    removeUndefined: removeUndefined,
  );
272 273 274 275
}

class GeneratorOptions {
  GeneratorOptions({
276
    required this.writeToFile,
277
    required this.removeUndefined,
278 279
    required this.materialOnly,
    required this.cupertinoOnly,
280
    required this.widgetsOnly,
281 282 283
  });

  final bool writeToFile;
284
  final bool removeUndefined;
285 286
  final bool materialOnly;
  final bool cupertinoOnly;
287
  final bool widgetsOnly;
288
}
289 290 291 292

// See also //master/tools/gen_locale.dart in the engine repo.
Map<String, List<String>> _parseSection(String section) {
  final Map<String, List<String>> result = <String, List<String>>{};
293
  late List<String> lastHeading;
294
  for (final String line in section.split('\n')) {
295
    if (line == '') {
296
      continue;
297
    }
298 299 300 301 302
    if (line.startsWith('  ')) {
      lastHeading[lastHeading.length - 1] = '${lastHeading.last}${line.substring(1)}';
      continue;
    }
    final int colon = line.indexOf(':');
303
    if (colon <= 0) {
304
      throw 'not sure how to deal with "$line"';
305
    }
306 307 308
    final String name = line.substring(0, colon);
    final String value = line.substring(colon + 2);
    lastHeading = result.putIfAbsent(name, () => <String>[]);
309
    result[name]!.add(value);
310 311 312 313 314 315 316 317 318 319 320 321 322
  }
  return result;
}

final Map<String, String> _languages = <String, String>{};
final Map<String, String> _regions = <String, String>{};
final Map<String, String> _scripts = <String, String>{};
const String kProvincePrefix = ', Province of ';
const String kParentheticalPrefix = ' (';

/// Prepares the data for the [describeLocale] method below.
///
/// The data is obtained from the official IANA registry.
323 324 325
void precacheLanguageAndRegionTags() {
  final List<Map<String, List<String>>> sections =
      languageSubtagRegistry.split('%%').skip(1).map<Map<String, List<String>>>(_parseSection).toList();
326
  for (final Map<String, List<String>> section in sections) {
327
    assert(section.containsKey('Type'), section.toString());
328
    final String type = section['Type']!.single;
329 330
    if (type == 'language' || type == 'region' || type == 'script') {
      assert(section.containsKey('Subtag') && section.containsKey('Description'), section.toString());
331 332
      final String subtag = section['Subtag']!.single;
      String description = section['Description']!.join(' ');
333
      if (description.startsWith('United ')) {
334
        description = 'the $description';
335 336
      }
      if (description.contains(kParentheticalPrefix)) {
337
        description = description.substring(0, description.indexOf(kParentheticalPrefix));
338 339
      }
      if (description.contains(kProvincePrefix)) {
340
        description = description.substring(0, description.indexOf(kProvincePrefix));
341 342
      }
      if (description.endsWith(' Republic')) {
343
        description = 'the $description';
344
      }
345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
      switch (type) {
        case 'language':
          _languages[subtag] = description;
        case 'region':
          _regions[subtag] = description;
        case 'script':
          _scripts[subtag] = description;
      }
    }
  }
}

String describeLocale(String tag) {
  final List<String> subtags = tag.split('_');
  assert(subtags.isNotEmpty);
  assert(_languages.containsKey(subtags[0]));
361
  final String language = _languages[subtags[0]]!;
362
  String output = language;
363 364
  String? region;
  String? script;
365 366 367
  if (subtags.length == 2) {
    region = _regions[subtags[1]];
    script = _scripts[subtags[1]];
368
    assert(region != null || script != null);
369 370 371 372
  } else if (subtags.length >= 3) {
    region = _regions[subtags[2]];
    script = _scripts[subtags[1]];
    assert(region != null && script != null);
373
  }
374
  if (region != null) {
375
    output += ', as used in $region';
376 377
  }
  if (script != null) {
378
    output += ', using the $script script';
379
  }
380
  return output;
381 382 383 384 385 386 387 388
}

/// Writes the header of each class which corresponds to a locale.
String generateClassDeclaration(
  LocaleInfo locale,
  String classNamePrefix,
  String superClass,
) {
389
  final String camelCaseName = locale.camelCase();
390 391 392 393 394 395
  return '''

/// The translations for ${describeLocale(locale.originalString)} (`${locale.originalString}`).
class $classNamePrefix$camelCaseName extends $superClass {''';
}

396
/// Return the input string as a Dart-parseable string.
397 398
///
/// ```
399 400 401 402
/// foo => 'foo'
/// foo "bar" => 'foo "bar"'
/// foo 'bar' => "foo 'bar'"
/// foo 'bar' "baz" => '''foo 'bar' "baz"'''
403 404
/// ```
///
405 406
/// This function is used by tools that take in a JSON-formatted file to
/// generate Dart code. For this reason, characters with special meaning
407 408 409
/// in JSON files are escaped. For example, the backspace character (\b)
/// has to be properly escaped by this function so that the generated
/// Dart code correctly represents this character:
410 411 412 413 414 415
/// ```
/// foo\bar => 'foo\\bar'
/// foo\nbar => 'foo\\nbar'
/// foo\\nbar => 'foo\\\\nbar'
/// foo\\bar => 'foo\\\\bar'
/// foo\ bar => 'foo\\ bar'
416
/// foo$bar = 'foo\$bar'
417
/// ```
418
String generateString(String value) {
419 420 421 422 423 424 425 426 427 428 429 430 431
  if (<String>['\n', '\f', '\t', '\r', '\b'].every((String pattern) => !value.contains(pattern))) {
    final bool hasDollar = value.contains(r'$');
    final bool hasBackslash = value.contains(r'\');
    final bool hasQuote = value.contains("'");
    final bool hasDoubleQuote = value.contains('"');
    if (!hasQuote) {
      return hasBackslash || hasDollar ? "r'$value'" : "'$value'";
    }
    if (!hasDoubleQuote) {
      return hasBackslash || hasDollar ? 'r"$value"' : '"$value"';
    }
  }

432 433 434 435 436 437 438 439 440
  const String backslash = '__BACKSLASH__';
  assert(
    !value.contains(backslash),
    'Input string cannot contain the sequence: '
    '"__BACKSLASH__", as it is used as part of '
    'backslash character processing.'
  );

  value = value
441 442
    // Replace backslashes with a placeholder for now to properly parse
    // other special characters.
443 444 445 446 447 448 449 450 451
    .replaceAll(r'\', backslash)
    .replaceAll(r'$', r'\$')
    .replaceAll("'", r"\'")
    .replaceAll('"', r'\"')
    .replaceAll('\n', r'\n')
    .replaceAll('\f', r'\f')
    .replaceAll('\t', r'\t')
    .replaceAll('\r', r'\r')
    .replaceAll('\b', r'\b')
452
    // Reintroduce escaped backslashes into generated Dart string.
453
    .replaceAll(backslash, r'\\');
454 455

  return "'$value'";
456
}
457 458 459 460

/// Only used to generate localization strings for the Kannada locale ('kn') because
/// some of the localized strings contain characters that can crash Emacs on Linux.
/// See packages/flutter_localizations/lib/src/l10n/README for more information.
461
String generateEncodedString(String? locale, String value) {
462
  if (locale != 'kn' || value.runes.every((int code) => code <= 0xFF)) {
463
    return generateString(value);
464
  }
465

466
  final String unicodeEscapes = value.runes.map((int code) => '\\u{${code.toRadixString(16)}}').join();
467 468
  return "'$unicodeEscapes'";
}