localizations_utils.dart 20.4 KB
Newer Older
1 2 3 4 5
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'package:meta/meta.dart';
6
import 'package:yaml/yaml.dart';
7

8
import '../base/common.dart';
9
import '../base/file_system.dart';
10
import '../base/logger.dart';
11
import '../runner/flutter_command.dart';
12
import 'gen_l10n_types.dart';
13 14 15 16 17 18 19 20 21 22 23 24 25
import 'language_subtag_registry.dart';

typedef HeaderGenerator = String Function(String regenerateInstructions);
typedef ConstructorGenerator = String Function(LocaleInfo locale);

int sortFilesByPath (File a, File b) {
  return a.path.compareTo(b.path);
}

/// Simple data class to hold parsed locale. Does not promise validity of any data.
@immutable
class LocaleInfo implements Comparable<LocaleInfo> {
  const LocaleInfo({
26 27 28 29 30
    required this.languageCode,
    required this.scriptCode,
    required this.countryCode,
    required this.length,
    required this.originalString,
31 32 33 34 35 36 37 38 39 40 41 42 43 44
  });

  /// Simple parser. Expects the locale string to be in the form of 'language_script_COUNTRY'
  /// where the language is 2 characters, script is 4 characters with the first uppercase,
  /// and country is 2-3 characters and all uppercase.
  ///
  /// 'language_COUNTRY' or 'language_script' are also valid. Missing fields will be null.
  ///
  /// When `deriveScriptCode` is true, if [scriptCode] was unspecified, it will
  /// be derived from the [languageCode] and [countryCode] if possible.
  factory LocaleInfo.fromString(String locale, { bool deriveScriptCode = false }) {
    final List<String> codes = locale.split('_'); // [language, script, country]
    assert(codes.isNotEmpty && codes.length < 4);
    final String languageCode = codes[0];
45 46
    String? scriptCode;
    String? countryCode;
47 48 49 50 51 52 53 54 55
    int length = codes.length;
    String originalString = locale;
    if (codes.length == 2) {
      scriptCode = codes[1].length >= 4 ? codes[1] : null;
      countryCode = codes[1].length < 4 ? codes[1] : null;
    } else if (codes.length == 3) {
      scriptCode = codes[1].length > codes[2].length ? codes[1] : codes[2];
      countryCode = codes[1].length < codes[2].length ? codes[1] : codes[2];
    }
56
    assert(codes[0].isNotEmpty);
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
    assert(countryCode == null || countryCode.isNotEmpty);
    assert(scriptCode == null || scriptCode.isNotEmpty);

    /// Adds scriptCodes to locales where we are able to assume it to provide
    /// finer granularity when resolving locales.
    ///
    /// The basis of the assumptions here are based off of known usage of scripts
    /// across various countries. For example, we know Taiwan uses traditional (Hant)
    /// script, so it is safe to apply (Hant) to Taiwanese languages.
    if (deriveScriptCode && scriptCode == null) {
      switch (languageCode) {
        case 'zh': {
          if (countryCode == null) {
            scriptCode = 'Hans';
          }
          switch (countryCode) {
            case 'CN':
            case 'SG':
              scriptCode = 'Hans';
            case 'TW':
            case 'HK':
            case 'MO':
              scriptCode = 'Hant';
          }
          break;
        }
        case 'sr': {
          if (countryCode == null) {
            scriptCode = 'Cyrl';
          }
          break;
        }
      }
      // Increment length if we were able to assume a scriptCode.
      if (scriptCode != null) {
        length += 1;
      }
      // Update the base string to reflect assumed scriptCodes.
      originalString = languageCode;
      if (scriptCode != null) {
97
        originalString += '_$scriptCode';
98 99
      }
      if (countryCode != null) {
100
        originalString += '_$countryCode';
101 102 103 104 105 106 107 108 109 110 111 112 113
      }
    }

    return LocaleInfo(
      languageCode: languageCode,
      scriptCode: scriptCode,
      countryCode: countryCode,
      length: length,
      originalString: originalString,
    );
  }

  final String languageCode;
114 115
  final String? scriptCode;
  final String? countryCode;
116 117 118 119 120 121 122
  final int length;             // The number of fields. Ranges from 1-3.
  final String originalString;  // Original un-parsed locale string.

  String camelCase() {
    return originalString
      .split('_')
      .map<String>((String part) => part.substring(0, 1).toUpperCase() + part.substring(1).toLowerCase())
123
      .join();
124 125 126 127 128 129 130 131 132
  }

  @override
  bool operator ==(Object other) {
    return other is LocaleInfo
        && other.originalString == originalString;
  }

  @override
133
  int get hashCode => originalString.hashCode;
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148

  @override
  String toString() {
    return originalString;
  }

  @override
  int compareTo(LocaleInfo other) {
    return originalString.compareTo(other.originalString);
  }
}

// See also //master/tools/gen_locale.dart in the engine repo.
Map<String, List<String>> _parseSection(String section) {
  final Map<String, List<String>> result = <String, List<String>>{};
149
  late List<String> lastHeading;
150 151 152 153 154 155 156 157 158 159
  for (final String line in section.split('\n')) {
    if (line == '') {
      continue;
    }
    if (line.startsWith('  ')) {
      lastHeading[lastHeading.length - 1] = '${lastHeading.last}${line.substring(1)}';
      continue;
    }
    final int colon = line.indexOf(':');
    if (colon <= 0) {
160
      throw Exception('not sure how to deal with "$line"');
161 162 163 164
    }
    final String name = line.substring(0, colon);
    final String value = line.substring(colon + 2);
    lastHeading = result.putIfAbsent(name, () => <String>[]);
165
    result[name]!.add(value);
166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183
  }
  return result;
}

final Map<String, String> _languages = <String, String>{};
final Map<String, String> _regions = <String, String>{};
final Map<String, String> _scripts = <String, String>{};
const String kProvincePrefix = ', Province of ';
const String kParentheticalPrefix = ' (';

/// Prepares the data for the [describeLocale] method below.
///
/// The data is obtained from the official IANA registry.
void precacheLanguageAndRegionTags() {
  final List<Map<String, List<String>>> sections =
      languageSubtagRegistry.split('%%').skip(1).map<Map<String, List<String>>>(_parseSection).toList();
  for (final Map<String, List<String>> section in sections) {
    assert(section.containsKey('Type'), section.toString());
184
    final String type = section['Type']!.single;
185 186
    if (type == 'language' || type == 'region' || type == 'script') {
      assert(section.containsKey('Subtag') && section.containsKey('Description'), section.toString());
187 188
      final String subtag = section['Subtag']!.single;
      String description = section['Description']!.join(' ');
189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215
      if (description.startsWith('United ')) {
        description = 'the $description';
      }
      if (description.contains(kParentheticalPrefix)) {
        description = description.substring(0, description.indexOf(kParentheticalPrefix));
      }
      if (description.contains(kProvincePrefix)) {
        description = description.substring(0, description.indexOf(kProvincePrefix));
      }
      if (description.endsWith(' Republic')) {
        description = 'the $description';
      }
      switch (type) {
        case 'language':
          _languages[subtag] = description;
        case 'region':
          _regions[subtag] = description;
        case 'script':
          _scripts[subtag] = description;
      }
    }
  }
}

String describeLocale(String tag) {
  final List<String> subtags = tag.split('_');
  assert(subtags.isNotEmpty);
216 217 218 219 220 221 222 223 224
  final String languageCode = subtags[0];
  if (!_languages.containsKey(languageCode)) {
    throw L10nException(
      '"$languageCode" is not a supported language code.\n'
      'See https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry '
      'for the supported list.',
    );
  }
  final String language = _languages[languageCode]!;
225
  String output = language;
226 227
  String? region;
  String? script;
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
  if (subtags.length == 2) {
    region = _regions[subtags[1]];
    script = _scripts[subtags[1]];
    assert(region != null || script != null);
  } else if (subtags.length >= 3) {
    region = _regions[subtags[2]];
    script = _scripts[subtags[1]];
    assert(region != null && script != null);
  }
  if (region != null) {
    output += ', as used in $region';
  }
  if (script != null) {
    output += ', using the $script script';
  }
  return output;
}

246
/// Return the input string as a Dart-parsable string.
247 248 249 250 251 252 253 254 255 256
///
/// ```
/// foo => 'foo'
/// foo "bar" => 'foo "bar"'
/// foo 'bar' => "foo 'bar'"
/// foo 'bar' "baz" => '''foo 'bar' "baz"'''
/// ```
///
/// This function is used by tools that take in a JSON-formatted file to
/// generate Dart code. For this reason, characters with special meaning
257 258 259
/// in JSON files are escaped. For example, the backspace character (\b)
/// has to be properly escaped by this function so that the generated
/// Dart code correctly represents this character:
260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
/// ```
/// foo\bar => 'foo\\bar'
/// foo\nbar => 'foo\\nbar'
/// foo\\nbar => 'foo\\\\nbar'
/// foo\\bar => 'foo\\\\bar'
/// foo\ bar => 'foo\\ bar'
/// foo$bar = 'foo\$bar'
/// ```
String generateString(String value) {
  const String backslash = '__BACKSLASH__';
  assert(
    !value.contains(backslash),
    'Input string cannot contain the sequence: '
    '"__BACKSLASH__", as it is used as part of '
    'backslash character processing.'
  );

  value = value
    // Replace backslashes with a placeholder for now to properly parse
    // other special characters.
280 281 282 283 284 285 286 287 288
    .replaceAll(r'\', backslash)
    .replaceAll(r'$', r'\$')
    .replaceAll("'", r"\'")
    .replaceAll('"', r'\"')
    .replaceAll('\n', r'\n')
    .replaceAll('\f', r'\f')
    .replaceAll('\t', r'\t')
    .replaceAll('\r', r'\r')
    .replaceAll('\b', r'\b')
289
    // Reintroduce escaped backslashes into generated Dart string.
290
    .replaceAll(backslash, r'\\');
291

292 293 294
  return value;
}

295 296 297 298
/// Given a list of normal strings or interpolated variables, concatenate them
/// into a single dart string to be returned. An example of a normal string
/// would be "'Hello world!'" and an example of a interpolated variable would be
/// "'$placeholder'".
299
///
300 301 302 303 304 305 306 307 308 309 310 311 312 313
/// Each of the strings in [expressions] should be a raw string, which, if it
/// were to be added to a dart file, would be a properly formatted dart string
/// with escapes and/or interpolation. The purpose of this function is to
/// concatenate these dart strings into a single dart string which can be
/// returned in the generated localization files.
///
/// The following rules describe the kinds of string expressions that can be
/// handled:
/// 1. If [expressions] is empty, return the empty string "''".
/// 2. If [expressions] has only one [String] which is an interpolated variable,
///    it is converted to the variable itself e.g. ["'$expr'"] -> "expr".
/// 3. If one string in [expressions] is an interpolation and the next begins
///    with an alphanumeric character, then the former interpolation should be
///    wrapped in braces e.g. ["'$expr1'", "'another'"] -> "'${expr1}another'".
314 315
String generateReturnExpr(List<String> expressions, { bool isSingleStringVar = false }) {
  if (expressions.isEmpty) {
316
    return "''";
317 318 319
  } else if (isSingleStringVar) {
    // If our expression is "$varName" where varName is a String, this is equivalent to just varName.
    return expressions[0].substring(1);
320
  } else {
321 322
    final String string = expressions.reversed.fold<String>('', (String string, String expression) {
      if (expression[0] != r'$') {
323
        return expression + string;
324 325
      }
      final RegExp alphanumeric = RegExp(r'^([0-9a-zA-Z]|_)+$');
326 327
      if (alphanumeric.hasMatch(expression.substring(1)) && !(string.isNotEmpty && alphanumeric.hasMatch(string[0]))) {
        return '$expression$string';
328
      } else {
329
        return '\${${expression.substring(1)}}$string';
330 331 332 333
      }
    });
    return "'$string'";
  }
334
}
335 336 337

/// Typed configuration from the localizations config file.
class LocalizationOptions {
338 339 340 341 342
  LocalizationOptions({
    required this.arbDir,
    this.outputDir,
    String? templateArbFile,
    String? outputLocalizationFile,
343
    this.untranslatedMessagesFile,
344
    String? outputClass,
345
    this.preferredSupportedLocales,
346
    this.header,
347
    this.headerFile,
348 349 350 351 352 353 354 355 356
    bool? useDeferredLoading,
    this.genInputsAndOutputsList,
    bool? syntheticPackage,
    this.projectDir,
    bool? requiredResourceAttributes,
    bool? nullableGetter,
    bool? format,
    bool? useEscaping,
    bool? suppressWarnings,
357
    bool? relaxSyntax,
358 359 360 361 362 363 364 365 366
  }) : templateArbFile = templateArbFile ?? 'app_en.arb',
       outputLocalizationFile = outputLocalizationFile ?? 'app_localizations.dart',
       outputClass = outputClass ?? 'AppLocalizations',
       useDeferredLoading = useDeferredLoading ?? false,
       syntheticPackage = syntheticPackage ?? true,
       requiredResourceAttributes = requiredResourceAttributes ?? false,
       nullableGetter = nullableGetter ?? true,
       format = format ?? false,
       useEscaping = useEscaping ?? false,
367 368
       suppressWarnings = suppressWarnings ?? false,
       relaxSyntax = relaxSyntax ?? false;
369 370 371 372

  /// The `--arb-dir` argument.
  ///
  /// The directory where all input localization files should reside.
373 374 375 376 377 378 379
  final String arbDir;

  /// The `--output-dir` argument.
  ///
  /// The directory where all output localization files should be generated.
  final String? outputDir;

380 381 382

  /// The `--template-arb-file` argument.
  ///
383 384
  /// This path is relative to [arbDirectory].
  final String templateArbFile;
385 386 387

  /// The `--output-localization-file` argument.
  ///
388 389
  /// This path is relative to [arbDir].
  final String outputLocalizationFile;
390 391 392

  /// The `--untranslated-messages-file` argument.
  ///
393 394
  /// This path is relative to [arbDir].
  final String? untranslatedMessagesFile;
395 396

  /// The `--output-class` argument.
397
  final String outputClass;
398 399

  /// The `--preferred-supported-locales` argument.
400
  final List<String>? preferredSupportedLocales;
401

402 403 404 405 406
  /// The `--header` argument.
  ///
  /// The header to prepend to the generated Dart localizations.
  final String? header;

407 408 409 410
  /// The `--header-file` argument.
  ///
  /// A file containing the header to prepend to the generated
  /// Dart localizations.
411
  final String? headerFile;
412 413 414 415 416

  /// The `--use-deferred-loading` argument.
  ///
  /// Whether to generate the Dart localization file with locales imported
  /// as deferred.
417 418 419 420 421 422
  final bool useDeferredLoading;

  /// The `--gen-inputs-and-outputs-list` argument.
  ///
  /// This path is relative to [arbDir].
  final String? genInputsAndOutputsList;
423 424 425 426 427

  /// The `--synthetic-package` argument.
  ///
  /// Whether to generate the Dart localization files in a synthetic package
  /// or in a custom directory.
428 429 430 431 432 433
  final bool syntheticPackage;

  /// The `--project-dir` argument.
  ///
  /// This path is relative to [arbDir].
  final String? projectDir;
434 435 436 437 438

  /// The `required-resource-attributes` argument.
  ///
  /// Whether to require all resource ids to contain a corresponding
  /// resource attribute.
439
  final bool requiredResourceAttributes;
440 441 442 443

  /// The `nullable-getter` argument.
  ///
  /// Whether or not the localizations class getter is nullable.
444
  final bool nullableGetter;
445 446 447 448 449

  /// The `format` argument.
  ///
  /// Whether or not to format the generated files.
  final bool format;
450 451 452 453 454

  /// The `use-escaping` argument.
  ///
  /// Whether or not the ICU escaping syntax is used.
  final bool useEscaping;
455 456 457 458 459

  /// The `suppress-warnings` argument.
  ///
  /// Whether or not to suppress warnings.
  final bool suppressWarnings;
460 461 462 463 464 465 466 467 468 469

  /// The `relax-syntax` argument.
  ///
  /// Whether or not to relax the syntax. When specified, the syntax will be
  /// relaxed so that the special character "{" is treated as a string if it is
  /// not followed by a valid placeholder and "}" is treated as a string if it
  /// does not close any previous "{" that is treated as a special character.
  /// This was added in for backward compatibility and is not recommended
  /// as it may mask errors.
  final bool relaxSyntax;
470 471 472 473 474 475 476
}

/// Parse the localizations configuration options from [file].
///
/// Throws [Exception] if any of the contents are invalid. Returns a
/// [LocalizationOptions] with all fields as `null` if the config file exists
/// but is empty.
477
LocalizationOptions parseLocalizationsOptionsFromYAML({
478 479
  required File file,
  required Logger logger,
480
  required String defaultArbDir,
481 482 483
}) {
  final String contents = file.readAsStringSync();
  if (contents.trim().isEmpty) {
484
    return LocalizationOptions(arbDir: defaultArbDir);
485
  }
486 487 488 489 490 491
  final YamlNode yamlNode;
  try {
    yamlNode = loadYamlNode(file.readAsStringSync());
  } on YamlException catch (err) {
    throwToolExit(err.message);
  }
492 493 494 495 496
  if (yamlNode is! YamlMap) {
    logger.printError('Expected ${file.path} to contain a map, instead was $yamlNode');
    throw Exception();
  }
  return LocalizationOptions(
497 498 499 500 501
    arbDir: _tryReadUri(yamlNode, 'arb-dir', logger)?.path ?? defaultArbDir,
    outputDir: _tryReadUri(yamlNode, 'output-dir', logger)?.path,
    templateArbFile: _tryReadUri(yamlNode, 'template-arb-file', logger)?.path,
    outputLocalizationFile: _tryReadUri(yamlNode, 'output-localization-file', logger)?.path,
    untranslatedMessagesFile: _tryReadUri(yamlNode, 'untranslated-messages-file', logger)?.path,
502
    outputClass: _tryReadString(yamlNode, 'output-class', logger),
503 504 505
    header: _tryReadString(yamlNode, 'header', logger),
    headerFile: _tryReadUri(yamlNode, 'header-file', logger)?.path,
    useDeferredLoading: _tryReadBool(yamlNode, 'use-deferred-loading', logger),
506
    preferredSupportedLocales: _tryReadStringList(yamlNode, 'preferred-supported-locales', logger),
507 508 509 510 511 512
    syntheticPackage: _tryReadBool(yamlNode, 'synthetic-package', logger),
    requiredResourceAttributes: _tryReadBool(yamlNode, 'required-resource-attributes', logger),
    nullableGetter: _tryReadBool(yamlNode, 'nullable-getter', logger),
    format: _tryReadBool(yamlNode, 'format', logger),
    useEscaping: _tryReadBool(yamlNode, 'use-escaping', logger),
    suppressWarnings: _tryReadBool(yamlNode, 'suppress-warnings', logger),
513
    relaxSyntax: _tryReadBool(yamlNode, 'relax-syntax', logger),
514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539
  );
}

/// Parse the localizations configuration from [FlutterCommand].
LocalizationOptions parseLocalizationsOptionsFromCommand({
  required FlutterCommand command,
  required String defaultArbDir,
}) {
  return LocalizationOptions(
    arbDir: command.stringArg('arb-dir') ?? defaultArbDir,
    outputDir: command.stringArg('output-dir'),
    outputLocalizationFile: command.stringArg('output-localization-file'),
    templateArbFile: command.stringArg('template-arb-file'),
    untranslatedMessagesFile: command.stringArg('untranslated-messages-file'),
    outputClass: command.stringArg('output-class'),
    header: command.stringArg('header'),
    headerFile: command.stringArg('header-file'),
    useDeferredLoading: command.boolArg('use-deferred-loading'),
    genInputsAndOutputsList: command.stringArg('gen-inputs-and-outputs-list'),
    syntheticPackage: command.boolArg('synthetic-package'),
    projectDir: command.stringArg('project-dir'),
    requiredResourceAttributes: command.boolArg('required-resource-attributes'),
    nullableGetter: command.boolArg('nullable-getter'),
    format: command.boolArg('format'),
    useEscaping: command.boolArg('use-escaping'),
    suppressWarnings: command.boolArg('suppress-warnings'),
540 541 542 543
  );
}

// Try to read a `bool` value or null from `yamlMap`, otherwise throw.
544 545
bool? _tryReadBool(YamlMap yamlMap, String key, Logger logger) {
  final Object? value = yamlMap[key];
546 547 548 549 550 551 552
  if (value == null) {
    return null;
  }
  if (value is! bool) {
    logger.printError('Expected "$key" to have a bool value, instead was "$value"');
    throw Exception();
  }
553
  return value;
554 555 556
}

// Try to read a `String` value or null from `yamlMap`, otherwise throw.
557 558
String? _tryReadString(YamlMap yamlMap, String key, Logger logger) {
  final Object? value = yamlMap[key];
559 560 561 562 563 564 565
  if (value == null) {
    return null;
  }
  if (value is! String) {
    logger.printError('Expected "$key" to have a String value, instead was "$value"');
    throw Exception();
  }
566
  return value;
567 568
}

569 570
List<String>? _tryReadStringList(YamlMap yamlMap, String key, Logger logger) {
  final Object? value = yamlMap[key];
571 572 573 574 575 576 577 578 579 580 581 582 583 584
  if (value == null) {
    return null;
  }
  if (value is String) {
    return <String>[value];
  }
  if (value is Iterable) {
    return value.map((dynamic e) => e.toString()).toList();
  }
  logger.printError('"$value" must be String or List.');
  throw Exception();
}

// Try to read a valid `Uri` or null from `yamlMap`, otherwise throw.
585 586
Uri? _tryReadUri(YamlMap yamlMap, String key, Logger logger) {
  final String? value = _tryReadString(yamlMap, key, logger);
587 588 589
  if (value == null) {
    return null;
  }
590
  final Uri? uri = Uri.tryParse(value);
591 592 593 594 595
  if (uri == null) {
    logger.printError('"$value" must be a relative file URI');
  }
  return uri;
}