dartdoc.dart 17.5 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6 7 8
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io';

9
import 'package:args/args.dart';
10
import 'package:intl/intl.dart';
11
import 'package:path/path.dart' as path;
12
import 'package:process/process.dart';
13

14 15
const String kDocsRoot = 'dev/docs';
const String kPublishRoot = '$kDocsRoot/doc';
16
const String kSnippetsRoot = 'dev/snippets';
Seth Ladd's avatar
Seth Ladd committed
17

18 19 20
/// This script expects to run with the cwd as the root of the flutter repo. It
/// will generate documentation for the packages in `//packages/` and write the
/// documentation to `//dev/docs/doc/api/`.
21
///
Seth Ladd's avatar
Seth Ladd committed
22 23 24 25
/// This script also updates the index.html file so that it can be placed
/// at the root of docs.flutter.io. We are keeping the files inside of
/// docs.flutter.io/flutter for now, so we need to manipulate paths
/// a bit. See https://github.com/flutter/flutter/issues/3900 for more info.
26 27 28 29 30
///
/// This will only work on UNIX systems, not Windows. It requires that 'git' be
/// in your path. It requires that 'flutter' has been run previously. It uses
/// the version of Dart downloaded by the 'flutter' tool in this repository and
/// will crash if that is absent.
31
Future<void> main(List<String> arguments) async {
32 33
  final ArgParser argParser = _createArgsParser();
  final ArgResults args = argParser.parse(arguments);
34
  if (args['help'] as bool) {
35 36 37 38
    print ('Usage:');
    print (argParser.usage);
    exit(0);
  }
39 40 41 42
  // If we're run from the `tools` dir, set the cwd to the repo root.
  if (path.basename(Directory.current.path) == 'tools')
    Directory.current = Directory.current.parent.parent;

43
  final ProcessResult flutter = Process.runSync('flutter', <String>[]);
44
  final File versionFile = File('version');
45
  if (flutter.exitCode != 0 || !versionFile.existsSync())
46
    throw Exception('Failed to determine Flutter version.');
47
  final String version = versionFile.readAsStringSync();
48

49
  // Create the pubspec.yaml file.
50
  final StringBuffer buf = StringBuffer();
51
  buf.writeln('name: Flutter');
52
  buf.writeln('homepage: https://flutter.dev');
53 54 55 56 57 58
  // TODO(dnfield): We should make DartDoc able to avoid emitting this. If we
  // use the real value here, every file will get marked as new instead of only
  // files that have otherwise changed. Instead, we replace it dynamically using
  // JavaScript so that fewer files get marked as changed.
  // https://github.com/dart-lang/dartdoc/issues/1982
  buf.writeln('version: 0.0.0');
59
  buf.writeln('dependencies:');
60
  for (final String package in findPackageNames()) {
61
    buf.writeln('  $package:');
62
    buf.writeln('    sdk: flutter');
63
  }
64 65 66 67
  buf.writeln('  platform_integration: 0.0.1');
  buf.writeln('dependency_overrides:');
  buf.writeln('  platform_integration:');
  buf.writeln('    path: platform_integration');
68
  File('$kDocsRoot/pubspec.yaml').writeAsStringSync(buf.toString());
69 70

  // Create the library file.
71
  final Directory libDir = Directory('$kDocsRoot/lib');
72 73
  libDir.createSync();

74
  final StringBuffer contents = StringBuffer('library temp_doc;\n\n');
75
  for (final String libraryRef in libraryRefs()) {
76
    contents.writeln("import 'package:$libraryRef';");
77
  }
78
  File('$kDocsRoot/lib/temp_doc.dart').writeAsStringSync(contents.toString());
79

80 81 82 83 84 85 86
  final String flutterRoot = Directory.current.path;
  final Map<String, String> pubEnvironment = <String, String>{
    'FLUTTER_ROOT': flutterRoot,
  };

  // If there's a .pub-cache dir in the flutter root, use that.
  final String pubCachePath = '$flutterRoot/.pub-cache';
87
  if (Directory(pubCachePath).existsSync()) {
88 89 90 91 92
    pubEnvironment['PUB_CACHE'] = pubCachePath;
  }

  final String pubExecutable = '$flutterRoot/bin/cache/dart-sdk/bin/pub';

93
  // Run pub.
94
  ProcessWrapper process = ProcessWrapper(await Process.start(
95
    pubExecutable,
96
    <String>['get'],
97
    workingDirectory: kDocsRoot,
98
    environment: pubEnvironment,
99
  ));
100 101
  printStream(process.stdout, prefix: 'pub:stdout: ');
  printStream(process.stderr, prefix: 'pub:stderr: ');
102
  final int code = await process.done;
103 104 105
  if (code != 0)
    exit(code);

106
  createFooter('$kDocsRoot/lib/', version);
107
  copyAssets();
108
  createSearchMetadata('$kDocsRoot/lib/opensearch.xml', '$kDocsRoot/doc/opensearch.xml');
109
  cleanOutSnippets();
110

111 112 113
  final List<String> dartdocBaseArgs = <String>[
    'global',
    'run',
114
    if (args['checked'] as bool) '-c',
115 116
    'dartdoc',
  ];
117

118
  // Verify which version of dartdoc we're using.
119
  final ProcessResult result = Process.runSync(
120
    pubExecutable,
121
    <String>[...dartdocBaseArgs, '--version'],
122
    workingDirectory: kDocsRoot,
123
    environment: pubEnvironment,
124
  );
125
  print('\n${result.stdout}flutter version: $version\n');
126

127
  // Generate the documentation.
128 129
  // We don't need to exclude flutter_tools in this list because it's not in the
  // recursive dependencies of the package defined at dev/docs/pubspec.yaml
130 131
  final List<String> dartdocArgs = <String>[
    ...dartdocBaseArgs,
132
    '--allow-tools',
133 134
    if (args['json'] as bool) '--json',
    if (args['validate-links'] as bool) '--validate-links' else '--no-validate-links',
135 136 137
    '--link-to-source-excludes', '../../bin/cache',
    '--link-to-source-root', '../..',
    '--link-to-source-uri-template', 'https://github.com/flutter/flutter/blob/master/%f%#L%l%',
138
    '--inject-html',
139 140
    '--header', 'styles.html',
    '--header', 'analytics.html',
141
    '--header', 'survey.html',
142
    '--header', 'snippets.html',
143
    '--header', 'opensearch.html',
144
    '--footer-text', 'lib/footer.html',
145 146
    '--allow-warnings-in-packages',
    <String>[
147 148 149 150 151 152 153
      'Flutter',
      'flutter',
      'platform_integration',
      'flutter_test',
      'flutter_driver',
      'flutter_localizations',
    ].join(','),
154
    '--exclude-packages',
155 156 157 158 159 160 161
    <String>[
      'analyzer',
      'args',
      'barback',
      'cli_util',
      'csslib',
      'flutter_goldens',
162
      'flutter_goldens_client',
163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184
      'front_end',
      'fuchsia_remote_debug_protocol',
      'glob',
      'html',
      'http_multi_server',
      'io',
      'isolate',
      'js',
      'kernel',
      'logging',
      'mime',
      'mockito',
      'node_preamble',
      'plugin',
      'shelf',
      'shelf_packages_handler',
      'shelf_static',
      'shelf_web_socket',
      'utf',
      'watcher',
      'yaml',
    ].join(','),
185
    '--exclude',
186 187 188 189 190 191 192 193 194 195
    <String>[
      'package:Flutter/temp_doc.dart',
      'package:http/browser_client.dart',
      'package:intl/intl_browser.dart',
      'package:matcher/mirror_matchers.dart',
      'package:quiver/io.dart',
      'package:quiver/mirrors.dart',
      'package:vm_service_client/vm_service_client.dart',
      'package:web_socket_channel/html.dart',
    ].join(','),
196
    '--favicon=favicon.ico',
197
    '--package-order', 'flutter,Dart,platform_integration,flutter_test,flutter_driver',
198
    '--auto-include-dependencies',
199
  ];
200

201
  String quote(String arg) => arg.contains(' ') ? "'$arg'" : arg;
202
  print('Executing: (cd $kDocsRoot ; $pubExecutable ${dartdocArgs.map<String>(quote).join(' ')})');
203

204
  process = ProcessWrapper(await Process.start(
205
    pubExecutable,
206
    dartdocArgs,
207
    workingDirectory: kDocsRoot,
208
    environment: pubEnvironment,
209
  ));
210 211
  printStream(process.stdout, prefix: args['json'] as bool ? '' : 'dartdoc:stdout: ',
    filter: args['verbose'] as bool ? const <Pattern>[] : <Pattern>[
212 213
      RegExp(r'^generating docs for library '), // unnecessary verbosity
      RegExp(r'^pars'), // unnecessary verbosity
214 215
    ],
  );
216 217
  printStream(process.stderr, prefix: args['json'] as bool ? '' : 'dartdoc:stderr: ',
    filter: args['verbose'] as bool ? const <Pattern>[] : <Pattern>[
218
      RegExp(r'^ warning: .+: \(.+/\.pub-cache/hosted/pub.dartlang.org/.+\)'), // packages outside our control
219 220
    ],
  );
221
  final int exitCode = await process.done;
Seth Ladd's avatar
Seth Ladd committed
222 223 224 225

  if (exitCode != 0)
    exit(exitCode);

226 227
  sanityCheckDocs();

Seth Ladd's avatar
Seth Ladd committed
228 229 230
  createIndexAndCleanup();
}

231
ArgParser _createArgsParser() {
232
  final ArgParser parser = ArgParser();
233 234 235 236 237 238 239 240 241 242
  parser.addFlag('help', abbr: 'h', negatable: false,
      help: 'Show command help.');
  parser.addFlag('verbose', negatable: true, defaultsTo: true,
      help: 'Whether to report all error messages (on) or attempt to '
          'filter out some known false positives (off).  Shut this off '
          'locally if you want to address Flutter-specific issues.');
  parser.addFlag('checked', abbr: 'c', negatable: true,
      help: 'Run dartdoc in checked mode.');
  parser.addFlag('json', negatable: true,
      help: 'Display json-formatted output from dartdoc and skip stdout/stderr prefixing.');
243 244
  parser.addFlag('validate-links', negatable: true,
      help: 'Display warnings for broken links generated by dartdoc (slow)');
245 246 247
  return parser;
}

248
final RegExp gitBranchRegexp = RegExp(r'^## (.*)');
249

250 251 252 253 254
String getBranchName() {
  final ProcessResult gitResult = Process.runSync('git', <String>['status', '-b', '--porcelain']);
  if (gitResult.exitCode != 0)
    throw 'git status exit with non-zero exit code: ${gitResult.exitCode}';
  final Match gitBranchMatch = gitBranchRegexp.firstMatch(
255
      (gitResult.stdout as String).trim().split('\n').first);
256 257 258
  return gitBranchMatch == null ? '' : gitBranchMatch.group(1).split('...').first;
}

259
String gitRevision() {
260 261
  const int kGitRevisionLength = 10;

262
  final ProcessResult gitResult = Process.runSync('git', <String>['rev-parse', 'HEAD']);
263
  if (gitResult.exitCode != 0)
264
    throw 'git rev-parse exit with non-zero exit code: ${gitResult.exitCode}';
265
  final String gitRevision = (gitResult.stdout as String).trim();
266

267 268
  return gitRevision.length > kGitRevisionLength ? gitRevision.substring(0, kGitRevisionLength) : gitRevision;
}
269

270
void createFooter(String footerPath, String version) {
271
  final String timestamp = DateFormat('yyyy-MM-dd HH:mm').format(DateTime.now());
272
  final String gitBranch = getBranchName();
273 274 275 276
  final String gitBranchOut = gitBranch.isEmpty ? '' : '• $gitBranch';
  File('${footerPath}footer.html').writeAsStringSync('<script src="footer.js"></script>');
  File('$kPublishRoot/api/footer.js')
    ..createSync(recursive: true)
277 278
    ..writeAsStringSync('''
(function() {
Dan Field's avatar
Dan Field committed
279 280 281 282 283 284 285 286 287
  var span = document.querySelector('footer>span');
  if (span) {
    span.innerText = 'Flutter $version  $timestamp  ${gitRevision()} $gitBranchOut';
  }
  var sourceLink = document.querySelector('a.source-link');
  if (sourceLink) {
    sourceLink.href = sourceLink.href.replace('/master/', '/${gitRevision()}/');
  }
})();
288
''');
289 290
}

291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
/// Generates an OpenSearch XML description that can be used to add a custom
/// search for Flutter API docs to the browser. Unfortunately, it has to know
/// the URL to which site to search, so we customize it here based upon the
/// branch name.
void createSearchMetadata(String templatePath, String metadataPath) {
  final String template = File(templatePath).readAsStringSync();
  final String branch = getBranchName();
  final String metadata = template.replaceAll(
    '{SITE_URL}',
    branch == 'stable' ? 'https://docs.flutter.io/' : 'https://master-docs.flutter.io/',
  );
  Directory(path.dirname(metadataPath)).create(recursive: true);
  File(metadataPath).writeAsStringSync(metadata);
}

306 307 308 309 310 311 312 313 314 315 316
/// Recursively copies `srcDir` to `destDir`, invoking [onFileCopied], if
/// specified, for each source/destination file pair.
///
/// Creates `destDir` if needed.
void copyDirectorySync(Directory srcDir, Directory destDir, [void onFileCopied(File srcFile, File destFile)]) {
  if (!srcDir.existsSync())
    throw Exception('Source directory "${srcDir.path}" does not exist, nothing to copy');

  if (!destDir.existsSync())
    destDir.createSync(recursive: true);

317
  for (final FileSystemEntity entity in srcDir.listSync()) {
318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
    final String newPath = path.join(destDir.path, path.basename(entity.path));
    if (entity is File) {
      final File newFile = File(newPath);
      entity.copySync(newPath);
      onFileCopied?.call(entity, newFile);
    } else if (entity is Directory) {
      copyDirectorySync(entity, Directory(newPath));
    } else {
      throw Exception('${entity.path} is neither File nor Directory');
    }
  }
}

void copyAssets() {
  final Directory assetsDir = Directory(path.join(kPublishRoot, 'assets'));
  if (assetsDir.existsSync()) {
    assetsDir.deleteSync(recursive: true);
  }
  copyDirectorySync(
      Directory(path.join(kDocsRoot, 'assets')),
      Directory(path.join(kPublishRoot, 'assets')),
          (File src, File dest) => print('Copied ${src.path} to ${dest.path}'));
}

342 343
/// Clean out any existing snippets so that we don't publish old files from
/// previous runs accidentally.
344 345 346 347 348 349 350 351 352
void cleanOutSnippets() {
  final Directory snippetsDir = Directory(path.join(kPublishRoot, 'snippets'));
  if (snippetsDir.existsSync()) {
    snippetsDir
      ..deleteSync(recursive: true)
      ..createSync(recursive: true);
  }
}

353
void sanityCheckDocs() {
354
  final List<String> canaries = <String>[
355 356 357 358 359 360 361 362 363
    '$kPublishRoot/assets/overrides.css',
    '$kPublishRoot/api/dart-io/File-class.html',
    '$kPublishRoot/api/dart-ui/Canvas-class.html',
    '$kPublishRoot/api/dart-ui/Canvas/drawRect.html',
    '$kPublishRoot/api/flutter_driver/FlutterDriver/FlutterDriver.connectedTo.html',
    '$kPublishRoot/api/flutter_test/WidgetTester/pumpWidget.html',
    '$kPublishRoot/api/material/Material-class.html',
    '$kPublishRoot/api/material/Tooltip-class.html',
    '$kPublishRoot/api/widgets/Widget-class.html',
364
  ];
365
  for (final String canary in canaries) {
366 367
    if (!File(canary).existsSync())
      throw Exception('Missing "$canary", which probably means the documentation failed to build correctly.');
368 369 370
  }
}

Seth Ladd's avatar
Seth Ladd committed
371 372 373
/// Creates a custom index.html because we try to maintain old
/// paths. Cleanup unused index.html files no longer needed.
void createIndexAndCleanup() {
374
  print('\nCreating a custom index.html in $kPublishRoot/index.html');
375
  removeOldFlutterDocsDir();
Seth Ladd's avatar
Seth Ladd committed
376 377 378
  renameApiDir();
  copyIndexToRootOfDocs();
  addHtmlBaseToIndex();
379
  changePackageToSdkInTitlebar();
Seth Ladd's avatar
Seth Ladd committed
380
  putRedirectInOldIndexLocation();
381
  writeSnippetsIndexFile();
Seth Ladd's avatar
Seth Ladd committed
382 383 384
  print('\nDocs ready to go!');
}

385 386
void removeOldFlutterDocsDir() {
  try {
387
    Directory('$kPublishRoot/flutter').deleteSync(recursive: true);
388
  } on FileSystemException {
389 390 391 392
    // If the directory does not exist, that's OK.
  }
}

Seth Ladd's avatar
Seth Ladd committed
393
void renameApiDir() {
394
  Directory('$kPublishRoot/api').renameSync('$kPublishRoot/flutter');
Seth Ladd's avatar
Seth Ladd committed
395 396
}

397
void copyIndexToRootOfDocs() {
398
  File('$kPublishRoot/flutter/index.html').copySync('$kPublishRoot/index.html');
Seth Ladd's avatar
Seth Ladd committed
399 400
}

401
void changePackageToSdkInTitlebar() {
402
  final File indexFile = File('$kPublishRoot/index.html');
403 404
  String indexContents = indexFile.readAsStringSync();
  indexContents = indexContents.replaceFirst(
405 406
    '<li><a href="https://flutter.dev">Flutter package</a></li>',
    '<li><a href="https://flutter.dev">Flutter SDK</a></li>',
407 408 409 410 411
  );

  indexFile.writeAsStringSync(indexContents);
}

Seth Ladd's avatar
Seth Ladd committed
412
void addHtmlBaseToIndex() {
413
  final File indexFile = File('$kPublishRoot/index.html');
Seth Ladd's avatar
Seth Ladd committed
414
  String indexContents = indexFile.readAsStringSync();
415 416 417 418
  indexContents = indexContents.replaceFirst(
    '</title>\n',
    '</title>\n  <base href="./flutter/">\n',
  );
419 420
  indexContents = indexContents.replaceAll(
    'href="Android/Android-library.html"',
421
    'href="/javadoc/"',
422
  );
423 424
  indexContents = indexContents.replaceAll(
      'href="iOS/iOS-library.html"',
425
      'href="/objcdoc/"',
426 427
  );

Seth Ladd's avatar
Seth Ladd committed
428 429 430 431
  indexFile.writeAsStringSync(indexContents);
}

void putRedirectInOldIndexLocation() {
432
  const String metaTag = '<meta http-equiv="refresh" content="0;URL=../index.html">';
433
  File('$kPublishRoot/flutter/index.html').writeAsStringSync(metaTag);
434 435
}

436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452

void writeSnippetsIndexFile() {
  final Directory snippetsDir = Directory(path.join(kPublishRoot, 'snippets'));
  if (snippetsDir.existsSync()) {
    const JsonEncoder jsonEncoder = JsonEncoder.withIndent('    ');
    final Iterable<File> files = snippetsDir
        .listSync()
        .whereType<File>()
        .where((File file) => path.extension(file.path) == '.json');
        // Combine all the metadata into a single JSON array.
    final Iterable<String> fileContents = files.map((File file) => file.readAsStringSync());
    final List<dynamic> metadataObjects = fileContents.map<dynamic>(json.decode).toList();
    final String jsonArray = jsonEncoder.convert(metadataObjects);
    File('$kPublishRoot/snippets/index.json').writeAsStringSync(jsonArray);
  }
}

Seth Ladd's avatar
Seth Ladd committed
453
List<String> findPackageNames() {
454
  return findPackages().map<String>((FileSystemEntity file) => path.basename(file.path)).toList();
455 456
}

457
/// Finds all packages in the Flutter SDK
458
List<Directory> findPackages() {
459
  return Directory('packages')
460
    .listSync()
461 462 463
    .where((FileSystemEntity entity) {
      if (entity is! Directory)
        return false;
464
      final File pubspec = File('${entity.path}/pubspec.yaml');
465 466
      // TODO(ianh): Use a real YAML parser here
      return !pubspec.readAsStringSync().contains('nodoc: true');
467
    })
468
    .cast<Directory>()
469 470 471
    .toList();
}

472
/// Returns import or on-disk paths for all libraries in the Flutter SDK.
473
Iterable<String> libraryRefs() sync* {
474
  for (final Directory dir in findPackages()) {
475
    final String dirName = path.basename(dir.path);
476
    for (final FileSystemEntity file in Directory('${dir.path}/lib').listSync()) {
477
      if (file is File && file.path.endsWith('.dart')) {
478 479
        yield '$dirName/${path.basename(file.path)}';
      }
480 481
    }
  }
482 483

  // Add a fake package for platform integration APIs.
484 485
  yield 'platform_integration/android.dart';
  yield 'platform_integration/ios.dart';
486 487
}

488
void printStream(Stream<List<int>> stream, { String prefix = '', List<Pattern> filter = const <Pattern>[] }) {
489 490
  assert(prefix != null);
  assert(filter != null);
491
  stream
492 493
    .transform<String>(utf8.decoder)
    .transform<String>(const LineSplitter())
494 495 496 497
    .listen((String line) {
      if (!filter.any((Pattern pattern) => line.contains(pattern)))
        print('$prefix$line'.trim());
    });
498
}