// Copyright 2014 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:convert'; import 'dart:io'; import 'dart:math' as math; import 'package:archive/archive_io.dart'; import 'package:args/args.dart'; import 'package:file/file.dart'; import 'package:file/local.dart'; import 'package:http/http.dart' as http; import 'package:intl/intl.dart'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as path; import 'package:platform/platform.dart'; import 'package:process/process.dart'; import 'package:pub_semver/pub_semver.dart'; import 'dartdoc_checker.dart'; const String kDummyPackageName = 'Flutter'; const String kPlatformIntegrationPackageName = 'platform_integration'; /// This script will generate documentation for the packages in `packages/` and /// write the documentation to the output directory specified on the command /// line. /// /// This script also updates the index.html file so that it can be placed at the /// root of api.flutter.dev. The files are kept inside of /// api.flutter.dev/flutter, so we need to manipulate paths a bit. See /// https://github.com/flutter/flutter/issues/3900 for more info. /// /// This will only work on UNIX systems, not Windows. It requires that 'git', /// 'zip', and 'tar' be in the PATH. It requires that 'flutter' has been run /// previously. It uses the version of Dart downloaded by the 'flutter' tool in /// this repository and will fail if that is absent. Future<void> main(List<String> arguments) async { const FileSystem filesystem = LocalFileSystem(); const ProcessManager processManager = LocalProcessManager(); const Platform platform = LocalPlatform(); // The place to find customization files and configuration files for docs // generation. final Directory docsRoot = filesystem .directory(FlutterInformation.instance.getFlutterRoot().childDirectory('dev').childDirectory('docs')) .absolute; final ArgParser argParser = _createArgsParser( publishDefault: docsRoot.childDirectory('doc').path, ); final ArgResults args = argParser.parse(arguments); if (args['help'] as bool) { print('Usage:'); print(argParser.usage); exit(0); } final Directory publishRoot = filesystem.directory(args['output-dir']! as String).absolute; final Directory packageRoot = publishRoot.parent; if (!filesystem.directory(packageRoot).existsSync()) { filesystem.directory(packageRoot).createSync(recursive: true); } if (!filesystem.directory(publishRoot).existsSync()) { filesystem.directory(publishRoot).createSync(recursive: true); } final Configurator configurator = Configurator( publishRoot: publishRoot, packageRoot: packageRoot, docsRoot: docsRoot, filesystem: filesystem, processManager: processManager, platform: platform, ); configurator.generateConfiguration(); final PlatformDocGenerator platformGenerator = PlatformDocGenerator(outputDir: publishRoot, filesystem: filesystem); platformGenerator.generatePlatformDocs(); final DartdocGenerator dartdocGenerator = DartdocGenerator( publishRoot: publishRoot, packageRoot: packageRoot, docsRoot: docsRoot, filesystem: filesystem, processManager: processManager, useJson: args['json'] as bool? ?? true, validateLinks: args['validate-links']! as bool, verbose: args['verbose'] as bool? ?? false, ); await dartdocGenerator.generateDartdoc(); await configurator.generateOfflineAssetsIfNeeded(); } ArgParser _createArgsParser({required String publishDefault}) { final ArgParser parser = ArgParser(); parser.addFlag('help', abbr: 'h', negatable: false, help: 'Show command help.'); parser.addFlag('verbose', defaultsTo: true, help: 'Whether to report all error messages (on) or attempt to ' 'filter out some known false positives (off). Shut this off ' 'locally if you want to address Flutter-specific issues.'); parser.addFlag('json', help: 'Display json-formatted output from dartdoc and skip stdout/stderr prefixing.'); parser.addFlag('validate-links', help: 'Display warnings for broken links generated by dartdoc (slow)'); parser.addOption('output-dir', defaultsTo: publishDefault, help: 'Sets the output directory for the documentation.'); return parser; } /// A class used to configure the staging area for building the docs in. /// /// The [generateConfiguration] function generates a dummy package with a /// pubspec. It copies any assets and customization files from the framework /// repo. It creates a metadata file for searches. /// /// Once the docs have been generated, [generateOfflineAssetsIfNeeded] will /// create offline assets like Dash/Zeal docsets and an offline ZIP file of the /// site if the build is a CI build that is not a presubmit build. class Configurator { Configurator({ required this.docsRoot, required this.publishRoot, required this.packageRoot, required this.filesystem, required this.processManager, required this.platform, }); /// The root of the directory in the Flutter repo where configuration data is /// stored. final Directory docsRoot; /// The root of the output area for the dartdoc docs. /// /// Typically this is a "doc" subdirectory under the [packageRoot]. final Directory publishRoot; /// The root of the staging area for creating docs. final Directory packageRoot; /// The [FileSystem] object used to create [File] and [Directory] objects. final FileSystem filesystem; /// The [ProcessManager] object used to invoke external processes. /// /// Can be replaced by tests to have a fake process manager. final ProcessManager processManager; /// The [Platform] to use for this run. /// /// Can be replaced by tests to test behavior on different plaforms. final Platform platform; void generateConfiguration() { final Version version = FlutterInformation.instance.getFlutterVersion(); _createDummyPubspec(); _createDummyLibrary(); _createPageFooter(packageRoot, version); _copyCustomizations(); _createSearchMetadata( docsRoot.childDirectory('lib').childFile('opensearch.xml'), publishRoot.childFile('opensearch.xml')); } Future<void> generateOfflineAssetsIfNeeded() async { // Only create the offline docs if we're running in a non-presubmit build: // it takes too long otherwise. if (platform.environment.containsKey('LUCI_CI') && (platform.environment['LUCI_PR'] ?? '').isEmpty) { _createOfflineZipFile(); await _createDocset(); _moveOfflineIntoPlace(); _createRobotsTxt(); } } /// Returns import or on-disk paths for all libraries in the Flutter SDK. Iterable<String> _libraryRefs() sync* { for (final Directory dir in findPackages(filesystem)) { final String dirName = dir.basename; for (final FileSystemEntity file in dir.childDirectory('lib').listSync()) { if (file is File && file.path.endsWith('.dart')) { yield '$dirName/${file.basename}'; } } } // Add a fake package for platform integration APIs. yield '$kPlatformIntegrationPackageName/android.dart'; yield '$kPlatformIntegrationPackageName/ios.dart'; } void _createDummyPubspec() { // Create the pubspec.yaml file. final List<String> pubspec = <String>[ 'name: $kDummyPackageName', 'homepage: https://flutter.dev', 'version: 0.0.0', 'environment:', " sdk: '>=3.0.0-0 <4.0.0'", 'dependencies:', for (final String package in findPackageNames(filesystem)) ' $package:\n sdk: flutter', ' $kPlatformIntegrationPackageName: 0.0.1', 'dependency_overrides:', ' $kPlatformIntegrationPackageName:', ' path: ${docsRoot.childDirectory(kPlatformIntegrationPackageName).path}', ]; packageRoot.childFile('pubspec.yaml').writeAsStringSync(pubspec.join('\n')); } void _createDummyLibrary() { final Directory libDir = packageRoot.childDirectory('lib'); libDir.createSync(); final StringBuffer contents = StringBuffer('library temp_doc;\n\n'); for (final String libraryRef in _libraryRefs()) { contents.writeln("import 'package:$libraryRef';"); } packageRoot.childDirectory('lib') ..createSync(recursive: true) ..childFile('temp_doc.dart').writeAsStringSync(contents.toString()); } void _createPageFooter(Directory footerPath, Version version) { final String timestamp = DateFormat('yyyy-MM-dd HH:mm').format(DateTime.now()); final String gitBranch = FlutterInformation.instance.getBranchName(); final String gitRevision = FlutterInformation.instance.getFlutterRevision(); final String gitBranchOut = gitBranch.isEmpty ? '' : '• $gitBranch'; footerPath.childFile('footer.html').writeAsStringSync('<script src="footer.js"></script>'); publishRoot.childDirectory('flutter').childFile('footer.js') ..createSync(recursive: true) ..writeAsStringSync(''' (function() { var span = document.querySelector('footer>span'); if (span) { span.innerText = 'Flutter $version • $timestamp • $gitRevision $gitBranchOut'; } var sourceLink = document.querySelector('a.source-link'); if (sourceLink) { sourceLink.href = sourceLink.href.replace('/master/', '/$gitRevision/'); } })(); '''); } void _copyCustomizations() { final List<String> files = <String>[ 'README.md', 'analysis_options.yaml', 'dartdoc_options.yaml', ]; for (final String file in files) { final File source = docsRoot.childFile(file); final File destination = packageRoot.childFile(file); // Have to canonicalize because otherwise things like /foo/bar/baz and // /foo/../foo/bar/baz won't compare as identical. if (path.canonicalize(source.absolute.path) != path.canonicalize(destination.absolute.path)) { source.copySync(destination.path); print('Copied ${path.canonicalize(source.absolute.path)} to ${path.canonicalize(destination.absolute.path)}'); } } final Directory assetsDir = filesystem.directory(publishRoot.childDirectory('assets')); final Directory assetSource = docsRoot.childDirectory('assets'); if (path.canonicalize(assetSource.absolute.path) == path.canonicalize(assetsDir.absolute.path)) { // Don't try and copy the directory over itself. return; } if (assetsDir.existsSync()) { assetsDir.deleteSync(recursive: true); } copyDirectorySync( docsRoot.childDirectory('assets'), assetsDir, onFileCopied: (File src, File dest) { print('Copied ${path.canonicalize(src.absolute.path)} to ${path.canonicalize(dest.absolute.path)}'); }, filesystem: filesystem, ); } /// Generates an OpenSearch XML description that can be used to add a custom /// search for Flutter API docs to the browser. Unfortunately, it has to know /// the URL to which site to search, so we customize it here based upon the /// branch name. void _createSearchMetadata(File templatePath, File metadataPath) { final String template = templatePath.readAsStringSync(); final String branch = FlutterInformation.instance.getBranchName(); final String metadata = template.replaceAll( '{SITE_URL}', branch == 'stable' ? 'https://api.flutter.dev/' : 'https://master-api.flutter.dev/', ); metadataPath.parent.create(recursive: true); metadataPath.writeAsStringSync(metadata); } Future<void> _createDocset() async { // Must have dashing installed: go get -u github.com/technosophos/dashing // Dashing produces a LOT of log output (~30MB), so we collect it, and just // show the end of it if there was a problem. print('${DateTime.now().toUtc()}: Building Flutter docset.'); // If dashing gets stuck, Cirrus will time out the build after an hour, and we // never get to see the logs. Thus, we run it in the background and tail the // logs only if it fails. final ProcessWrapper result = ProcessWrapper( await processManager.start( <String>[ 'dashing', 'build', '--source', publishRoot.path, '--config', docsRoot.childFile('dashing.json').path, ], workingDirectory: packageRoot.path, ), ); final List<int> buffer = <int>[]; result.stdout.listen(buffer.addAll); result.stderr.listen(buffer.addAll); // If the dashing process exited with an error, print the last 200 lines of stderr and exit. final int exitCode = await result.done; if (exitCode != 0) { print('Dashing docset generation failed with code $exitCode'); final List<String> output = systemEncoding.decode(buffer).split('\n'); print(output.sublist(math.max(output.length - 200, 0)).join('\n')); exit(exitCode); } buffer.clear(); // Copy the favicon file to the output directory. final File faviconFile = publishRoot.childDirectory('flutter').childDirectory('static-assets').childFile('favicon.png'); final File iconFile = packageRoot.childDirectory('flutter.docset').childFile('icon.png'); faviconFile ..createSync(recursive: true) ..copySync(iconFile.path); // Post-process the dashing output. final File infoPlist = packageRoot.childDirectory('flutter.docset').childDirectory('Contents').childFile('Info.plist'); String contents = infoPlist.readAsStringSync(); // Since I didn't want to add the XML package as a dependency just for this, // I just used a regular expression to make this simple change. final RegExp findRe = RegExp(r'(\s*<key>DocSetPlatformFamily</key>\s*<string>)[^<]+(</string>)', multiLine: true); contents = contents.replaceAllMapped(findRe, (Match match) { return '${match.group(1)}dartlang${match.group(2)}'; }); infoPlist.writeAsStringSync(contents); final Directory offlineDir = publishRoot.childDirectory('offline'); if (!offlineDir.existsSync()) { offlineDir.createSync(recursive: true); } tarDirectory(packageRoot, offlineDir.childFile('flutter.docset.tar.gz'), processManager: processManager); // Write the Dash/Zeal XML feed file. final bool isStable = platform.environment['LUCI_BRANCH'] == 'stable'; offlineDir.childFile('flutter.xml').writeAsStringSync('<entry>\n' ' <version>${FlutterInformation.instance.getFlutterVersion()}</version>\n' ' <url>https://${isStable ? '' : 'master-'}api.flutter.dev/offline/flutter.docset.tar.gz</url>\n' '</entry>\n'); } // Creates the offline ZIP file containing all of the website HTML files. void _createOfflineZipFile() { print('${DateTime.now().toLocal()}: Creating offline docs archive.'); zipDirectory(publishRoot, packageRoot.childFile('flutter.docs.zip'), processManager: processManager); } // Moves the generated offline archives into the publish directory so that // they can be included in the output ZIP file. void _moveOfflineIntoPlace() { print('${DateTime.now().toUtc()}: Moving offline docs into place.'); final Directory offlineDir = publishRoot.childDirectory('offline')..createSync(recursive: true); packageRoot.childFile('flutter.docs.zip').renameSync(offlineDir.childFile('flutter.docs.zip').path); } // Creates a robots.txt file that disallows indexing unless the branch is the // stable branch. void _createRobotsTxt() { final File robotsTxt = publishRoot.childFile('robots.txt'); if (FlutterInformation.instance.getBranchName() == 'stable') { robotsTxt.writeAsStringSync('# All robots welcome!'); } else { robotsTxt.writeAsStringSync('User-agent: *\nDisallow: /'); } } } /// Runs Dartdoc inside of the given pre-prepared staging area, prepared by /// [Configurator.generateConfiguration]. /// /// Performs a sanity check of the output once the generation is complete. class DartdocGenerator { DartdocGenerator({ required this.docsRoot, required this.publishRoot, required this.packageRoot, required this.filesystem, required this.processManager, this.useJson = true, this.validateLinks = true, this.verbose = false, }); /// The root of the directory in the Flutter repo where configuration data is /// stored. final Directory docsRoot; /// The root of the output area for the dartdoc docs. /// /// Typically this is a "doc" subdirectory under the [packageRoot]. final Directory publishRoot; /// The root of the staging area for creating docs. final Directory packageRoot; /// The [FileSystem] object used to create [File] and [Directory] objects. final FileSystem filesystem; /// The [ProcessManager] object used to invoke external processes. /// /// Can be replaced by tests to have a fake process manager. final ProcessManager processManager; /// Whether or not dartdoc should output an index.json file of the /// documentation. final bool useJson; // Whether or not to have dartdoc validate its own links. final bool validateLinks; /// Whether or not to filter overly verbose log output from dartdoc. final bool verbose; Future<void> generateDartdoc() async { final Directory flutterRoot = FlutterInformation.instance.getFlutterRoot(); final Map<String, String> pubEnvironment = <String, String>{ 'FLUTTER_ROOT': flutterRoot.absolute.path, }; // If there's a .pub-cache dir in the Flutter root, use that. final File pubCache = flutterRoot.childFile('.pub-cache'); if (pubCache.existsSync()) { pubEnvironment['PUB_CACHE'] = pubCache.path; } // Run pub. ProcessWrapper process = ProcessWrapper(await runPubProcess( arguments: <String>['get'], workingDirectory: packageRoot, environment: pubEnvironment, filesystem: filesystem, processManager: processManager, )); printStream(process.stdout, prefix: 'pub:stdout: '); printStream(process.stderr, prefix: 'pub:stderr: '); final int code = await process.done; if (code != 0) { exit(code); } final Version version = FlutterInformation.instance.getFlutterVersion(); // Verify which version of snippets and dartdoc we're using. final ProcessResult snippetsResult = Process.runSync( FlutterInformation.instance.getDartBinaryPath().path, <String>[ 'pub', 'global', 'list', ], workingDirectory: packageRoot.path, environment: pubEnvironment, stdoutEncoding: utf8, ); print(''); final Iterable<RegExpMatch> versionMatches = RegExp(r'^(?<name>snippets|dartdoc) (?<version>[^\s]+)', multiLine: true) .allMatches(snippetsResult.stdout as String); for (final RegExpMatch match in versionMatches) { print('${match.namedGroup('name')} version: ${match.namedGroup('version')}'); } print('flutter version: $version\n'); // Dartdoc warnings and errors in these packages are considered fatal. // All packages owned by flutter should be in the list. final List<String> flutterPackages = <String>[ kDummyPackageName, kPlatformIntegrationPackageName, ...findPackageNames(filesystem), // TODO(goderbauer): Figure out how to only include `dart:ui` of // `sky_engine` below, https://github.com/dart-lang/dartdoc/issues/2278. // 'sky_engine', ]; // Generate the documentation. We don't need to exclude flutter_tools in // this list because it's not in the recursive dependencies of the package // defined at packageRoot final List<String> dartdocArgs = <String>[ 'global', 'run', '--enable-asserts', 'dartdoc', '--output', publishRoot.childDirectory('flutter').path, '--allow-tools', if (useJson) '--json', if (validateLinks) '--validate-links' else '--no-validate-links', '--link-to-source-excludes', flutterRoot.childDirectory('bin').childDirectory('cache').path, '--link-to-source-root', flutterRoot.path, '--link-to-source-uri-template', 'https://github.com/flutter/flutter/blob/master/%f%#L%l%', '--inject-html', '--use-base-href', '--header', docsRoot.childFile('styles.html').path, '--header', docsRoot.childFile('analytics.html').path, '--header', docsRoot.childFile('survey.html').path, '--header', docsRoot.childFile('snippets.html').path, '--header', docsRoot.childFile('opensearch.html').path, '--footer-text', packageRoot.childFile('footer.html').path, '--allow-warnings-in-packages', flutterPackages.join(','), '--exclude-packages', <String>[ 'analyzer', 'args', 'barback', 'csslib', 'flutter_goldens', 'flutter_goldens_client', 'front_end', 'fuchsia_remote_debug_protocol', 'glob', 'html', 'http_multi_server', 'io', 'isolate', 'js', 'kernel', 'logging', 'mime', 'mockito', 'node_preamble', 'plugin', 'shelf', 'shelf_packages_handler', 'shelf_static', 'shelf_web_socket', 'utf', 'watcher', 'yaml', ].join(','), '--exclude', <String>[ 'dart:io/network_policy.dart', // dart-lang/dartdoc#2437 'package:Flutter/temp_doc.dart', 'package:http/browser_client.dart', 'package:intl/intl_browser.dart', 'package:matcher/mirror_matchers.dart', 'package:quiver/io.dart', 'package:quiver/mirrors.dart', 'package:vm_service_client/vm_service_client.dart', 'package:web_socket_channel/html.dart', ].join(','), '--favicon', docsRoot.childFile('favicon.ico').absolute.path, '--package-order', 'flutter,Dart,$kPlatformIntegrationPackageName,flutter_test,flutter_driver', '--auto-include-dependencies', ]; String quote(String arg) => arg.contains(' ') ? "'$arg'" : arg; print('Executing: (cd "${packageRoot.path}" ; ' '${FlutterInformation.instance.getDartBinaryPath().path} ' '${dartdocArgs.map<String>(quote).join(' ')})'); process = ProcessWrapper(await runPubProcess( arguments: dartdocArgs, workingDirectory: packageRoot, environment: pubEnvironment, )); printStream( process.stdout, prefix: useJson ? '' : 'dartdoc:stdout: ', filter: <Pattern>[ if (!verbose) RegExp(r'^Generating docs for library '), // Unnecessary verbosity ], ); printStream( process.stderr, prefix: useJson ? '' : 'dartdoc:stderr: ', filter: <Pattern>[ if (!verbose) RegExp( // Remove warnings from packages outside our control r'^ warning: .+: \(.+[\\/]\.pub-cache[\\/]hosted[\\/]pub.dartlang.org[\\/].+\)', ), ], ); final int exitCode = await process.done; if (exitCode != 0) { exit(exitCode); } _sanityCheckDocs(); checkForUnresolvedDirectives(publishRoot.childDirectory('flutter').path); _createIndexAndCleanup(); print('Documentation written to ${publishRoot.path}'); } void _sanityCheckExample(String fileString, String regExpString) { final File file = filesystem.file(fileString); if (file.existsSync()) { final RegExp regExp = RegExp(regExpString, dotAll: true); final String contents = file.readAsStringSync(); if (!regExp.hasMatch(contents)) { throw Exception("Missing example code matching '$regExpString' in ${file.path}."); } } else { throw Exception( "Missing example code sanity test file ${file.path}. Either it didn't get published, or you might have to update the test to look at a different file."); } } /// Runs a sanity check by running a test. void _sanityCheckDocs([Platform platform = const LocalPlatform()]) { final Directory flutterDirectory = publishRoot.childDirectory('flutter'); final Directory widgetsDirectory = flutterDirectory.childDirectory('widgets'); final List<File> canaries = <File>[ publishRoot.childDirectory('assets').childFile('overrides.css'), flutterDirectory.childDirectory('dart-io').childFile('File-class.html'), flutterDirectory.childDirectory('dart-ui').childFile('Canvas-class.html'), flutterDirectory.childDirectory('dart-ui').childDirectory('Canvas').childFile('drawRect.html'), flutterDirectory .childDirectory('flutter_driver') .childDirectory('FlutterDriver') .childFile('FlutterDriver.connectedTo.html'), flutterDirectory.childDirectory('flutter_test').childDirectory('WidgetTester').childFile('pumpWidget.html'), flutterDirectory.childDirectory('material').childFile('Material-class.html'), flutterDirectory.childDirectory('material').childFile('Tooltip-class.html'), widgetsDirectory.childFile('Widget-class.html'), widgetsDirectory.childFile('Listener-class.html'), ]; for (final File canary in canaries) { if (!canary.existsSync()) { throw Exception('Missing "${canary.path}", which probably means the documentation failed to build correctly.'); } } // Make sure at least one example of each kind includes source code. // Check a "sample" example, any one will do. _sanityCheckExample( widgetsDirectory.childFile('showGeneralDialog.html').path, r'\s*<pre\s+id="longSnippet1".*<code\s+class="language-dart">\s*import 'package:flutter/material.dart';', ); // Check a "snippet" example, any one will do. _sanityCheckExample( widgetsDirectory.childDirectory('ModalRoute').childFile('barrierColor.html').path, r'\s*<pre.*id="sample-code">.*Color\s+get\s+barrierColor.*</pre>', ); // Check a "dartpad" example, any one will do, and check for the correct URL // arguments. // Just use "master" for any branch other than the LUCI_BRANCH. final String? luciBranch = platform.environment['LUCI_BRANCH']?.trim(); final String expectedBranch = luciBranch != null && luciBranch.isNotEmpty ? luciBranch : 'master'; final List<String> argumentRegExps = <String>[ r'split=\d+', r'run=true', r'sample_id=widgets\.Listener\.\d+', 'sample_channel=$expectedBranch', 'channel=$expectedBranch', ]; for (final String argumentRegExp in argumentRegExps) { _sanityCheckExample( widgetsDirectory.childFile('Listener-class.html').path, r'\s*<iframe\s+class="snippet-dartpad"\s+src="' r'https:\/\/dartpad.dev\/embed-flutter.html\?.*?\b' '$argumentRegExp' r'\b.*">\s*<\/iframe>', ); } } /// Creates a custom index.html because we try to maintain old /// paths. Cleanup unused index.html files no longer needed. void _createIndexAndCleanup() { print('\nCreating a custom index.html in ${publishRoot.childFile('index.html').path}'); _copyIndexToRootOfDocs(); _addHtmlBaseToIndex(); _changePackageToSdkInTitlebar(); _putRedirectInOldIndexLocation(); _writeSnippetsIndexFile(); print('\nDocs ready to go!'); } void _copyIndexToRootOfDocs() { publishRoot.childDirectory('flutter').childFile('index.html').copySync(publishRoot.childFile('index.html').path); } void _changePackageToSdkInTitlebar() { final File indexFile = publishRoot.childFile('index.html'); String indexContents = indexFile.readAsStringSync(); indexContents = indexContents.replaceFirst( '<li><a href="https://flutter.dev">Flutter package</a></li>', '<li><a href="https://flutter.dev">Flutter SDK</a></li>', ); indexFile.writeAsStringSync(indexContents); } void _addHtmlBaseToIndex() { final File indexFile = publishRoot.childFile('index.html'); String indexContents = indexFile.readAsStringSync(); indexContents = indexContents.replaceFirst( '</title>\n', '</title>\n <base href="./flutter/">\n', ); indexContents = indexContents.replaceAll( 'href="Android/Android-library.html"', 'href="/javadoc/"', ); indexContents = indexContents.replaceAll( 'href="iOS/iOS-library.html"', 'href="/objcdoc/"', ); indexFile.writeAsStringSync(indexContents); } void _putRedirectInOldIndexLocation() { const String metaTag = '<meta http-equiv="refresh" content="0;URL=../index.html">'; publishRoot.childDirectory('flutter').childFile('index.html').writeAsStringSync(metaTag); } void _writeSnippetsIndexFile() { final Directory snippetsDir = publishRoot.childDirectory('snippets'); if (snippetsDir.existsSync()) { const JsonEncoder jsonEncoder = JsonEncoder.withIndent(' '); final Iterable<File> files = snippetsDir.listSync().whereType<File>().where((File file) => path.extension(file.path) == '.json'); // Combine all the metadata into a single JSON array. final Iterable<String> fileContents = files.map((File file) => file.readAsStringSync()); final List<dynamic> metadataObjects = fileContents.map<dynamic>(json.decode).toList(); final String jsonArray = jsonEncoder.convert(metadataObjects); snippetsDir.childFile('index.json').writeAsStringSync(jsonArray); } } } /// Downloads and unpacks the platform specific documentation generated by the /// engine build. /// /// Unpacks and massages the data so that it can be properly included in the /// output archive. class PlatformDocGenerator { PlatformDocGenerator({required this.outputDir, required this.filesystem}); final FileSystem filesystem; final Directory outputDir; final String engineRevision = FlutterInformation.instance.getEngineRevision(); final String engineRealm = FlutterInformation.instance.getEngineRealm(); /// This downloads an archive of platform docs for the engine from the artifact /// store and extracts them to the location used for Dartdoc. void generatePlatformDocs() { final String realm = engineRealm.isNotEmpty ? '$engineRealm/' : ''; final String javadocUrl = 'https://storage.googleapis.com/${realm}flutter_infra_release/flutter/$engineRevision/android-javadoc.zip'; _extractDocs(javadocUrl, 'javadoc', 'io/flutter/view/FlutterView.html', outputDir); final String objcdocUrl = 'https://storage.googleapis.com/${realm}flutter_infra_release/flutter/$engineRevision/ios-objcdoc.zip'; _extractDocs(objcdocUrl, 'objcdoc', 'Classes/FlutterViewController.html', outputDir); } /// Fetches the zip archive at the specified url. /// /// Returns null if the archive fails to download after [maxTries] attempts. Future<Archive?> _fetchArchive(String url, int maxTries) async { List<int>? responseBytes; for (int i = 0; i < maxTries; i++) { final http.Response response = await http.get(Uri.parse(url)); if (response.statusCode == 200) { responseBytes = response.bodyBytes; break; } stderr.writeln('Failed attempt ${i + 1} to fetch $url.'); // On failure print a short snipped from the body in case it's helpful. final int bodyLength = math.min(1024, response.body.length); stderr.writeln('Response status code ${response.statusCode}. Body: ${response.body.substring(0, bodyLength)}'); sleep(const Duration(seconds: 1)); } return responseBytes == null ? null : ZipDecoder().decodeBytes(responseBytes); } Future<void> _extractDocs(String url, String docName, String checkFile, Directory outputDir) async { const int maxTries = 5; final Archive? archive = await _fetchArchive(url, maxTries); if (archive == null) { stderr.writeln('Failed to fetch zip archive from: $url after $maxTries attempts. Giving up.'); exit(1); } final Directory output = outputDir.childDirectory(docName); print('Extracting $docName to ${output.path}'); output.createSync(recursive: true); for (final ArchiveFile af in archive) { if (!af.name.endsWith('/')) { final File file = filesystem.file('${output.path}/${af.name}'); file.createSync(recursive: true); file.writeAsBytesSync(af.content as List<int>); } } /// If object then copy files to old location if the archive is using the new location. final Directory objcDocsDir = output.childDirectory('objectc_docs'); if (objcDocsDir.existsSync()) { copyDirectorySync(objcDocsDir, output, filesystem: filesystem); } final File testFile = output.childFile(checkFile); if (!testFile.existsSync()) { print('Expected file ${testFile.path} not found'); exit(1); } print('$docName ready to go!'); } } /// Recursively copies `srcDir` to `destDir`, invoking [onFileCopied], if /// specified, for each source/destination file pair. /// /// Creates `destDir` if needed. void copyDirectorySync(Directory srcDir, Directory destDir, {void Function(File srcFile, File destFile)? onFileCopied, required FileSystem filesystem}) { if (!srcDir.existsSync()) { throw Exception('Source directory "${srcDir.path}" does not exist, nothing to copy'); } if (!destDir.existsSync()) { destDir.createSync(recursive: true); } for (final FileSystemEntity entity in srcDir.listSync()) { final String newPath = path.join(destDir.path, path.basename(entity.path)); if (entity is File) { final File newFile = filesystem.file(newPath); entity.copySync(newPath); onFileCopied?.call(entity, newFile); } else if (entity is Directory) { copyDirectorySync(entity, filesystem.directory(newPath), filesystem: filesystem); } else { throw Exception('${entity.path} is neither File nor Directory'); } } } void printStream(Stream<List<int>> stream, {String prefix = '', List<Pattern> filter = const <Pattern>[]}) { stream.transform<String>(utf8.decoder).transform<String>(const LineSplitter()).listen((String line) { if (!filter.any((Pattern pattern) => line.contains(pattern))) { print('$prefix$line'.trim()); } }); } void zipDirectory(Directory src, File output, {required ProcessManager processManager}) { // We would use the archive package to do this in one line, but it // is a lot slower, and doesn't do compression nearly as well. final ProcessResult zipProcess = processManager.runSync( <String>[ 'zip', '-r', '-9', '-q', output.path, '.', ], workingDirectory: src.path, ); if (zipProcess.exitCode != 0) { print('Creating offline ZIP archive ${output.path} failed:'); print(zipProcess.stderr); exit(1); } } void tarDirectory(Directory src, File output, {required ProcessManager processManager}) { // We would use the archive package to do this in one line, but it // is a lot slower, and doesn't do compression nearly as well. final ProcessResult tarProcess = processManager.runSync( <String>[ 'tar', 'cf', output.path, '--use-compress-program', 'gzip --best', 'flutter.docset', ], workingDirectory: src.path, ); if (tarProcess.exitCode != 0) { print('Creating a tarball ${output.path} failed:'); print(tarProcess.stderr); exit(1); } } Future<Process> runPubProcess({ required List<String> arguments, Directory? workingDirectory, Map<String, String>? environment, @visibleForTesting ProcessManager processManager = const LocalProcessManager(), @visibleForTesting FileSystem filesystem = const LocalFileSystem(), }) { return processManager.start( <Object>[FlutterInformation.instance.getDartBinaryPath().path, 'pub', ...arguments], workingDirectory: (workingDirectory ?? filesystem.currentDirectory).path, environment: environment, ); } List<String> findPackageNames(FileSystem filesystem) { return findPackages(filesystem).map<String>((FileSystemEntity file) => path.basename(file.path)).toList(); } /// Finds all packages in the Flutter SDK List<Directory> findPackages(FileSystem filesystem) { return FlutterInformation.instance .getFlutterRoot() .childDirectory('packages') .listSync() .where((FileSystemEntity entity) { if (entity is! Directory) { return false; } final File pubspec = filesystem.file('${entity.path}/pubspec.yaml'); if (!pubspec.existsSync()) { print("Unexpected package '${entity.path}' found in packages directory"); return false; } // TODO(ianh): Use a real YAML parser here return !pubspec.readAsStringSync().contains('nodoc: true'); }) .cast<Directory>() .toList(); } /// An exception class used to indicate problems when collecting information. class DartdocException implements Exception { DartdocException(this.message, {this.file, this.line}); final String message; final String? file; final int? line; @override String toString() { if (file != null || line != null) { final String fileStr = file == null ? '' : '$file:'; final String lineStr = line == null ? '' : '$line:'; return '$runtimeType: $fileStr$lineStr: $message'; } else { return '$runtimeType: $message'; } } } /// A singleton used to consolidate the way in which information about the /// Flutter repo and environment is collected. /// /// Collects the information once, and caches it for any later access. /// /// The singleton instance can be overridden by tests by setting [instance]. class FlutterInformation { FlutterInformation({ this.platform = const LocalPlatform(), this.processManager = const LocalProcessManager(), this.filesystem = const LocalFileSystem(), }); final Platform platform; final ProcessManager processManager; final FileSystem filesystem; static FlutterInformation? _instance; static FlutterInformation get instance => _instance ??= FlutterInformation(); @visibleForTesting static set instance(FlutterInformation? value) => _instance = value; /// The path to the Dart binary in the Flutter repo. /// /// This is probably a shell script. File getDartBinaryPath() { return getFlutterRoot().childDirectory('bin').childFile('dart'); } /// The path to the Flutter repo root directory. /// /// If the environment variable `FLUTTER_ROOT` is set, will use that instead /// of looking for it. /// /// Otherwise, uses the output of `flutter --version --machine` to find the /// Flutter root. Directory getFlutterRoot() { if (platform.environment['FLUTTER_ROOT'] != null) { return filesystem.directory(platform.environment['FLUTTER_ROOT']); } return getFlutterInformation()['flutterRoot']! as Directory; } /// Gets the semver version of the Flutter framework in the repo. Version getFlutterVersion() => getFlutterInformation()['frameworkVersion']! as Version; /// Gets the git hash of the engine used by the Flutter framework in the repo. String getEngineRevision() => getFlutterInformation()['engineRevision']! as String; /// Gets the value stored in bin/internal/engine.realm used by the Flutter /// framework repo. String getEngineRealm() => getFlutterInformation()['engineRealm']! as String; /// Gets the git hash of the Flutter framework in the repo. String getFlutterRevision() => getFlutterInformation()['flutterGitRevision']! as String; /// Gets the name of the current branch in the Flutter framework in the repo. String getBranchName() => getFlutterInformation()['branchName']! as String; Map<String, Object>? _cachedFlutterInformation; /// Gets a Map of various kinds of information about the Flutter repo. Map<String, Object> getFlutterInformation() { if (_cachedFlutterInformation != null) { return _cachedFlutterInformation!; } String flutterVersionJson; if (platform.environment['FLUTTER_VERSION'] != null) { flutterVersionJson = platform.environment['FLUTTER_VERSION']!; } else { String flutterCommand; if (platform.environment['FLUTTER_ROOT'] != null) { flutterCommand = filesystem .directory(platform.environment['FLUTTER_ROOT']) .childDirectory('bin') .childFile('flutter') .absolute .path; } else { flutterCommand = 'flutter'; } ProcessResult result; try { result = processManager.runSync(<String>[flutterCommand, '--version', '--machine'], stdoutEncoding: utf8); } on ProcessException catch (e) { throw DartdocException( 'Unable to determine Flutter information. Either set FLUTTER_ROOT, or place flutter command in your path.\n$e'); } if (result.exitCode != 0) { throw DartdocException('Unable to determine Flutter information, because of abnormal exit to flutter command.'); } flutterVersionJson = (result.stdout as String) .replaceAll('Waiting for another flutter command to release the startup lock...', ''); } final Map<String, dynamic> flutterVersion = json.decode(flutterVersionJson) as Map<String, dynamic>; if (flutterVersion['flutterRoot'] == null || flutterVersion['frameworkVersion'] == null || flutterVersion['dartSdkVersion'] == null) { throw DartdocException( 'Flutter command output has unexpected format, unable to determine flutter root location.'); } final Map<String, Object> info = <String, Object>{}; final Directory flutterRoot = filesystem.directory(flutterVersion['flutterRoot']! as String); info['flutterRoot'] = flutterRoot; info['frameworkVersion'] = Version.parse(flutterVersion['frameworkVersion'] as String); info['engineRevision'] = flutterVersion['engineRevision'] as String; info['engineRealm'] = filesystem.file( path.join(flutterRoot.path, 'bin', 'internal', 'engine.realm', )).readAsStringSync().trim(); final RegExpMatch? dartVersionRegex = RegExp(r'(?<base>[\d.]+)(?:\s+\(build (?<detail>[-.\w]+)\))?') .firstMatch(flutterVersion['dartSdkVersion'] as String); if (dartVersionRegex == null) { throw DartdocException( 'Flutter command output has unexpected format, unable to parse dart SDK version ${flutterVersion['dartSdkVersion']}.'); } info['dartSdkVersion'] = Version.parse(dartVersionRegex.namedGroup('detail') ?? dartVersionRegex.namedGroup('base')!); info['branchName'] = _getBranchName(); info['flutterGitRevision'] = _getFlutterGitRevision(); _cachedFlutterInformation = info; return info; } // Get the name of the release branch. // // On LUCI builds, the git HEAD is detached, so first check for the env // variable "LUCI_BRANCH"; if it is not set, fall back to calling git. String _getBranchName() { final String? luciBranch = platform.environment['LUCI_BRANCH']; if (luciBranch != null && luciBranch.trim().isNotEmpty) { return luciBranch.trim(); } final ProcessResult gitResult = processManager.runSync(<String>['git', 'status', '-b', '--porcelain']); if (gitResult.exitCode != 0) { throw 'git status exit with non-zero exit code: ${gitResult.exitCode}'; } final RegExp gitBranchRegexp = RegExp(r'^## (.*)'); final RegExpMatch? gitBranchMatch = gitBranchRegexp.firstMatch((gitResult.stdout as String).trim().split('\n').first); return gitBranchMatch == null ? '' : gitBranchMatch.group(1)!.split('...').first; } // Get the git revision for the repo. String _getFlutterGitRevision() { const int kGitRevisionLength = 10; final ProcessResult gitResult = Process.runSync('git', <String>['rev-parse', 'HEAD']); if (gitResult.exitCode != 0) { throw 'git rev-parse exit with non-zero exit code: ${gitResult.exitCode}'; } final String gitRevision = (gitResult.stdout as String).trim(); return gitRevision.length > kGitRevisionLength ? gitRevision.substring(0, kGitRevisionLength) : gitRevision; } }