Upload sample catalog screenshots to cloud storage (#10462)

dev/devicelab/bin/tasks/android_sample_catalog_generator.dart

@@ -6,9 +6,10 @@ import 'dart:async';
 
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
 import 'package:flutter_devicelab/tasks/sample_catalog_generator.dart';
 
-Future<Null> main() async {
+Future<Null> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
-  await task(samplePageCatalogGenerator);
+  await task(() => samplePageCatalogGenerator(extractCloudAuthTokenArg(args)));
 }

dev/devicelab/bin/tasks/ios_sample_catalog_generator.dart

@@ -6,9 +6,10 @@ import 'dart:async';
 
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
 import 'package:flutter_devicelab/tasks/sample_catalog_generator.dart';
 
-Future<Null> main() async {
+Future<Null> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.ios;
-  await task(samplePageCatalogGenerator);
+  await task(() => samplePageCatalogGenerator(extractCloudAuthTokenArg(args)));
 }

dev/devicelab/lib/framework/utils.dart

@@ -6,6 +6,7 @@ import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 
+import 'package:args/args.dart';
 import 'package:meta/meta.dart';
 import 'package:path/path.dart' as path;
 import 'package:process/process.dart';
@@ -452,3 +453,23 @@ Future<int> findAvailablePort() async {
 }
 
 bool canRun(String path) => _processManager.canRun(path);
+
+String extractCloudAuthTokenArg(List<String> rawArgs) {
+  final ArgParser argParser = new ArgParser()..addOption('cloud-auth-token');
+  ArgResults args;
+  try {
+    args = argParser.parse(rawArgs);
+  } on FormatException catch(error) {
+    stderr.writeln('${error.message}\n');
+    stderr.writeln('Usage:\n');
+    stderr.writeln(argParser.usage);
+    return null;
+  }
+
+  final String token = args['cloud-auth-token'];
+  if (token == null) {
+    stderr.writeln('Required option --cloud-auth-token not found');
+    return null;
+  }
+  return token;
+}

dev/devicelab/lib/tasks/sample_catalog_generator.dart

@@ -10,7 +10,7 @@ import '../framework/framework.dart';
 import '../framework/ios.dart';
 import '../framework/utils.dart';
 
-Future<TaskResult> samplePageCatalogGenerator() async {
+Future<TaskResult> samplePageCatalogGenerator(String authorizationToken) async {
   final Device device = await devices.workingDevice;
   await device.unlock();
   final String deviceId = device.deviceId;
@@ -30,6 +30,12 @@ Future<TaskResult> samplePageCatalogGenerator() async {
       '--device-id',
       deviceId,
     ]);
+
+    await dart(<String>[
+      'bin/save_screenshots.dart',
+      await getCurrentFlutterRepoCommit(),
+      authorizationToken,
+    ]);
   });
 
   return new TaskResult.success(null);

examples/catalog/animated_list.iml

-<?xml version="1.0" encoding="UTF-8"?>
-<module type="FLUTTER_MODULE_TYPE" version="4">
-  <component name="NewModuleRootManager" inherit-compiler-output="true">
-    <exclude-output />
-    <content url="file://$MODULE_DIR$">
-      <excludeFolder url="file://$MODULE_DIR$/.idea" />
-      <excludeFolder url="file://$MODULE_DIR$/.pub" />
-      <excludeFolder url="file://$MODULE_DIR$/build" />
-      <excludeFolder url="file://$MODULE_DIR$/packages" />
-    </content>
-    <orderEntry type="sourceFolder" forTests="false" />
-    <orderEntry type="library" name="Dart SDK" level="project" />
-    <orderEntry type="library" name="Dart SDK" level="application" />
-    <orderEntry type="library" name="Dart Packages" level="project" />
-  </component>
-</module>
\ No newline at end of file

examples/catalog/bin/sample_page.dart

@@ -7,11 +7,13 @@
 
 import 'dart:io';
 
+import 'package:path/path.dart';
+
 class SampleError extends Error {
   SampleError(this.message);
   final String message;
   @override
-  String toString() => message;
+  String toString() => 'SampleError($message)';
 }
 
 // Sample apps are .dart files in the lib directory which contain a block
@@ -83,14 +85,7 @@ class SampleGenerator {
 
   // If sourceFile is lib/foo.dart then sourceName is foo. The sourceName
   // is used to create derived filenames like foo.md or foo.png.
-  String get sourceName {
-    // In /foo/bar/baz.dart, matches baz.dart, match[1] == 'baz'
-    final RegExp nameRE = new RegExp(r'(\w+)\.dart$');
-    final Match nameMatch = nameRE.firstMatch(sourceFile.path);
-    if (nameMatch.groupCount != 1)
-      throw new SampleError('bad source file name ${sourceFile.path}');
-    return nameMatch[1];
-  }
+  String get sourceName => basenameWithoutExtension(sourceFile.path);
 
   // The name of the widget class that defines this sample app, like 'FooSample'.
   String get sampleClass => commentValues["sample"];
@@ -161,6 +156,8 @@ void generate() {
     }
   });
 
+  // Causes the generated imports to appear in alphabetical order.
+  // Avoid complaints from flutter lint.
   samples.sort((SampleGenerator a, SampleGenerator b) {
     return a.sourceName.compareTo(b.sourceName);
   });

examples/catalog/bin/save_screenshots.dart

+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+import 'dart:math' as math;
+
+import 'package:image/image.dart';
+import 'package:path/path.dart';
+
+String authorizationToken;
+
+class UploadError extends Error {
+  UploadError(this.message);
+  final String message;
+  @override
+  String toString() => 'UploadError($message)';
+}
+
+void logMessage(String s) { print(s); }
+
+class Upload {
+  Upload(this.fromPath, this.largeName, this.smallName);
+
+  static math.Random random;
+  static const String uriAuthority = 'www.googleapis.com';
+  static const String uriPath = 'upload/storage/v1/b/flutter-catalog/o';
+
+  final String fromPath;
+  final String largeName;
+  final String smallName;
+
+  List<int> largeImage;
+  List<int> smallImage;
+  bool largeImageSaved;
+  int retryCount = 0;
+  bool isComplete = false;
+
+  // Exponential backoff per https://cloud.google.com/storage/docs/exponential-backoff
+  Duration get timeLimit {
+    if (retryCount == 0)
+      return const Duration(milliseconds: 1000);
+    random ??= new math.Random();
+    return new Duration(milliseconds: random.nextInt(1000) + math.pow(2, retryCount) * 1000);
+  }
+
+  Future<bool> save(HttpClient client, String name, List<int> content) async {
+    try {
+      final Uri uri = new Uri.https(uriAuthority, uriPath, <String, String>{
+        'uploadType': 'media',
+        'name': name,
+      });
+      final HttpClientRequest request = await client.postUrl(uri);
+      request
+        ..headers.contentType = new ContentType('image', 'png')
+        ..headers.add('Authorization', 'Bearer $authorizationToken')
+        ..add(content);
+
+      final HttpClientResponse response = await request.close().timeout(timeLimit);
+      if (response.statusCode == HttpStatus.OK) {
+        await response.drain<Null>();
+      } else {
+        // TODO(hansmuller): only retry on 5xx and 429 responses
+        logMessage('Request to save "$name" (length ${content.length}) failed with status ${response.statusCode}, will retry');
+        logMessage(await response.transform(UTF8.decoder).join());
+      }
+      return response.statusCode == HttpStatus.OK;
+    } on TimeoutException catch (_) {
+      logMessage('Request to save "$name" (length ${content.length}) timed out, will retry');
+      return false;
+    }
+  }
+
+  Future<bool> run(HttpClient client) async {
+    assert(!isComplete);
+    if (retryCount > 2)
+      throw new UploadError('upload of "$fromPath" to "$largeName" and "$smallName" failed after 2 retries');
+
+    largeImage ??= await new File(fromPath).readAsBytes();
+    smallImage ??= encodePng(copyResize(decodePng(largeImage), 400));
+
+    if (!largeImageSaved)
+      largeImageSaved = await save(client, largeName, largeImage);
+    isComplete = largeImageSaved && await save(client, smallName, smallImage);
+
+    retryCount += 1;
+    return isComplete;
+  }
+
+  static bool isNotComplete(Upload upload) => !upload.isComplete;
+}
+
+Future<Null> saveScreenshots(List<String> fromPaths, List<String> largeNames, List<String> smallNames) async {
+  assert(fromPaths.length == largeNames.length);
+  assert(fromPaths.length == smallNames.length);
+
+  List<Upload> uploads = new List<Upload>(fromPaths.length);
+  for (int index = 0; index < uploads.length; index += 1)
+    uploads[index] = new Upload(fromPaths[index], largeNames[index], smallNames[index]);
+
+  final HttpClient client = new HttpClient();
+  while(uploads.any(Upload.isNotComplete)) {
+    uploads = uploads.where(Upload.isNotComplete).toList();
+    await Future.wait(uploads.map((Upload upload) => upload.run(client)));
+  }
+  client.close();
+}
+
+
+// If path is lib/foo.png then screenshotName is foo.
+String screenshotName(String path) => basenameWithoutExtension(path);
+
+Future<Null> main(List<String> args) async {
+  if (args.length != 2)
+    throw new UploadError('Usage: dart bin/save_screenshots.dart commit authorization');
+
+  final Directory outputDirectory = new Directory('.generated');
+  final List<String> screenshots = <String>[];
+  outputDirectory.listSync().forEach((FileSystemEntity entity) {
+    if (entity is File && entity.path.endsWith('.png')) {
+      final File file = entity;
+      screenshots.add(file.path);
+    }
+  });
+
+  final String commit = args[0];
+  final List<String> largeNames = <String>[]; // Cloud storage names for the full res screenshots.
+  final List<String> smallNames = <String>[]; // Likewise for the scaled down screenshots.
+  for (String path in screenshots) {
+    final String name = screenshotName(path);
+    largeNames.add('$commit/$name.png');
+    smallNames.add('$commit/${name}_small.png');
+  }
+
+  authorizationToken = args[1];
+  await saveScreenshots(screenshots, largeNames, smallNames);
+}

examples/catalog/bin/screenshot_test.dart.template

@@ -8,6 +8,7 @@ import 'package:test/test.dart';
 
 void main() {
   group('sample screenshots', () {
+    final String prefix = Platform.isMacOS ? 'ios_' : "";
     FlutterDriver driver;
 
     setUpAll(() async {
@@ -19,7 +20,6 @@ void main() {
     });
 
     test('take sample screenshots', () async {
-      final String prefix = Platform.isMacOS ? 'ios_' : "";
       final List<String> paths = <String>[
         @(paths)
       ];

examples/catalog/pubspec.yaml

-name: animated_list
-description: A sample app for AnimatedList
+name: sample_catalog
+description: A collection of Flutter sample apps
 dependencies:
   flutter:
     sdk: flutter
+  image:
+  path: ^1.4.0
 
 dev_dependencies:
   flutter_test: