Port our microbenchmarks to the new world (#4121)

Our microbenchmarks now run on real devices.

dev/benchmarks/microbenchmarks/.gitignore

+.DS_Store
+.atom/
+.idea
+.packages
+.pub/
+build/
+ios/.generated/
+packages
+pubspec.lock

dev/benchmarks/microbenchmarks/README.md

+# microbenchmarks
+
+To run these benchmarks on a device:
+
+```
+flutter run --release -t lib/gestures/velocity_tracker_data.dart
+flutter run --release -t lib/stocks/animation_bench.dart
+flutter run --release -t lib/stocks/build_bench.dart
+flutter run --release -t lib/stocks/layout_bench.dart
+```
+
+The results should be in the device logs.

dev/benchmarks/microbenchmarks/android/AndroidManifest.xml

+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.yourcompany.microbenchmarks"
+    android:versionCode="1"
+    android:versionName="0.0.1">
+
+    <uses-sdk android:minSdkVersion="16" android:targetSdkVersion="21" />
+    <uses-permission android:name="android.permission.INTERNET"/>
+
+    <application android:name="org.domokit.sky.shell.SkyApplication" android:label="microbenchmarks" android:icon="@mipmap/ic_launcher">
+        <activity android:name="org.domokit.sky.shell.SkyActivity"
+                  android:launchMode="singleTop"
+                  android:theme="@android:style/Theme.Black.NoTitleBar"
+                  android:configChanges="orientation|keyboardHidden|keyboard|screenSize|locale|layoutDirection"
+                  android:hardwareAccelerated="true"
+                  android:windowSoftInputMode="adjustResize">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN"/>
+                <category android:name="android.intent.category.LAUNCHER"/>
+            </intent-filter>
+        </activity>
+    </application>
+</manifest>

dev/benchmarks/microbenchmarks/flutter.yaml

+name: microbenchmarks
+uses-material-design: true

dev/benchmarks/microbenchmarks/ios/Assets.xcassets/AppIcon.appiconset/Contents.json

+{
+  "images" : [
+    {
+      "size" : "29x29",
+      "idiom" : "iphone",
+      "filename" : "Icon-Small@2x.png",
+      "scale" : "2x"
+    },
+    {
+      "size" : "29x29",
+      "idiom" : "iphone",
+      "filename" : "Icon-Small@3x.png",
+      "scale" : "3x"
+    },
+    {
+      "size" : "40x40",
+      "idiom" : "iphone",
+      "filename" : "Icon-Small-40@2x.png",
+      "scale" : "2x"
+    },
+    {
+      "size" : "40x40",
+      "idiom" : "iphone",
+      "filename" : "Icon-Small-40@3x.png",
+      "scale" : "3x"
+    },
+    {
+      "size" : "60x60",
+      "idiom" : "iphone",
+      "filename" : "Icon-60@2x.png",
+      "scale" : "2x"
+    },
+    {
+      "size" : "60x60",
+      "idiom" : "iphone",
+      "filename" : "Icon-60@3x.png",
+      "scale" : "3x"
+    },
+    {
+      "size" : "29x29",
+      "idiom" : "ipad",
+      "filename" : "Icon-Small.png",
+      "scale" : "1x"
+    },
+    {
+      "size" : "29x29",
+      "idiom" : "ipad",
+      "filename" : "Icon-Small@2x.png",
+      "scale" : "2x"
+    },
+    {
+      "size" : "40x40",
+      "idiom" : "ipad",
+      "filename" : "Icon-Small-40.png",
+      "scale" : "1x"
+    },
+    {
+      "size" : "40x40",
+      "idiom" : "ipad",
+      "filename" : "Icon-Small-40@2x.png",
+      "scale" : "2x"
+    },
+    {
+      "size" : "76x76",
+      "idiom" : "ipad",
+      "filename" : "Icon-76.png",
+      "scale" : "1x"
+    },
+    {
+      "size" : "76x76",
+      "idiom" : "ipad",
+      "filename" : "Icon-76@2x.png",
+      "scale" : "2x"
+    },
+    {
+      "size" : "83.5x83.5",
+      "idiom" : "ipad",
+      "filename" : "Icon-83.5@2x.png",
+      "scale" : "2x"
+    },
+    {
+      "size" : "16x16",
+      "idiom" : "mac",
+      "filename" : "icon_16x16.png",
+      "scale" : "1x"
+    },
+    {
+      "size" : "16x16",
+      "idiom" : "mac",
+      "filename" : "icon_16x16@2x.png",
+      "scale" : "2x"
+    },
+    {
+      "size" : "32x32",
+      "idiom" : "mac",
+      "filename" : "icon_32x32.png",
+      "scale" : "1x"
+    },
+    {
+      "size" : "32x32",
+      "idiom" : "mac",
+      "filename" : "icon_32x32@2x.png",
+      "scale" : "2x"
+    },
+    {
+      "size" : "128x128",
+      "idiom" : "mac",
+      "filename" : "icon_128x128.png",
+      "scale" : "1x"
+    },
+    {
+      "size" : "128x128",
+      "idiom" : "mac",
+      "filename" : "icon_128x128@2x.png",
+      "scale" : "2x"
+    },
+    {
+      "size" : "256x256",
+      "idiom" : "mac",
+      "filename" : "icon_256x256.png",
+      "scale" : "1x"
+    },
+    {
+      "size" : "256x256",
+      "idiom" : "mac",
+      "filename" : "icon_256x256@2x.png",
+      "scale" : "2x"
+    },
+    {
+      "size" : "512x512",
+      "idiom" : "mac",
+      "filename" : "icon_512x512.png",
+      "scale" : "1x"
+    },
+    {
+      "size" : "512x512",
+      "idiom" : "mac",
+      "filename" : "icon_512x512@2x.png",
+      "scale" : "2x"
+    }
+  ]
+}
\ No newline at end of file

dev/benchmarks/microbenchmarks/ios/Info.plist

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>en</string>
+	<key>CFBundleExecutable</key>
+	<string>Runner</string>
+	<key>CFBundleIdentifier</key>
+	<string>com.yourcompany.microbenchmarks</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>microbenchmarks</string>
+	<key>CFBundlePackageType</key>
+	<string>APPL</string>
+	<key>CFBundleShortVersionString</key>
+	<string>1.0</string>
+	<key>CFBundleSignature</key>
+	<string>????</string>
+	<key>CFBundleVersion</key>
+	<string>1</string>
+	<key>LSRequiresIPhoneOS</key>
+	<true/>
+	<key>UILaunchStoryboardName</key>
+	<string>LaunchScreen</string>
+	<key>UIRequiredDeviceCapabilities</key>
+	<array>
+		<string>arm64</string>
+	</array>
+	<key>UISupportedInterfaceOrientations</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationLandscapeLeft</string>
+		<string>UIInterfaceOrientationLandscapeRight</string>
+	</array>
+	<key>UISupportedInterfaceOrientations~ipad</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
+		<string>UIInterfaceOrientationLandscapeLeft</string>
+		<string>UIInterfaceOrientationLandscapeRight</string>
+	</array>
+	<key>UIViewControllerBasedStatusBarAppearance</key>
+	<false/>
+</dict>
+</plist>

dev/benchmarks/microbenchmarks/ios/LaunchScreen.storyboard

+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="9531" systemVersion="15C50" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" initialViewController="01J-lp-oVM">
+    <dependencies>
+        <deployment identifier="iOS"/>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="9529"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <layoutGuides>
+                        <viewControllerLayoutGuide type="top" id="Llm-lL-Icb"/>
+                        <viewControllerLayoutGuide type="bottom" id="xb3-aO-Qok"/>
+                    </layoutGuides>
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="600" height="600"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <color key="backgroundColor" white="1" alpha="1" colorSpace="custom" customColorSpace="calibratedWhite"/>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="53" y="375"/>
+        </scene>
+    </scenes>
+</document>

packages/flutter/benchmark/gestures/velocity_tracker_data.dart → dev/benchmarks/microbenchmarks/lib/gestures/data/velocity_tracker_data.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
 import 'package:flutter/gestures.dart';
 
 /// Data for velocity_tracker_bench.dart

dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+import 'package:flutter/gestures.dart';
+import 'data/velocity_tracker_data.dart';
+
+const int _kNumIters = 10000;
+
+void main() {
+  final VelocityTracker tracker = new VelocityTracker();
+  final Stopwatch watch = new Stopwatch();
+  print('Velocity tracker benchmark...');
+  watch.start();
+  for (int i = 0; i < _kNumIters; i += 1) {
+    for (PointerEvent event in velocityEventData) {
+      if (event is PointerDownEvent || event is PointerMoveEvent)
+        tracker.addPosition(event.timeStamp, event.position);
+      if (event is PointerUpEvent)
+        tracker.getVelocity();
+    }
+  }
+  watch.stop();
+  print('Velocity tracker: ${(watch.elapsedMicroseconds / _kNumIters).toStringAsFixed(1)}µs per iteration');
+  exit(0);
+}

dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter/rendering.dart';
+import 'package:flutter/scheduler.dart';
+import 'package:stocks/main.dart' as stocks;
+import 'package:stocks/stock_data.dart' as stock_data;
+
+const Duration kBenchmarkTime = const Duration(seconds: 15);
+
+class BenchmarkingBinding extends LiveTestWidgetsFlutterBinding {
+  BenchmarkingBinding(this.stopwatch);
+
+  final Stopwatch stopwatch;
+
+  @override
+  void handleBeginFrame(Duration rawTimeStamp) {
+    stopwatch.start();
+    super.handleBeginFrame(rawTimeStamp);
+    stopwatch.stop();
+  }
+}
+
+Future<Null> main() async {
+  assert(false); // don't run this in checked mode! Use --release.
+  stock_data.StockDataFetcher.actuallyFetchData = false;
+
+  final Stopwatch wallClockWatch = new Stopwatch();
+  final Stopwatch cpuWatch = new Stopwatch();
+  new BenchmarkingBinding(cpuWatch);
+
+  int totalOpenFrameElapsedMicroseconds = 0;
+  int totalOpenIterationCount = 0;
+  int totalCloseFrameElapsedMicroseconds = 0;
+  int totalCloseIterationCount = 0;
+  int totalSubsequentFramesElapsedMicroseconds = 0;
+  int totalSubsequentFramesIterationCount = 0;
+
+  await benchmarkWidgets((WidgetTester tester) async {
+    stocks.main();
+    await tester.pump(); // Start startup animation
+    await tester.pump(const Duration(seconds: 1)); // Complete startup animation
+
+    bool drawerIsOpen = false;
+    wallClockWatch.start();
+    while (wallClockWatch.elapsed < kBenchmarkTime) {
+      cpuWatch.reset();
+      if (drawerIsOpen) {
+        await tester.tapAt(const Point(780.0, 250.0)); // Close drawer
+        await tester.pump();
+        totalCloseIterationCount += 1;
+        totalCloseFrameElapsedMicroseconds += cpuWatch.elapsedMicroseconds;
+      } else {
+        await tester.tapAt(const Point(20.0, 50.0)); // Open drawer
+        await tester.pump();
+        totalOpenIterationCount += 1;
+        totalOpenFrameElapsedMicroseconds += cpuWatch.elapsedMicroseconds;
+      }
+      drawerIsOpen = !drawerIsOpen;
+
+      // Time how long each frame takes
+      cpuWatch.reset();
+      while (SchedulerBinding.instance.hasScheduledFrame) {
+        await tester.pump();
+        totalSubsequentFramesIterationCount += 1;
+      }
+      totalSubsequentFramesElapsedMicroseconds += cpuWatch.elapsedMicroseconds;
+    }
+  });
+
+  print('Stock animation (ran for ${(wallClockWatch.elapsedMicroseconds / (1000 * 1000)).toStringAsFixed(1)}s):');
+  print('  Opening first frame average time: ${(totalOpenFrameElapsedMicroseconds / (totalOpenIterationCount)).toStringAsFixed(1)}µs per frame ($totalOpenIterationCount frames)');
+  print('  Closing first frame average time: ${(totalCloseFrameElapsedMicroseconds / (totalCloseIterationCount)).toStringAsFixed(1)}µs per frame ($totalCloseIterationCount frames)');
+  print('  Subsequent frames average time: ${(totalSubsequentFramesElapsedMicroseconds / (totalSubsequentFramesIterationCount)).toStringAsFixed(1)}µs per frame ($totalSubsequentFramesIterationCount frames)');
+
+  exit(0);
+}

dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart

+import 'dart:io';
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter/rendering.dart';
+import 'package:stocks/main.dart' as stocks;
+import 'package:stocks/stock_data.dart' as stock_data;
+
+const Duration kBenchmarkTime = const Duration(seconds: 15);
+
+void _doNothing() { }
+
+Future<Null> main() async {
+  assert(false); // don't run this in checked mode! Use --release.
+  stock_data.StockDataFetcher.actuallyFetchData = false;
+
+  final Stopwatch watch = new Stopwatch();
+  int iterations = 0;
+
+  await benchmarkWidgets((WidgetTester tester) async {
+    stocks.main();
+    await tester.pump(); // Start startup animation
+    await tester.pump(const Duration(seconds: 1)); // Complete startup animation
+    await tester.tapAt(new Point(20.0, 20.0)); // Open drawer
+    await tester.pump(); // Start drawer animation
+    await tester.pump(const Duration(seconds: 1)); // Complete drawer animation
+
+
+    final stocks.StocksAppState appState = tester.state(find.byType(stocks.StocksApp));
+    final BuildOwner buildOwner = WidgetsBinding.instance.buildOwner;
+
+    watch.start();
+    while (watch.elapsed < kBenchmarkTime) {
+      appState.setState(_doNothing);
+      buildOwner.buildDirtyElements();
+      iterations += 1;
+    }
+    watch.stop();
+  });
+
+  print('Stock build: ${(watch.elapsedMicroseconds / iterations).toStringAsFixed(1)}µs per iteration');
+  exit(0);
+}

dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart

+import 'dart:io';
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter/rendering.dart';
+import 'package:stocks/main.dart' as stocks;
+import 'package:stocks/stock_data.dart' as stock_data;
+
+const Duration kBenchmarkTime = const Duration(seconds: 15);
+
+
+
+Future<Null> main() async {
+  assert(false); // don't run this in checked mode! Use --release.
+  stock_data.StockDataFetcher.actuallyFetchData = false;
+
+  final Stopwatch watch = new Stopwatch();
+  int iterations = 0;
+
+  await benchmarkWidgets((WidgetTester tester) async {
+    stocks.main();
+    await tester.pump(); // Start startup animation
+    await tester.pump(const Duration(seconds: 1)); // Complete startup animation
+    await tester.tapAt(new Point(20.0, 20.0)); // Open drawer
+    await tester.pump(); // Start drawer animation
+    await tester.pump(const Duration(seconds: 1)); // Complete drawer animation
+
+    final TestViewConfiguration big = new TestViewConfiguration(size: const Size(360.0, 640.0));
+    final TestViewConfiguration small = new TestViewConfiguration(size: const Size(355.0, 635.0));
+    final RenderView renderView = WidgetsBinding.instance.renderView;
+
+    watch.start();
+    while (watch.elapsed < kBenchmarkTime) {
+      renderView.configuration = (iterations % 2 == 0) ? big : small;
+      RendererBinding.instance.pipelineOwner.flushLayout();
+      iterations += 1;
+    }
+    watch.stop();
+  });
+
+  print('Stock layout: ${(watch.elapsedMicroseconds / iterations).toStringAsFixed(1)}µs per iteration');
+  exit(0);
+}

dev/benchmarks/microbenchmarks/pubspec.yaml

+name: microbenchmarks
+description: A new flutter project.
+dependencies:
+  flutter:
+    path: ../../../packages/flutter
+  flutter_test:
+    path: ../../../packages/flutter_test
+  stocks:
+    path: ../../../examples/stocks

packages/flutter/benchmark/README.md

-# Benchmarks
-
-This directory (and its sub-directories) contain benchmarks for
-Flutter. The reporting format for benchmarks is not standardized yet,
-so benchmarks here are typically run by hand. To run a benchmark:
-
-1. Build `sky_shell` for Linux Release using the instructions in the
-   [Engine repository](https://github.com/flutter/engine).
-
-2. Run `pub get` in the `packages/flutter` directory.
-
-3. Run the benchmarks by running the following command from the root
-   of the flutter repository. Replace `stocks/layout_bench.dart` with
-   the path to whichever benchmark you want to run. If you didn't
-   build the engine in the recommended place, then also update the
-   path accordingly. If you made changes to sky_services, you'll also
-   need to update the `pubspec.yaml` file to point to that using a
-   dependency_override.
-
-```
-../engine/src/out/Release/sky_shell packages/flutter/benchmark/stocks/layout_bench.dart --package-root=packages/flutter/packages
-```

packages/flutter/benchmark/gestures/velocity_tracker_bench.dart

-// Copyright 2015 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'package:flutter/gestures.dart';
-import 'package:test/test.dart';
-import 'velocity_tracker_data.dart';
-
-const int _kNumIters = 10000;
-
-void main() {
-  test('Dart velocity tracker performance', () {
-    VelocityTracker tracker = new VelocityTracker();
-    Stopwatch watch = new Stopwatch();
-    watch.start();
-    for (int i = 0; i < _kNumIters; i++) {
-      for (PointerEvent event in velocityEventData) {
-        if (event is PointerDownEvent || event is PointerMoveEvent)
-          tracker.addPosition(event.timeStamp, event.position);
-        if (event is PointerUpEvent)
-          tracker.getVelocity();
-      }
-    }
-    watch.stop();
-    print("Dart tracker: " + watch.elapsed.toString());
-  });
-}

packages/flutter/benchmark/stocks/animation_bench.dart

-import 'dart:io';
-
-import 'package:flutter_test/flutter_test.dart';
-import 'package:flutter/material.dart';
-import 'package:flutter/rendering.dart';
-import 'package:stocks/main.dart' as stocks;
-import 'package:stocks/stock_data.dart' as stock_data;
-
-const int _kNumberOfIterations = 50000;
-const bool _kRunForever = false;
-
-void main() {
-  stock_data.StockDataFetcher.actuallyFetchData = false;
-
-  const Duration _kAnimationDuration = const Duration(milliseconds: 200);
-  const Duration tickDuration = const Duration(milliseconds: 2);
-  int numberOfTicks = _kAnimationDuration.inMicroseconds ~/ tickDuration.inMicroseconds;
-  int numberOfRounts = _kNumberOfIterations ~/ numberOfTicks;
-
-  Stopwatch watch = new Stopwatch()
-    ..start();
-
-  benchmarkWidgets((WidgetTester tester) {
-    stocks.main();
-    tester.pump(); // Start startup animation
-    tester.pump(const Duration(seconds: 1)); // Complete startup animation
-
-    bool drawerIsOpen = false;
-
-    for (int i = 0; i < numberOfRounts || _kRunForever; ++i) {
-      if (drawerIsOpen)
-        tester.tapAt(const Point(20.0, 20.0)); // Open drawer
-      else
-        tester.tapAt(const Point(780.0, 20.0)); // Close drawer
-
-      tester.pump(); // Start drawer animation
-
-      for (int j = 0; j < numberOfTicks; ++j)
-        tester.pump(tickDuration);
-
-      tester.pump(const Duration(seconds: 1)); // Complete animation
-      drawerIsOpen = !drawerIsOpen;
-    }
-  });
-
-  watch.stop();
-  print("Stock animation: " + watch.elapsed.toString());
-
-  exit(0);
-}

packages/flutter/benchmark/stocks/build_bench.dart

-import 'dart:io';
-
-import 'package:flutter_test/flutter_test.dart';
-import 'package:flutter/material.dart';
-import 'package:flutter/rendering.dart';
-import 'package:stocks/main.dart' as stocks;
-import 'package:stocks/stock_data.dart' as stock_data;
-
-const int _kNumberOfIterations = 100000;
-const bool _kRunForever = false;
-
-void _doNothing() { }
-
-void main() {
-  stock_data.StockDataFetcher.actuallyFetchData = false;
-
-  stocks.StocksAppState appState;
-
-  benchmarkWidgets((WidgetTester tester) {
-    stocks.main();
-    tester.pump(); // Start startup animation
-    tester.pump(const Duration(seconds: 1)); // Complete startup animation
-    tester.tapAt(new Point(20.0, 20.0)); // Open drawer
-    tester.pump(); // Start drawer animation
-    tester.pump(const Duration(seconds: 1)); // Complete drawer animation
-    appState = tester.state(find.byType(stocks.StocksApp));
-  });
-
-  BuildOwner buildOwner = WidgetsBinding.instance.buildOwner;
-
-  Stopwatch watch = new Stopwatch()
-    ..start();
-
-  for (int i = 0; i < _kNumberOfIterations || _kRunForever; ++i) {
-    appState.setState(_doNothing);
-    buildOwner.buildDirtyElements();
-  }
-
-  watch.stop();
-  print("Stock build: " + watch.elapsed.toString());
-  exit(0);
-}

packages/flutter/benchmark/stocks/layout_bench.dart

-import 'dart:io';
-
-import 'package:flutter_test/flutter_test.dart';
-import 'package:flutter/material.dart';
-import 'package:flutter/rendering.dart';
-import 'package:stocks/main.dart' as stocks;
-import 'package:stocks/stock_data.dart' as stock_data;
-
-const int _kNumberOfIterations = 100000;
-const bool _kRunForever = false;
-
-void main() {
-  stock_data.StockDataFetcher.actuallyFetchData = false;
-
-  benchmarkWidgets((WidgetTester tester) {
-    stocks.main();
-    tester.pump(); // Start startup animation
-    tester.pump(const Duration(seconds: 1)); // Complete startup animation
-    tester.tapAt(new Point(20.0, 20.0)); // Open drawer
-    tester.pump(); // Start drawer animation
-    tester.pump(const Duration(seconds: 1)); // Complete drawer animation
-  });
-
-  ViewConfiguration big = const ViewConfiguration(size: const Size(360.0, 640.0));
-  ViewConfiguration small = const ViewConfiguration(size: const Size(355.0, 635.0));
-  RenderView renderView = WidgetsBinding.instance.renderView;
-
-  Stopwatch watch = new Stopwatch()
-    ..start();
-
-  for (int i = 0; i < _kNumberOfIterations || _kRunForever; ++i) {
-    renderView.configuration = (i % 2 == 0) ? big : small;
-    RendererBinding.instance.pipelineOwner.flushLayout();
-  }
-
-  watch.stop();
-  print("Stock layout: " + watch.elapsed.toString());
-  exit(0);
-}

packages/flutter/pubspec.yaml

@@ -22,8 +22,6 @@ dependencies:
 dev_dependencies:
   flutter_test:
     path: ../flutter_test
-  stocks: # for the benchmarks
-    path: ../../examples/stocks
 
 environment:
   sdk: '>=1.12.0 <2.0.0'

packages/flutter_test/lib/src/binding.dart

@@ -67,7 +67,7 @@ enum TestBindingEventSource {
   device,
 }
 
-const Size _kTestViewportSize = const Size(800.0, 600.0);
+const Size _kDefaultTestViewportSize = const Size(800.0, 600.0);
 
 /// Base class for bindings used by widgets library tests.
 ///
@@ -94,9 +94,9 @@ abstract class TestWidgetsFlutterBinding extends BindingBase
   static WidgetsBinding ensureInitialized() {
     if (WidgetsBinding.instance == null) {
       if (Platform.environment.containsKey('FLUTTER_TEST')) {
-        new AutomatedTestWidgetsFlutterBinding._();
+        new AutomatedTestWidgetsFlutterBinding();
       } else {
-        new LiveTestWidgetsFlutterBinding._();
+        new LiveTestWidgetsFlutterBinding();
       }
     }
     assert(WidgetsBinding.instance is TestWidgetsFlutterBinding);
@@ -394,8 +394,6 @@ abstract class TestWidgetsFlutterBinding extends BindingBase
 /// This binding controls time, allowing tests to verify long
 /// animation sequences without having to execute them in real time.
 class AutomatedTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
-  AutomatedTestWidgetsFlutterBinding._();
-
   @override
   void initInstances() {
     debugPrint = debugPrintSynchronously;
@@ -542,8 +540,6 @@ class AutomatedTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
 /// doesn't trigger a paint, since then you could not see anything
 /// anyway.)
 class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
-  LiveTestWidgetsFlutterBinding._();
-
   @override
   bool get inTest => _inTest;
   bool _inTest = false;
@@ -676,11 +672,7 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
 
   @override
   ViewConfiguration createViewConfiguration() {
-    return new _TestViewConfiguration(
-      // TODO(ianh): that these are not the same is https://github.com/flutter/flutter/issues/1360
-      _getMatrix(ui.window.devicePixelRatio),
-      _getMatrix(1.0)
-    );
+    return new TestViewConfiguration();
   }
 
   @override
@@ -697,12 +689,23 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
     Matrix4 transform = renderView.configuration.toHitTestMatrix();
     return MatrixUtils.transformPoint(transform, point);
   }
+}
 
-  Matrix4 _getMatrix(double devicePixelRatio) {
+/// A [ViewConfiguration] that pretends the display is of a particular size. The
+/// size is in logical pixels. The resulting ViewConfiguration maps the given
+/// size onto the actual display using the [ImageFit.contain] algorithm.
+class TestViewConfiguration extends ViewConfiguration {
+  /// Creates a [TestViewConfiguration] with the given size. Defaults to 800x600.
+  TestViewConfiguration({ Size size: _kDefaultTestViewportSize })
+    : _paintMatrix = _getMatrix(size, ui.window.devicePixelRatio),
+      _hitTestMatrix = _getMatrix(size, 1.0),
+      super(size: size);
+
+  static Matrix4 _getMatrix(Size size, double devicePixelRatio) {
     final double actualWidth = ui.window.size.width * devicePixelRatio;
     final double actualHeight = ui.window.size.height * devicePixelRatio;
-    final double desiredWidth = _kTestViewportSize.width;
-    final double desiredHeight = _kTestViewportSize.height;
+    final double desiredWidth = size.width;
+    final double desiredHeight = size.height;
     double scale, shiftX, shiftY;
     if ((actualWidth / actualHeight) > (desiredWidth / desiredHeight)) {
       scale = actualHeight / desiredHeight;
@@ -720,18 +723,24 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
     );
     return matrix;
   }
-}
 
-class _TestViewConfiguration extends ViewConfiguration {
-  _TestViewConfiguration(this.paintMatrix, this.hitTestMatrix) : super(size: _kTestViewportSize);
-
-  final Matrix4 paintMatrix;
-  final Matrix4 hitTestMatrix;
+  final Matrix4 _paintMatrix;
+  final Matrix4 _hitTestMatrix;
 
   @override
-  Matrix4 toMatrix() => paintMatrix.clone();
+  Matrix4 toMatrix() => _paintMatrix.clone();
 
-  Matrix4 toHitTestMatrix() => hitTestMatrix.clone();
+  /// Provides the transformation matrix that converts coordinates in the test
+  /// coordinate space to coordinates in logical pixels on the real display.
+  ///
+  /// This is essenitally the same as [toMatrix] but ignoring the device pixel
+  /// ratio.
+  ///
+  /// This is useful because pointers are described in logical pixels, as
+  /// opposed to graphics which are expressed in physical pixels.
+  // TODO(ianh): We should make graphics and pointers use the same coordinate space.
+  //             See: https://github.com/flutter/flutter/issues/1360
+  Matrix4 toHitTestMatrix() => _hitTestMatrix.clone();
 
   @override
   String toString() => 'TestViewConfiguration';
@@ -758,7 +767,9 @@ class _LiveTestRenderView extends RenderView {
   }) : super(configuration: configuration);
 
   @override
-  _TestViewConfiguration get configuration => super.configuration;
+  TestViewConfiguration get configuration => super.configuration;
+  @override
+  set configuration(TestViewConfiguration value) { super.configuration = value; }
 
   final Map<int, _LiveTestPointerRecord> _pointers = <int, _LiveTestPointerRecord>{};
 

packages/flutter_test/lib/src/widget_tester.dart

@@ -65,14 +65,18 @@ void testWidgets(String description, WidgetTesterCallback callback, {
 /// The callback can be asynchronous (using `async`/`await` or using
 /// explicit [Future]s). If it is, then [benchmarkWidgets] will return
 /// a [Future] that completes when the callback's does. Otherwise, it
-/// will return a widget that is always complete.
+/// will return a Future that is always complete.
+///
+/// If the callback is asynchronous, make sure you `await` the call
+/// to [benchmarkWidgets], otherwise it won't run!
 ///
 /// Benchmarks must not be run in checked mode. To avoid this, this
-/// function will assert if it is run in checked mode.
+/// function will print a big message if it is run in checked mode.
 ///
 /// Example:
 ///
 ///     main() async {
+///       assert(false); // fail in checked mode
 ///       await benchmarkWidgets((WidgetTester tester) {
 ///         tester.pumpWidget(new MyWidget());
 ///         final Stopwatch timer = new Stopwatch()..start();
@@ -86,7 +90,20 @@ void testWidgets(String description, WidgetTesterCallback callback, {
 ///       exit(0);
 ///     }
 Future<Null> benchmarkWidgets(WidgetTesterCallback callback) {
-  assert(false); // Don't run benchmarks in checked mode.
+  assert(() {
+    print('┏╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍┓');
+    print('┇ ⚠ THIS BENCHMARK IS BEING RUN WITH ASSERTS ENABLED ⚠  ┇');
+    print('┡╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍┦');
+    print('│                                                       │');
+    print('│  Numbers obtained from a benchmark while asserts are  │');
+    print('│  enabled will not accurately reflect the performance  │');
+    print('│  that will be experienced by end users using release  ╎');
+    print('│  builds. Benchmarks should be run using this command  ┆');
+    print('│  line:  flutter run --release -t benchmark.dart       ┊');
+    print('│                                                        ');
+    print('└─────────────────────────────────────────────────╌┄┈  🐢');
+    return true;
+  });
   TestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized();
   assert(binding is! AutomatedTestWidgetsFlutterBinding);
   WidgetTester tester = new WidgetTester._(binding);