Test performance of "flutter test" (#18810)

dev/devicelab/bin/tasks/flutter_test_performance.dart

+// Copyright (c) 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This test runs `flutter test` on the `trivial_widget_test.dart` four times.
+//
+// The first time, the result is ignored, on the basis that it's warming the
+// cache.
+//
+// The second time tests how long a regular test takes to run.
+//
+// Before the third time, a change is made to the implementation of one of the
+// files that the test depends on (indirectly).
+//
+// Before the fourth time, a change is made to the interface in that same file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:path/path.dart' as path;
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+
+// Matches the output of package:test, e.g.: "00:01 +1 loading foo"
+final RegExp testOutputPattern = new RegExp(r'^[0-9][0-9]:[0-9][0-9] \+[0-9]+: (.+?) *$');
+
+enum TestStep {
+  starting,
+  buildingFlutterTool,
+  runningPubGet,
+  testWritesFirstCarriageReturn,
+  testLoading,
+  testRunning,
+  testPassed,
+}
+
+Future<int> runTest() async {
+  final Stopwatch clock = new Stopwatch()..start();
+  final Process analysis = await startProcess(
+    path.join(flutterDirectory.path, 'bin', 'flutter'),
+    <String>['test', path.join('flutter_test', 'trivial_widget_test.dart')],
+    workingDirectory: path.join(flutterDirectory.path, 'dev', 'automated_tests'),
+  );
+  int badLines = 0;
+  TestStep step = TestStep.starting;
+  await for (String entry in analysis.stdout.transform(utf8.decoder).transform(const LineSplitter())) {
+    print('test stdout ($step): $entry');
+    if (step == TestStep.starting && entry == 'Building flutter tool...') {
+      // ignore this line
+      step = TestStep.buildingFlutterTool;
+    } else if (step.index < TestStep.runningPubGet.index && entry == 'Running "flutter packages get" in automated_tests...') {
+      // ignore this line
+      step = TestStep.runningPubGet;
+    } else if (step.index < TestStep.testWritesFirstCarriageReturn.index && entry == '') {
+      // we have a blank line at the start
+      step = TestStep.testWritesFirstCarriageReturn;
+    } else {
+      final Match match = testOutputPattern.matchAsPrefix(entry);
+      if (match == null) {
+        badLines += 1;
+      } else {
+        if (step.index >= TestStep.testWritesFirstCarriageReturn.index && step.index <= TestStep.testLoading.index && match.group(1).startsWith('loading ')) {
+          // first the test loads
+          step = TestStep.testLoading;
+        } else if (step.index <= TestStep.testRunning.index && match.group(1) == 'A trivial widget test') {
+          // then the test runs
+          step = TestStep.testRunning;
+        } else if (step.index < TestStep.testPassed.index && match.group(1) == 'All tests passed!') {
+          // then the test finishes
+          step = TestStep.testPassed;
+        } else {
+          badLines += 1;
+        }
+      }
+    }
+  }
+  await for (String entry in analysis.stderr.transform(utf8.decoder).transform(const LineSplitter())) {
+    print('test stderr: $entry');
+    badLines += 1;
+  }
+  final int result = await analysis.exitCode;
+  clock.stop();
+  if (result != 0)
+    throw new Exception('flutter test failed with exit code $result');
+  if (badLines > 0)
+    throw new Exception('flutter test renderered unexpected output ($badLines bad lines)');
+  if (step != TestStep.testPassed)
+    throw new Exception('flutter test did not finish (only reached step $step)');
+  print('elapsed time: ${clock.elapsedMilliseconds}ms');
+  return clock.elapsedMilliseconds;
+}
+
+void main() {
+  task(() async {
+    final File nodeSourceFile = new File(path.join(
+      flutterDirectory.path, 'packages', 'flutter', 'lib', 'src', 'foundation', 'node.dart',
+    ));
+    final String originalSource = await nodeSourceFile.readAsString();
+    try {
+      await runTest(); // first number is meaningless; could have had to build the tool, run pub get, have a cache, etc
+      final int withoutChange = await runTest(); // run test again with no change
+      await nodeSourceFile.writeAsString( // only change implementation
+        originalSource
+          .replaceAll('_owner', '_xyzzy')
+      );
+      final int implementationChange = await runTest(); // run test again with implementation changed
+      await nodeSourceFile.writeAsString( // change interface as well
+        originalSource
+          .replaceAll('_owner', '_xyzzy')
+          .replaceAll('owner', '_owner')
+          .replaceAll('_xyzzy', 'owner')
+      );
+      final int interfaceChange = await runTest(); // run test again with interface changed
+      final Map<String, dynamic> data = <String, dynamic>{
+        'without_change_elapsed_time_ms': withoutChange,
+        'implementation_change_elapsed_time_ms': implementationChange,
+        'interface_change_elapsed_time_ms': interfaceChange,
+      };
+      return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
+    } finally {
+      await nodeSourceFile.writeAsString(originalSource);
+    }
+  });
+}

dev/devicelab/manifest.yaml

@@ -425,6 +425,13 @@ tasks:
     stage: devicelab
     required_agent_capabilities: ["linux/android"]
 
+  flutter_test_performance:
+    description: >
+      Measures performance of running flutter test.
+    stage: devicelab
+    required_agent_capabilities: ["linux/android"]
+    flaky: true
+
   technical_debt__cost:
     description: >
       Estimates our technical debt (TODOs, analyzer ignores, etc).

packages/flutter/lib/src/foundation/node.dart

@@ -4,6 +4,10 @@
 
 import 'package:meta/meta.dart';
 
+// This file gets mutated by //dev/devicelab/bin/tasks/flutter_test_performance.dart
+// during device lab performance tests. When editing this file, check to make sure
+// that it didn't break that test.
+
 /// An abstract node in a tree.
 ///
 /// AbstractNode has as notion of depth, attachment, and parent, but does not
@@ -142,5 +146,4 @@ class AbstractNode {
     if (attached)
       child.detach();
   }
-
 }