flutter_test_performance.dart 5.91 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// This test runs `flutter test` on the `trivial_widget_test.dart` four times.
//
// The first time, the result is ignored, on the basis that it's warming the
// cache.
//
// The second time tests how long a regular test takes to run.
//
// Before the third time, a change is made to the implementation of one of the
// files that the test depends on (indirectly).
//
// Before the fourth time, a change is made to the interface in that same file.

import 'dart:async';
import 'dart:convert';
import 'dart:io';

import 'package:flutter_devicelab/framework/framework.dart';
22
import 'package:flutter_devicelab/framework/task_result.dart';
23
import 'package:flutter_devicelab/framework/utils.dart';
24
import 'package:path/path.dart' as path;
25

26
// Matches the output of the "test" package, e.g.: "00:01 +1 loading foo"
27
final RegExp testOutputPattern = RegExp(r'^[0-9][0-9]:[0-9][0-9] \+[0-9]+: (.+?) *$');
28 29 30 31 32 33 34 35 36 37 38

enum TestStep {
  starting,
  buildingFlutterTool,
  runningPubGet,
  testWritesFirstCarriageReturn,
  testLoading,
  testRunning,
  testPassed,
}

39
Future<int> runTest({bool coverage = false, bool noPub = false}) async {
40
  final Stopwatch clock = Stopwatch()..start();
41 42
  final List<String> arguments = flutterCommandArgs('test', <String>[
    if (coverage) '--coverage',
43
    if (noPub) '--no-pub',
44 45
    path.join('flutter_test', 'trivial_widget_test.dart'),
  ]);
46 47
  final Process analysis = await startProcess(
    path.join(flutterDirectory.path, 'bin', 'flutter'),
48
    arguments,
49 50 51 52
    workingDirectory: path.join(flutterDirectory.path, 'dev', 'automated_tests'),
  );
  int badLines = 0;
  TestStep step = TestStep.starting;
53 54

  analysis.stdout.transform<String>(utf8.decoder).transform<String>(const LineSplitter()).listen((String entry) {
55 56 57 58
    print('test stdout ($step): $entry');
    if (step == TestStep.starting && entry == 'Building flutter tool...') {
      // ignore this line
      step = TestStep.buildingFlutterTool;
59 60
    } else if (step == TestStep.testPassed && entry.contains('Collecting coverage information...')) {
      // ignore this line
61
    } else if (step.index < TestStep.runningPubGet.index && entry == 'Running "flutter pub get" in automated_tests...') {
62 63
      // ignore this line
      step = TestStep.runningPubGet;
64
    } else if (step.index <= TestStep.testWritesFirstCarriageReturn.index && entry.trim() == '') {
65 66 67
      // we have a blank line at the start
      step = TestStep.testWritesFirstCarriageReturn;
    } else {
68
      final Match? match = testOutputPattern.matchAsPrefix(entry);
69 70 71
      if (match == null) {
        badLines += 1;
      } else {
72
        if (step.index >= TestStep.testWritesFirstCarriageReturn.index && step.index <= TestStep.testLoading.index && match.group(1)!.startsWith('loading ')) {
73 74 75 76 77 78 79 80 81 82 83 84 85
          // first the test loads
          step = TestStep.testLoading;
        } else if (step.index <= TestStep.testRunning.index && match.group(1) == 'A trivial widget test') {
          // then the test runs
          step = TestStep.testRunning;
        } else if (step.index < TestStep.testPassed.index && match.group(1) == 'All tests passed!') {
          // then the test finishes
          step = TestStep.testPassed;
        } else {
          badLines += 1;
        }
      }
    }
86 87
  });
  analysis.stderr.transform<String>(utf8.decoder).transform<String>(const LineSplitter()).listen((String entry) {
88 89
    print('test stderr: $entry');
    badLines += 1;
90
  });
91 92 93
  final int result = await analysis.exitCode;
  clock.stop();
  if (result != 0)
94
    throw Exception('flutter test failed with exit code $result');
95
  if (badLines > 0)
Chris Bracken's avatar
Chris Bracken committed
96
    throw Exception('flutter test rendered unexpected output ($badLines bad lines)');
97
  if (step != TestStep.testPassed)
98
    throw Exception('flutter test did not finish (only reached step $step)');
99 100 101 102
  print('elapsed time: ${clock.elapsedMilliseconds}ms');
  return clock.elapsedMilliseconds;
}

103 104 105 106 107 108 109 110
Future<void> pubGetDependencies(List<Directory> directories) async {
  for (final Directory directory in directories) {
    await inDirectory<void>(directory, () async {
      await flutter('pub', options: <String>['get']);
    });
  }
}

111 112
void main() {
  task(() async {
113
    final File nodeSourceFile = File(path.join(
114 115
      flutterDirectory.path, 'packages', 'flutter', 'lib', 'src', 'foundation', 'node.dart',
    ));
116
    await pubGetDependencies(<Directory>[Directory(path.join(flutterDirectory.path, 'dev', 'automated_tests')),]);
117 118
    final String originalSource = await nodeSourceFile.readAsString();
    try {
119
      await runTest(noPub: true); // first number is meaningless; could have had to build the tool, run pub get, have a cache, etc
120
      final int withoutChange = await runTest(noPub: true); // run test again with no change
121 122 123 124
      await nodeSourceFile.writeAsString( // only change implementation
        originalSource
          .replaceAll('_owner', '_xyzzy')
      );
125
      final int implementationChange = await runTest(noPub: true); // run test again with implementation changed
126 127 128 129 130 131
      await nodeSourceFile.writeAsString( // change interface as well
        originalSource
          .replaceAll('_owner', '_xyzzy')
          .replaceAll('owner', '_owner')
          .replaceAll('_xyzzy', 'owner')
      );
132
      final int interfaceChange = await runTest(noPub: true); // run test again with interface changed
133
      // run test with coverage enabled.
134
      final int withCoverage = await runTest(coverage: true, noPub: true);
135 136 137 138
      final Map<String, dynamic> data = <String, dynamic>{
        'without_change_elapsed_time_ms': withoutChange,
        'implementation_change_elapsed_time_ms': implementationChange,
        'interface_change_elapsed_time_ms': interfaceChange,
139
        'with_coverage_time_ms': withCoverage,
140
      };
141
      return TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
142 143 144 145 146
    } finally {
      await nodeSourceFile.writeAsString(originalSource);
    }
  });
}