overall_experience_test.dart 18 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// The purpose of this test is to verify the end-to-end behavior of
// "flutter run" and other such commands, as closely as possible to
// the default behavior. To that end, it avoids the use of any test
// features that are not critical (-dflutter-test being the primary
// example of a test feature that it does use). For example, no use
// is made of "--machine" in these tests.

// There are a number of risks when it comes to writing a test such
// as this one. Typically these tests are hard to debug if they are
// in a failing condition, because they just hang as they await the
// next expected line that never comes. To avoid this, here we have
// the policy of looking for multiple lines, printing what expected
// lines were not seen when a short timeout expires (but timing out
// does not cause the test to fail, to reduce flakes), and wherever
// possible recording all output and comparing the actual output to
// the expected output only once the test is completed.

// To aid in debugging, consider passing the `debug: true` argument
// to the runFlutter function.

25
// This file intentionally assumes the tests run in order.
26
@Tags(<String>['no-shuffle'])
27
library;
28

29 30 31
import 'dart:io';

import '../src/common.dart';
32
import 'test_utils.dart' show fileSystem;
33
import 'transition_test_utils.dart';
34

35 36 37 38 39
void main() {
  testWithoutContext('flutter run writes and clears pidfile appropriately', () async {
    final String tempDirectory = fileSystem.systemTempDirectory.createTempSync('flutter_overall_experience_test.').resolveSymbolicLinksSync();
    final String pidFile = fileSystem.path.join(tempDirectory, 'flutter.pid');
    final String testDirectory = fileSystem.path.join(flutterRoot, 'examples', 'hello_world');
40
    bool? existsDuringTest;
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67
    try {
      expect(fileSystem.file(pidFile).existsSync(), isFalse);
      final ProcessTestResult result = await runFlutter(
        <String>['run', '-dflutter-tester', '--pid-file', pidFile],
        testDirectory,
        <Transition>[
          Barrier('q Quit (terminate the application on the device).', handler: (String line) {
            existsDuringTest = fileSystem.file(pidFile).existsSync();
            return 'q';
          }),
          const Barrier('Application finished.'),
        ],
      );
      expect(existsDuringTest, isNot(isNull));
      expect(existsDuringTest, isTrue);
      expect(result.exitCode, 0, reason: 'subprocess failed; $result');
      expect(fileSystem.file(pidFile).existsSync(), isFalse);
      // This first test ignores the stdout and stderr, so that if the
      // first run outputs "building flutter", or the "there's a new
      // flutter" banner, or other such first-run messages, they won't
      // fail the tests. This does mean that running this test first is
      // actually important in the case where you're running the tests
      // manually. (On CI, all those messages are expected to be seen
      // long before we get here, e.g. because we run "flutter doctor".)
    } finally {
      tryToDelete(fileSystem.directory(tempDirectory));
    }
68
  }, skip: Platform.isWindows); // [intended] Windows doesn't support sending signals so we don't care if it can store the PID.
69 70

  testWithoutContext('flutter run handle SIGUSR1/2 run', () async {
71 72 73 74
    final String tempDirectory = fileSystem.systemTempDirectory.createTempSync('flutter_overall_experience_test.').resolveSymbolicLinksSync();
    final String pidFile = fileSystem.path.join(tempDirectory, 'flutter.pid');
    final String testDirectory = fileSystem.path.join(flutterRoot, 'dev', 'integration_tests', 'ui');
    final String testScript = fileSystem.path.join('lib', 'commands.dart');
75
    late int pid;
76 77 78 79 80 81 82 83 84
    final List<String> command = <String>[
      'run',
      '-dflutter-tester',
      '--report-ready',
      '--pid-file',
      pidFile,
      '--no-devtools',
      testScript,
    ];
85 86
    try {
      final ProcessTestResult result = await runFlutter(
87
        command,
88 89
        testDirectory,
        <Transition>[
90
          Multiple(<Pattern>['Flutter run key commands.', 'called paint'], handler: (String line) {
91 92 93 94
            pid = int.parse(fileSystem.file(pidFile).readAsStringSync());
            processManager.killPid(pid, ProcessSignal.sigusr1);
            return null;
          }),
95
          Barrier('Performing hot reload...'.padRight(progressMessageWidth), logging: true),
96
          Multiple(<Pattern>[RegExp(r'^Reloaded 0 libraries in [0-9]+ms \(compile: \d+ ms, reload: \d+ ms, reassemble: \d+ ms\)\.$'), 'called reassemble', 'called paint'], handler: (String line) {
97 98 99
            processManager.killPid(pid, ProcessSignal.sigusr2);
            return null;
          }),
100
          Barrier('Performing hot restart...'.padRight(progressMessageWidth)),
101 102
          // This could look like 'Restarted application in 1,237ms.'
          Multiple(<Pattern>[RegExp(r'^Restarted application in .+m?s.$'), 'called main', 'called paint'], handler: (String line) {
103 104 105 106 107 108 109 110 111
            return 'q';
          }),
          const Barrier('Application finished.'),
        ],
        logging: false, // we ignore leading log lines to avoid making this test sensitive to e.g. the help message text
      );
      // We check the output from the app (all starts with "called ...") and the output from the tool
      // (everything else) separately, because their relative timing isn't guaranteed. Their rough timing
      // is verified by the expected transitions above.
112
      expect(result.stdout.where((String line) => line.startsWith('called ')), <Object>[
113 114
        // logs start after we receive the response to sending SIGUSR1
        // SIGUSR1:
115
        'called reassemble',
116 117
        'called paint',
        // SIGUSR2:
118
        'called main',
119 120
        'called paint',
      ]);
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
      expect(
        result.stdout.where((String line) => !line.startsWith('called ')),
        <Object>[
          // logs start after we receive the response to sending SIGUSR1
          'Performing hot reload...'.padRight(progressMessageWidth),
          startsWith('Reloaded 0 libraries in '),
          'Performing hot restart...'.padRight(progressMessageWidth),
          startsWith('Restarted application in '),
          '', // this newline is the one for after we hit "q"
          'Application finished.',
          'ready',
        ],
        reason: 'stdout from command ${command.join(' ')} was unexpected, '
          'full Stdout:\n\n${result.stdout.join('\n')}\n\n'
          'Stderr:\n\n${result.stderr.join('\n')}',
      );
137 138 139 140
      expect(result.exitCode, 0);
    } finally {
      tryToDelete(fileSystem.directory(tempDirectory));
    }
141
  }, skip: Platform.isWindows); // [intended] Windows doesn't support sending signals.
142 143 144 145 146

  testWithoutContext('flutter run can hot reload and hot restart, handle "p" key', () async {
    final String tempDirectory = fileSystem.systemTempDirectory.createTempSync('flutter_overall_experience_test.').resolveSymbolicLinksSync();
    final String testDirectory = fileSystem.path.join(flutterRoot, 'dev', 'integration_tests', 'ui');
    final String testScript = fileSystem.path.join('lib', 'commands.dart');
147 148 149 150 151 152 153
    final List<String> command = <String>[
      'run',
      '-dflutter-tester',
      '--report-ready',
      '--no-devtools',
      testScript,
    ];
154 155
    try {
      final ProcessTestResult result = await runFlutter(
156
        command,
157 158
        testDirectory,
        <Transition>[
159
          Multiple(<Pattern>['Flutter run key commands.', 'called main', 'called paint'], handler: (String line) {
160 161
            return 'r';
          }),
162 163
          Barrier('Performing hot reload...'.padRight(progressMessageWidth), logging: true),
          Multiple(<Pattern>['ready', 'called reassemble', 'called paint'], handler: (String line) {
164 165
            return 'R';
          }),
166
          Barrier('Performing hot restart...'.padRight(progressMessageWidth)),
167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
          Multiple(<Pattern>['ready', 'called main', 'called paint'], handler: (String line) {
            return 'p';
          }),
          Multiple(<Pattern>['ready', 'called paint', 'called debugPaintSize'], handler: (String line) {
            return 'p';
          }),
          Multiple(<Pattern>['ready', 'called paint'], handler: (String line) {
            return 'q';
          }),
          const Barrier('Application finished.'),
        ],
        logging: false, // we ignore leading log lines to avoid making this test sensitive to e.g. the help message text
      );
      // We check the output from the app (all starts with "called ...") and the output from the tool
      // (everything else) separately, because their relative timing isn't guaranteed. Their rough timing
      // is verified by the expected transitions above.
183 184
      expect(result.stdout.where((String line) => line.startsWith('called ')), <Object>[
        // logs start after we initiate the hot reload
185
        // hot reload:
186
        'called reassemble',
187 188 189 190 191 192 193 194 195 196
        'called paint',
        // hot restart:
        'called main',
        'called paint',
        // debugPaintSizeEnabled = true:
        'called paint',
        'called debugPaintSize',
        // debugPaintSizeEnabled = false:
        'called paint',
      ]);
197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
      expect(
        result.stdout.where((String line) => !line.startsWith('called ')), <Object>[
          // logs start after we receive the response to hitting "r"
          'Performing hot reload...'.padRight(progressMessageWidth),
          startsWith('Reloaded 0 libraries in '),
          'ready',
          '', // this newline is the one for after we hit "R"
          'Performing hot restart...'.padRight(progressMessageWidth),
          startsWith('Restarted application in '),
          'ready',
          '', // newline for after we hit "p" the first time
          'ready',
          '', // newline for after we hit "p" the second time
          'ready',
          '', // this newline is the one for after we hit "q"
          'Application finished.',
          'ready',
        ],
        reason: 'stdout from command ${command.join(' ')} was unexpected, '
          'full Stdout:\n\n${result.stdout.join('\n')}\n\n'
          'Stderr:\n\n${result.stderr.join('\n')}',
      );
219 220 221 222
      expect(result.exitCode, 0);
    } finally {
      tryToDelete(fileSystem.directory(tempDirectory));
    }
223
  });
224 225 226

  testWithoutContext('flutter error messages include a DevTools link', () async {
    final String testDirectory = fileSystem.path.join(flutterRoot, 'dev', 'integration_tests', 'ui');
227
    final String tempDirectory = fileSystem.systemTempDirectory.createTempSync('flutter_overall_experience_test.').resolveSymbolicLinksSync();
228 229 230 231 232 233
    final String testScript = fileSystem.path.join('lib', 'overflow.dart');
    try {
      final ProcessTestResult result = await runFlutter(
        <String>['run', '-dflutter-tester', testScript],
        testDirectory,
        <Transition>[
234
          Barrier(RegExp(r'^A Dart VM Service on Flutter test device is available at: ')),
235 236 237
          Barrier(RegExp(r'^The Flutter DevTools debugger and profiler on Flutter test device is available at: '), handler: (String line) {
            return 'r';
          }),
238
          Barrier('Performing hot reload...'.padRight(progressMessageWidth), logging: true),
239 240 241 242 243 244 245
          Barrier(RegExp(r'^Reloaded 0 libraries in [0-9]+ms.'), handler: (String line) {
            return 'q';
          }),
        ],
        logging: false,
      );
      expect(result.exitCode, 0);
246
      expect(result.stdout, <Object>[
247
        startsWith('Performing hot reload...'),
248
        '',
249 250 251 252 253
        '══╡ EXCEPTION CAUGHT BY RENDERING LIBRARY ╞═════════════════════════════════════════════════════════',
        'The following assertion was thrown during layout:',
        'A RenderFlex overflowed by 69200 pixels on the right.',
        '',
        'The relevant error-causing widget was:',
254
        matches(RegExp(r'^  Row .+flutter/dev/integration_tests/ui/lib/overflow\.dart:32:18$')),
255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
        '',
        'To inspect this widget in Flutter DevTools, visit:',
        startsWith('http'),
        '',
        'The overflowing RenderFlex has an orientation of Axis.horizontal.',
        'The edge of the RenderFlex that is overflowing has been marked in the rendering with a yellow and',
        'black striped pattern. This is usually caused by the contents being too big for the RenderFlex.',
        'Consider applying a flex factor (e.g. using an Expanded widget) to force the children of the',
        'RenderFlex to fit within the available space instead of being sized to their natural size.',
        'This is considered an error condition because it indicates that there is content that cannot be',
        'seen. If the content is legitimately bigger than the available space, consider clipping it with a',
        'ClipRect widget before putting it in the flex, or using a scrollable container rather than a Flex,',
        'like a ListView.',
        matches(RegExp(r'^The specific RenderFlex in question is: RenderFlex#..... OVERFLOWING:$')),
        startsWith('  creator: Row ← Test ← '),
        contains(' ← '),
271
        endsWith(' ⋯'),
272 273 274 275 276 277 278 279 280 281 282
        '  parentData: <none> (can use size)',
        '  constraints: BoxConstraints(w=800.0, h=600.0)',
        '  size: Size(800.0, 600.0)',
        '  direction: horizontal',
        '  mainAxisAlignment: start',
        '  mainAxisSize: max',
        '  crossAxisAlignment: center',
        '  textDirection: ltr',
        '  verticalDirection: down',
        '◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤◢◤',
        '════════════════════════════════════════════════════════════════════════════════════════════════════',
283
        '',
284 285 286 287 288 289 290
        startsWith('Reloaded 0 libraries in '),
        '',
        'Application finished.',
      ]);
    } finally {
      tryToDelete(fileSystem.directory(tempDirectory));
    }
291
  });
292 293 294 295 296 297

  testWithoutContext('flutter run help output', () async {
    // This test enables all logging so that it checks the exact text of starting up an application.
    // The idea is to verify that we're not outputting spurious messages.
    // WHEN EDITING THIS TEST PLEASE CAREFULLY CONSIDER WHETHER THE NEW OUTPUT IS AN IMPROVEMENT.
    final String testDirectory = fileSystem.path.join(flutterRoot, 'examples', 'hello_world');
298
    final RegExp finalLine = RegExp(r'^The Flutter DevTools');
299
    final ProcessTestResult result = await runFlutter(
300
      <String>['run', '-dflutter-tester'],
301 302 303 304 305 306 307 308 309 310 311 312 313
      testDirectory,
      <Transition>[
        Barrier(finalLine, handler: (String line) {
          return 'h';
        }),
        Barrier(finalLine, handler: (String line) {
          return 'q';
        }),
        const Barrier('Application finished.'),
      ],
    );
    expect(result.exitCode, 0);
    expect(result.stderr, isEmpty);
314
    expect(result.stdout, containsAllInOrder(<Object>[
315 316 317 318 319 320
      startsWith('Launching '),
      startsWith('Syncing files to device Flutter test device...'),
      '',
      'Flutter run key commands.',
      startsWith('r Hot reload.'),
      'R Hot restart.',
321
      'h List all available interactive commands.',
322 323 324 325
      'd Detach (terminate "flutter run" but leave application running).',
      'c Clear the screen',
      'q Quit (terminate the application on the device).',
      '',
326
      startsWith('A Dart VM Service on Flutter test device is available at: http://'),
327
      startsWith('The Flutter DevTools debugger and profiler on Flutter test device is available at: http://'),
328 329 330 331
      '',
      'Flutter run key commands.',
      startsWith('r Hot reload.'),
      'R Hot restart.',
332
      'v Open Flutter DevTools.',
333 334 335
      'w Dump widget hierarchy to the console.                                               (debugDumpApp)',
      't Dump rendering tree to the console.                                          (debugDumpRenderTree)',
      'L Dump layer tree to the console.                                               (debugDumpLayerTree)',
336
      'f Dump focus tree to the console.                                               (debugDumpFocusTree)',
337 338 339 340
      'S Dump accessibility tree in traversal order.                                   (debugDumpSemantics)',
      'U Dump accessibility tree in inverse hit test order.                            (debugDumpSemantics)',
      'i Toggle widget inspector.                                  (WidgetsApp.showWidgetInspectorOverride)',
      'p Toggle the display of construction lines.                                  (debugPaintSizeEnabled)',
341
      'I Toggle oversized image inversion.                                     (debugInvertOversizedImages)',
342
      'o Simulate different operating systems.                                      (defaultTargetPlatform)',
343
      'b Toggle platform brightness (dark and light mode).                        (debugBrightnessOverride)',
344 345
      'P Toggle performance overlay.                                    (WidgetsApp.showPerformanceOverlay)',
      'a Toggle timeline events for all widget build methods.                    (debugProfileWidgetBuilds)',
346 347
      'M Write SkSL shaders to a unique file in the project directory.',
      'g Run source code generators.',
348
      'j Dump frame raster stats for the current frame. (Unsupported for web)',
349 350 351 352
      'h Repeat this help message.',
      'd Detach (terminate "flutter run" but leave application running).',
      'c Clear the screen',
      'q Quit (terminate the application on the device).',
353
      '',
354
      startsWith('A Dart VM Service on Flutter test device is available at: http://'),
355
      startsWith('The Flutter DevTools debugger and profiler on Flutter test device is available at: http://'),
356 357
      '',
      'Application finished.',
358
    ]));
359
  });
360
}