Randomize Framework tests, opt out some tests that currently fail. (#85159)

This turns on order shuffling for all tests that don't fail with it on, marking those tests that do fail with a tag so that they will be run without shuffling on.

To determine which tests fail with it on, I ran all the tests 100 times with different random shuffle seeds, and then also ran it with the date seeds from today until the end of July, and tagged all of the test suites (files) that fail, with a seed that caused them to fail.

dev/bots/run_command.dart

@@ -22,6 +22,7 @@ Stream<String> runAndGetStdout(String executable, List<String> arguments, {
   String workingDirectory,
   Map<String, String> environment,
   bool expectNonZeroExit = false,
+  bool expectNoTests = false,
 }) async* {
   final StreamController<String> output = StreamController<String>();
   final Future<CommandResult> command = runCommand(
@@ -30,6 +31,7 @@ Stream<String> runAndGetStdout(String executable, List<String> arguments, {
     workingDirectory: workingDirectory,
     environment: environment,
     expectNonZeroExit: expectNonZeroExit,
+    expectNoTests: expectNoTests,
     // Capture the output so it's not printed to the console by default.
     outputMode: OutputMode.capture,
     outputListener: (String line, io.Process process) {
@@ -113,7 +115,8 @@ Future<Command> startCommand(String executable, List<String> arguments, {
     environment: environment,
   );
 
-  Future<List<List<int>>> savedStdout, savedStderr;
+  Future<List<List<int>>> savedStdout = Future<List<List<int>>>.value(<List<int>>[]);
+  Future<List<List<int>>> savedStderr = Future<List<List<int>>>.value(<List<int>>[]);
   final Stream<List<int>> stdoutSource = process.stdout
     .transform<String>(const Utf8Decoder())
     .transform(const LineSplitter())
@@ -128,8 +131,14 @@ Future<Command> startCommand(String executable, List<String> arguments, {
     .transform(const Utf8Encoder());
   switch (outputMode) {
     case OutputMode.print:
-      stdoutSource.listen(io.stdout.add);
-      process.stderr.listen(io.stderr.add);
+      stdoutSource.listen((List<int> output) {
+        io.stdout.add(output);
+        savedStdout.then((List<List<int>> list) => list.add(output));
+      });
+      process.stderr.listen((List<int> output) {
+        io.stderr.add(output);
+        savedStdout.then((List<List<int>> list) => list.add(output));
+      });
       break;
     case OutputMode.capture:
       savedStdout = stdoutSource.toList();
@@ -158,6 +167,7 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, {
   String workingDirectory,
   Map<String, String> environment,
   bool expectNonZeroExit = false,
+  bool expectNoTests = false,
   int expectedExitCode,
   String failureMessage,
   OutputMode outputMode = OutputMode.print,
@@ -182,7 +192,21 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, {
 
   final CommandResult result = await command.onExit;
 
-  if ((result.exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && result.exitCode != expectedExitCode)) {
+  // Currently, the test infrastructure fails if it doesn't find any tests to
+  // run, but in the case of tests tagged as "no-shuffle", there might either be
+  // none that can be shuffled, or none that shouldn't be shuffled, and since
+  // we're running it twice to get all the tests in either category, it
+  // shouldn't fail if no tests are run.
+  //
+  // TODO(gspencergoog): This is a workaround until
+  // https://github.com/dart-lang/test/issues/1546 is addressed. Remove the
+  // workaround (parsing the test output) when/if that issue is fixed.
+  final bool skipErrorExit = expectNoTests &&
+      result != null &&
+      result.flattenedStdout != null &&
+      result.flattenedStdout.trimRight().endsWith('No tests ran.');
+
+  if (!skipErrorExit && ((result.exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && result.exitCode != expectedExitCode))) {
     // Print the output when we get unexpected results (unless output was
     // printed already).
     switch (outputMode) {

dev/bots/test.dart

@@ -620,6 +620,7 @@ Future<void> _runFrameworkTests() async {
         path.join(flutterRoot, 'packages', 'flutter'),
         options: <String>[trackWidgetCreationOption, ...soundNullSafetyOptions],
         tests: <String>[ path.join('test', 'widgets') + path.separator ],
+        shuffleOrder: true,
       );
     }
     // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation
@@ -627,6 +628,7 @@ Future<void> _runFrameworkTests() async {
       await _runFlutterTest(
         path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'),
         options: <String>[trackWidgetCreationOption],
+        shuffleOrder: true,
       );
     }
     // Run release mode tests (see packages/flutter/test_release/README.md)
@@ -634,6 +636,7 @@ Future<void> _runFrameworkTests() async {
       path.join(flutterRoot, 'packages', 'flutter'),
       options: <String>['--dart-define=dart.vm.product=true', ...soundNullSafetyOptions],
       tests: <String>['test_release${path.separator}'],
+      shuffleOrder: true,
     );
   }
 
@@ -650,6 +653,7 @@ Future<void> _runFrameworkTests() async {
         path.join(flutterRoot, 'packages', 'flutter'),
         options: <String>[trackWidgetCreationOption, ...soundNullSafetyOptions],
         tests: tests,
+        shuffleOrder: true,
       );
     }
   }
@@ -1376,6 +1380,18 @@ Future<void> _pubRunTest(String workingDirectory, {
   }
 }
 
+// Used as an argument to the runTests internal function in the _runFlutterTest
+// function.
+enum _ShuffleMode {
+  // Runs only the tests not tagged with "no-shuffle" in a shuffled order.
+  shuffled,
+  // Runs only the tests that would fail if shuffled (those tagged with
+  // "no-shuffle").
+  unshuffled,
+  // Runs all tests without shuffling.
+  sequential,
+}
+
 Future<void> _runFlutterTest(String workingDirectory, {
   String script,
   bool expectFailure = false,
@@ -1383,20 +1399,20 @@ Future<void> _runFlutterTest(String workingDirectory, {
   OutputChecker outputChecker,
   List<String> options = const <String>[],
   bool skip = false,
+  bool shuffleOrder = false,
   Map<String, String> environment,
   List<String> tests = const <String>[],
 }) async {
   assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both');
 
-  final List<String> args = <String>[
-    'test',
+  final List<String> testArgs = <String>[
     ...options,
     ...?flutterTestArgs,
   ];
 
   final bool shouldProcessOutput = useFlutterTestFormatter && !expectFailure && !options.contains('--coverage');
   if (shouldProcessOutput)
-    args.add('--machine');
+    testArgs.add('--machine');
 
   if (script != null) {
     final String fullScriptPath = path.join(workingDirectory, script);
@@ -1410,56 +1426,98 @@ Future<void> _runFlutterTest(String workingDirectory, {
         print('This is one of the tests that is normally skipped in this configuration.');
       exit(1);
     }
-    args.add(script);
+    testArgs.add(script);
   }
 
-  args.addAll(tests);
-
-  if (!shouldProcessOutput) {
-    final OutputMode outputMode = outputChecker == null && printOutput
-      ? OutputMode.print
-      : OutputMode.capture;
-
-    final CommandResult result = await runCommand(
-      flutter,
-      args,
-      workingDirectory: workingDirectory,
-      expectNonZeroExit: expectFailure,
-      outputMode: outputMode,
-      skip: skip,
-      environment: environment,
-    );
-
-    if (outputChecker != null) {
-      final String message = outputChecker(result);
-      if (message != null)
-        exitWithError(<String>[message]);
+  testArgs.addAll(tests);
+
+  Future<void> runTests(_ShuffleMode shuffleMode) async {
+    int seed = 0;
+    final List<String> shuffleArgs = <String>[];
+    switch (shuffleMode) {
+      case _ShuffleMode.shuffled:
+        final DateTime now = DateTime.now();
+        // Generates YYYYMMDD as the seed, so that testing continues to fail on the
+        // day it was originally run, and on other days the seed can be used to
+        // replicate failures.
+        seed = now.year * 10000 + now.month * 100 + now.day;
+        print('Shuffling test order using arbitrary daily seed $seed.');
+        print('To run locally, run "flutter test -x no-shuffle --test-randomize-ordering-seed=$seed '
+            '&& flutter test -t no-shuffle" to run all tests (the latter part runs all the tests that '
+            "currently don't work when shuffled, the former skips those tests and shuffles the order "
+            'of the remaining tests).');
+        shuffleArgs.addAll(<String>['-x', 'no-shuffle', '--test-randomize-ordering-seed=$seed']);
+        break;
+      case _ShuffleMode.unshuffled:
+        shuffleArgs.addAll(<String>['-t', 'no-shuffle']);
+        break;
+      case _ShuffleMode.sequential:
+        break;
     }
-    return;
-  }
+    final List<String> args = <String>[
+      'test',
+      ...shuffleArgs,
+      ...testArgs,
+    ];
+    if (!shouldProcessOutput) {
+      final OutputMode outputMode = outputChecker == null && printOutput
+        ? OutputMode.print
+        : OutputMode.capture;
 
-  if (useFlutterTestFormatter) {
-    final FlutterCompactFormatter formatter = FlutterCompactFormatter();
-    Stream<String> testOutput;
-    try {
-      testOutput = runAndGetStdout(
+      final CommandResult result = await runCommand(
         flutter,
         args,
         workingDirectory: workingDirectory,
         expectNonZeroExit: expectFailure,
+        expectNoTests: shuffleMode == _ShuffleMode.unshuffled,
+        outputMode: outputMode,
+        skip: skip,
         environment: environment,
       );
-    } finally {
-      formatter.finish();
+      if (outputChecker != null) {
+        final String message = outputChecker(result);
+        if (message != null)
+          exitWithError(<String>[message]);
+      }
+      return;
     }
-    await _processTestOutput(formatter, testOutput);
+
+    if (useFlutterTestFormatter) {
+      final FlutterCompactFormatter formatter = FlutterCompactFormatter();
+      Stream<String> testOutput;
+      try {
+        testOutput = runAndGetStdout(
+          flutter,
+          args,
+          workingDirectory: workingDirectory,
+          expectNonZeroExit: expectFailure,
+          expectNoTests: shuffleMode == _ShuffleMode.unshuffled,
+          environment: environment,
+        );
+      } finally {
+        formatter.finish();
+      }
+      await _processTestOutput(formatter, testOutput);
+    } else {
+      await runCommand(
+        flutter,
+        args,
+        workingDirectory: workingDirectory,
+        expectNonZeroExit: expectFailure,
+        expectNoTests: shuffleMode == _ShuffleMode.unshuffled,
+      );
+    }
+  }
+
+  if (shuffleOrder) {
+    // Runs only the tests not tagged with "no-shuffle" in a shuffled order.
+    await runTests(_ShuffleMode.shuffled);
+    // Runs only the tests that would fail if shuffled (those tagged with
+    // "no-shuffle").
+    await runTests(_ShuffleMode.unshuffled);
   } else {
-    await runCommand(
-      flutter,
-      args,
-      workingDirectory: workingDirectory,
-      expectNonZeroExit: expectFailure,
-    );
+    // Runs all tests without shuffling.
+    await runTests(_ShuffleMode.sequential);
   }
 }
 

packages/flutter/dart_test.yaml

+tags:
+  # This tag tells the test framework to not shuffle the test order according to
+  # the --test-randomize-ordering-seed for the suites that have this tag.
+  no-shuffle:

packages/flutter/test/animation/animation_controller_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'dart:ui' as ui;
 
 import 'package:flutter/foundation.dart';

packages/flutter/test/cupertino/action_sheet_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/cupertino.dart';
 import 'package:flutter/material.dart';
 

packages/flutter/test/cupertino/date_picker_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'dart:ui';
 
 import 'package:flutter/cupertino.dart';

packages/flutter/test/cupertino/nav_bar_transition_test.dart

@@ -3,6 +3,12 @@
 // found in the LICENSE file.
 
 @TestOn('!chrome')
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=456"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/cupertino.dart';
 import 'package:flutter/rendering.dart';
 import 'package:flutter_test/flutter_test.dart';

packages/flutter/test/cupertino/text_field_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'dart:ui' as ui show BoxHeightStyle, BoxWidthStyle, Color;
 
 import 'package:flutter/cupertino.dart';

packages/flutter/test/foundation/service_extensions_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'dart:async';
 import 'dart:convert';
 import 'dart:ui' as ui;

packages/flutter/test/gestures/debug_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/foundation.dart';
 import 'package:flutter/gestures.dart';
 import 'package:flutter_test/flutter_test.dart';

packages/flutter/test/gestures/long_press_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/gestures.dart';
 import 'package:flutter_test/flutter_test.dart';
 

packages/flutter/test/gestures/tap_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=1000"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/foundation.dart';
 import 'package:flutter/gestures.dart';
 import 'package:flutter_test/flutter_test.dart';

packages/flutter/test/material/app_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/cupertino.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';

packages/flutter/test/material/dropdown_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=1408669812"
+@Tags(<String>['no-shuffle'])
+
 import 'dart:math' as math;
 import 'dart:ui' show window;
 

packages/flutter/test/material/paginated_data_table_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=1000"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/gestures.dart' show DragStartBehavior;
 import 'package:flutter/material.dart';
 import 'package:flutter/rendering.dart';

packages/flutter/test/material/progress_indicator_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=456"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/cupertino.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter/rendering.dart';

packages/flutter/test/material/scaffold_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/foundation.dart';
 import 'package:flutter/gestures.dart' show DragStartBehavior;
 import 'package:flutter/material.dart';

packages/flutter/test/material/scrollbar_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=382757700"
+@Tags(<String>['no-shuffle'])
+
 import 'dart:ui' as ui;
 
 import 'package:flutter/cupertino.dart';

packages/flutter/test/material/text_field_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=3890307731"
+@Tags(<String>['no-shuffle'])
+
 import 'dart:math' as math;
 import 'dart:ui' as ui show window, BoxHeightStyle, BoxWidthStyle;
 

packages/flutter/test/painting/image_stream_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=456"
+@Tags(<String>['no-shuffle'])
+
 import 'dart:async';
 import 'dart:typed_data';
 import 'dart:ui';

packages/flutter/test/physics/newton_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/foundation.dart';
 import 'package:flutter/physics.dart';
 import 'package:flutter/widgets.dart';

packages/flutter/test/rendering/editable_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=20210704"
+@Tags(<String>['no-shuffle'])
+
 import 'dart:io' show Platform;
 
 import 'package:flutter/foundation.dart';

packages/flutter/test/rendering/viewport_test.dart

@@ -7,6 +7,12 @@
 // initialize a binding, which rendering_tester will attempt to re-initialize
 // (or vice versa).
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/foundation.dart';
 import 'package:flutter/rendering.dart';
 import 'package:flutter/widgets.dart';

packages/flutter/test/scheduler/scheduler_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'dart:async';
 import 'dart:ui' show window;
 

packages/flutter/test/scheduler/ticker_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=4281596210"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/foundation.dart';
 import 'package:flutter/scheduler.dart';
 import 'package:flutter/services.dart';

packages/flutter/test/services/platform_channel_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=456"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/services.dart';
 import 'package:flutter_test/flutter_test.dart';
 

packages/flutter/test/widgets/app_overrides_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/widgets.dart';
 import 'package:flutter_test/flutter_test.dart';
 

packages/flutter/test/widgets/binding_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=382757700"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/foundation.dart';
 import 'package:flutter/services.dart';
 import 'package:flutter/widgets.dart';

packages/flutter/test/widgets/clip_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=456"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/material.dart';
 import 'package:flutter/rendering.dart';
 import 'package:flutter_test/flutter_test.dart';

packages/flutter/test/widgets/debug_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/foundation.dart';
 import 'package:flutter/widgets.dart';
 import 'package:flutter_test/flutter_test.dart';

packages/flutter/test/widgets/dismissible_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=456"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/gestures.dart' show DragStartBehavior;
 import 'package:flutter/widgets.dart';
 import 'package:flutter_test/flutter_test.dart';

packages/flutter/test/widgets/draggable_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'package:flutter/gestures.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter/semantics.dart';

packages/flutter/test/widgets/platform_view_test.dart

@@ -3,6 +3,12 @@
 // found in the LICENSE file.
 
 @TestOn('!chrome')
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=4281596210"
+@Tags(<String>['no-shuffle'])
+
 import 'dart:async';
 import 'dart:typed_data';
 import 'dart:ui';

packages/flutter/test/widgets/routes_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'dart:collection';
 
 import 'package:flutter/material.dart';

packages/flutter/test/widgets/scroll_aware_image_provider_test.dart

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=123"
+@Tags(<String>['no-shuffle'])
+
 import 'dart:ui' as ui show Image;
 
 import 'package:flutter/widgets.dart';

packages/flutter/test/widgets/widget_inspector_test.dart

@@ -3,6 +3,13 @@
 // found in the LICENSE file.
 
 @TestOn('!chrome')
+
+// TODO(gspencergoog): Remove this tag once this test's state leaks/test
+// dependency have been fixed.
+// https://github.com/flutter/flutter/issues/85160
+// Fails with "flutter test --test-randomize-ordering-seed=456"
+@Tags(<String>['no-shuffle'])
+
 import 'dart:async';
 import 'dart:convert';
 import 'dart:math';
@@ -1983,7 +1990,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService {
       _CreationLocation location = knownLocations[id]!;
       expect(location.file, equals(file));
       // ClockText widget.
-      expect(location.line, equals(53));
+      expect(location.line, equals(60));
       expect(location.column, equals(9));
       expect(location.name, equals('ClockText'));
       expect(count, equals(1));
@@ -1993,7 +2000,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService {
       location = knownLocations[id]!;
       expect(location.file, equals(file));
       // Text widget in _ClockTextState build method.
-      expect(location.line, equals(91));
+      expect(location.line, equals(98));
       expect(location.column, equals(12));
       expect(location.name, equals('Text'));
       expect(count, equals(1));
@@ -2020,7 +2027,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService {
       location = knownLocations[id]!;
       expect(location.file, equals(file));
       // ClockText widget.
-      expect(location.line, equals(53));
+      expect(location.line, equals(60));
       expect(location.column, equals(9));
       expect(location.name, equals('ClockText'));
       expect(count, equals(3)); // 3 clock widget instances rebuilt.
@@ -2030,7 +2037,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService {
       location = knownLocations[id]!;
       expect(location.file, equals(file));
       // Text widget in _ClockTextState build method.
-      expect(location.line, equals(91));
+      expect(location.line, equals(98));
       expect(location.column, equals(12));
       expect(location.name, equals('Text'));
       expect(count, equals(3)); // 3 clock widget instances rebuilt.