Unverified Commit 9c0bd18c authored by Dan Field's avatar Dan Field Committed by GitHub

Reland eliminate timeouts from integration tests (#85141)

parent 568f135a
......@@ -7,7 +7,6 @@ import 'dart:async';
import 'package:integration_test/integration_test_driver.dart' as driver;
Future<void> main() => driver.integrationDriver(
timeout: const Duration(minutes: 5),
responseDataCallback: (Map<String, dynamic>? data) async {
await driver.writeResponseData(
data,
......
......@@ -55,12 +55,12 @@ void main() {
test('complex_layout_scroll_perf', () async {
await testScrollPerf('complex-scroll', 'complex_layout_scroll_perf');
});
}, timeout: Timeout.none);
test('tiles_scroll_perf', () async {
await driver.tap(find.byTooltip('Open navigation menu'));
await driver.tap(find.byValueKey('scroll-switcher'));
await testScrollPerf('tiles-scroll', 'tiles_scroll_perf');
});
}, timeout: Timeout.none);
});
}
......@@ -40,6 +40,6 @@ void main() {
final String jsonEncoded = json.encode(<String, dynamic>{'initialSemanticsTreeCreation': semanticsTreeCreation.inMilliseconds});
File(p.join(testOutputsDirectory, 'complex_layout_semantics_perf.json')).writeAsStringSync(jsonEncoded);
});
}, timeout: Timeout.none);
});
}
......@@ -15,6 +15,5 @@ Future<void> main() async {
kCullOpacityRouteName,
pageDelay: const Duration(seconds: 1),
duration: const Duration(seconds: 10),
timeout: const Duration(seconds: 45),
);
}
......@@ -12,6 +12,5 @@ void main() {
kMultiWidgetConstructionRouteName,
pageDelay: const Duration(seconds: 1),
duration: const Duration(seconds: 10),
timeout: const Duration(seconds: 45),
);
}
......@@ -15,7 +15,6 @@ void macroPerfTestE2E(
String routeName, {
Duration pageDelay,
Duration duration = const Duration(seconds: 3),
Duration timeout = const Duration(seconds: 30),
ControlCallback body,
ControlCallback setup,
}) {
......@@ -63,5 +62,5 @@ void macroPerfTestE2E(
}
await durationFuture;
});
}, semanticsEnabled: false, timeout: Timeout(timeout));
}, semanticsEnabled: false, timeout: Timeout.none);
}
......@@ -20,5 +20,5 @@ Future<void> main() async {
await summary.writeTimelineToFile(fileName, pretty: true);
await driver.close();
});
}, timeout: Timeout.none);
}
......@@ -12,6 +12,5 @@ void main() {
kCullOpacityRouteName,
pageDelay: const Duration(seconds: 1),
duration: const Duration(seconds: 10),
timeout: const Duration(minutes: 2),
);
}
......@@ -5,7 +5,6 @@
import 'package:integration_test/integration_test_driver.dart' as driver;
Future<void> main() => driver.integrationDriver(
timeout: const Duration(minutes: 5),
responseDataCallback: (Map<String, dynamic> data) async {
await driver.writeResponseData(
data['performance'] as Map<String, dynamic>,
......
......@@ -7,7 +7,6 @@ import 'dart:io';
import 'package:integration_test/integration_test_driver.dart' as driver;
Future<void> main() => driver.integrationDriver(
timeout: const Duration(minutes: 1),
responseDataCallback: (Map<String, dynamic> data) async {
final Map<String, dynamic> benchmarkLiveResult =
data['benchmarkLive'] as Map<String,dynamic>;
......
......@@ -40,5 +40,5 @@ Future<void> main() async {
}
await driver.close();
});
}, timeout: Timeout.none);
}
......@@ -12,6 +12,5 @@ void main() {
kMultiWidgetConstructionRouteName,
pageDelay: const Duration(seconds: 1),
duration: const Duration(seconds: 10),
timeout: const Duration(seconds: 45),
);
}
......@@ -11,7 +11,6 @@ void main() {
macroPerfTest(
'picture_cache_perf',
kPictureCacheRouteName,
timeout: const Duration(seconds: 60),
pageDelay: const Duration(seconds: 1),
driverOps: (FlutterDriver driver) async {
final SerializableFinder tabBarView = find.byValueKey('tabbar_view');
......
......@@ -30,7 +30,7 @@ void main() {
await file.writeAsString(_encodeJson(<String, dynamic>{
'stack_size': stackSizeInBytes,
}));
}, timeout: const Timeout(kTimeout));
}, timeout: Timeout.none);
}
String _encodeJson(Map<String, dynamic> jsonObject) {
......
......@@ -6,8 +6,6 @@ import 'package:flutter_driver/flutter_driver.dart';
import 'package:macrobenchmarks/common.dart';
import 'package:test/test.dart' hide TypeMatcher, isInstanceOf;
const Duration kTimeout = Duration(seconds: 30);
typedef DriverTestCallBack = Future<void> Function(FlutterDriver driver);
Future<void> runDriverTestForRoute(String routeName, DriverTestCallBack body) async {
......@@ -34,14 +32,13 @@ Future<void> runDriverTestForRoute(String routeName, DriverTestCallBack body) as
}
void macroPerfTest(
String testName,
String routeName,
{ Duration pageDelay,
Duration duration = const Duration(seconds: 3),
Duration timeout = kTimeout,
Future<void> Function(FlutterDriver driver) driverOps,
Future<void> Function(FlutterDriver driver) setupOps,
}) {
String testName,
String routeName, {
Duration pageDelay,
Duration duration = const Duration(seconds: 3),
Future<void> Function(FlutterDriver driver) driverOps,
Future<void> Function(FlutterDriver driver) setupOps,
}) {
test(testName, () async {
Timeline timeline;
await runDriverTestForRoute(routeName, (FlutterDriver driver) async {
......@@ -67,5 +64,5 @@ void macroPerfTest(
final TimelineSummary summary = TimelineSummary.summarize(timeline);
await summary.writeTimelineToFile(testName, pretty: true);
}, timeout: Timeout(timeout));
}, timeout: Timeout.none);
}
......@@ -55,6 +55,6 @@ void main() {
await driver.runUnsynchronized(() async {
await testScrollPerf('platform-views-scroll', 'platform_views_scroll_perf');
});
});
}, timeout: Timeout.none);
});
}
......@@ -55,6 +55,6 @@ void main() {
await driver.runUnsynchronized(() async {
await testScrollPerf('platform-views-scroll', 'platform_views_scroll_perf_hybrid_composition');
});
});
}, timeout: Timeout.none);
});
}
......@@ -40,6 +40,6 @@ void main() {
final TimelineSummary summary = TimelineSummary.summarize(timeline);
await summary.writeTimelineToFile('stocks_scroll_perf', pretty: true);
});
}, timeout: Timeout.none);
});
}
......@@ -22,7 +22,7 @@ void main() {
test('Stock list is shown', () async {
final SerializableFinder stockList = find.byValueKey('stock-list');
expect(stockList, isNotNull);
});
}, timeout: Timeout.none);
test('open AAPL stock', () async {
final SerializableFinder stockList = find.byValueKey('stock-list');
......@@ -36,10 +36,9 @@ void main() {
final SerializableFinder stockOption =
find.byValueKey('AAPL_symbol_name');
final String symbol = await driver.getText(stockOption,
timeout: const Duration(milliseconds: 500));
final String symbol = await driver.getText(stockOption);
expect(symbol, 'AAPL');
}, skip: 'Needs to be fixed on Fuchsia.');
}, skip: 'Needs to be fixed on Fuchsia.', timeout: Timeout.none);
});
}
......@@ -72,6 +72,9 @@ Future<void> run(List<String> arguments) async {
print('$clock Internationalization...');
await verifyInternationalizations();
print('$clock Integration test timeouts...');
await verifyIntegrationTestTimeouts(flutterRoot);
// Ensure that all package dependencies are in sync.
print('$clock Package dependencies...');
await runCommand(flutter, <String>['update-packages', '--verify-only'],
......@@ -394,6 +397,31 @@ Future<void> verifyNoBadImportsInFlutterTools(String workingDirectory) async {
}
}
Future<void> verifyIntegrationTestTimeouts(String workingDirectory) async {
final List<String> errors = <String>[];
final String dev = path.join(workingDirectory, 'dev');
final List<File> files = await _allFiles(dev, 'dart', minimumMatches: 1)
.where((File file) => file.path.contains('test_driver') && (file.path.endsWith('_test.dart') || file.path.endsWith('util.dart')))
.toList();
for (final File file in files) {
final String contents = file.readAsStringSync();
final int testCount = ' test('.allMatches(contents).length;
final int timeoutNoneCount = 'timeout: Timeout.none'.allMatches(contents).length;
if (testCount != timeoutNoneCount) {
errors.add('$yellow${file.path}$reset has at least $testCount test(s) but only $timeoutNoneCount `Timeout.none`(s).');
}
}
if (errors.isNotEmpty) {
exitWithError(<String>[
if (errors.length == 1)
'${bold}An error was detected when looking at import dependencies within the flutter_tools package:$reset'
else
'${bold}Multiple errors were detected when looking at import dependencies within the flutter_tools package:$reset',
...errors.map((String paragraph) => '$paragraph\n'),
]);
}
}
Future<void> verifyInternationalizations() async {
final EvalResult materialGenResult = await _evalCommand(
dart,
......@@ -1006,7 +1034,7 @@ Future<List<File>> _gitFiles(String workingDirectory, {bool runSilently = true})
);
if (evalResult.exitCode != 0) {
exitWithError(<String>[
'git ls-filese failed with exit code ${evalResult.exitCode}',
'git ls-files failed with exit code ${evalResult.exitCode}',
'${bold}stdout:$reset',
evalResult.stdout,
'${bold}stderr:$reset',
......
......@@ -161,7 +161,7 @@ void main() {
],
),
);
});
}, timeout: Timeout.none);
test('password TextField has correct Android semantics', () async {
final SerializableFinder passwordTextField = find.descendant(
......@@ -229,7 +229,7 @@ void main() {
],
),
);
});
}, timeout: Timeout.none);
tearDownAll(() async {
await driver.tap(find.byValueKey('back'));
......@@ -287,7 +287,7 @@ void main() {
],
),
);
});
}, timeout: Timeout.none);
test('Radio has correct Android semantics', () async {
Future<AndroidSemanticsNode> getRadioSemantics(String key) async {
return getSemantics(find.byValueKey(key));
......@@ -323,7 +323,7 @@ void main() {
],
),
);
});
}, timeout: Timeout.none);
test('Switch has correct Android semantics', () async {
Future<AndroidSemanticsNode> getSwitchSemantics(String key) async {
return getSemantics(find.byValueKey(key));
......@@ -359,7 +359,7 @@ void main() {
],
),
);
});
}, timeout: Timeout.none);
// Regression test for https://github.com/flutter/flutter/issues/20820.
test('Switch can be labeled', () async {
......@@ -381,7 +381,7 @@ void main() {
],
),
);
});
}, timeout: Timeout.none);
tearDownAll(() async {
await driver.tap(find.byValueKey('back'));
......@@ -461,7 +461,7 @@ void main() {
} finally {
await driver.tap(find.byValueKey('$popupKeyValue.${popupItems.first}'));
}
});
}, timeout: Timeout.none);
test('Dropdown Menu has correct Android semantics', () async {
expect(
......@@ -551,7 +551,7 @@ void main() {
),
);
}
});
}, timeout: Timeout.none);
test('Modal alert dialog has correct Android semantics', () async {
expect(
......@@ -647,7 +647,7 @@ void main() {
} finally {
await driver.tap(find.byValueKey('$alertKeyValue.OK'));
}
});
}, timeout: Timeout.none);
tearDownAll(() async {
await Future<void>.delayed(const Duration(milliseconds: 500));
......@@ -665,14 +665,14 @@ void main() {
await getSemantics(find.byValueKey(appBarTitleKeyValue)),
hasAndroidSemantics(isHeading: true),
);
});
}, timeout: Timeout.none);
test('body text does not have Android heading semantics', () async {
expect(
await getSemantics(find.byValueKey(bodyTextKeyValue)),
hasAndroidSemantics(isHeading: false),
);
});
}, timeout: Timeout.none);
tearDownAll(() async {
await driver.tap(find.byValueKey('back'));
......
......@@ -27,7 +27,7 @@ Future<void> main() async {
expect(errorMessage, '');
final SerializableFinder backButton = find.byValueKey('back');
await driver.tap(backButton);
});
}, timeout: Timeout.none);
group('WindowManager', ()
{
......@@ -49,7 +49,7 @@ Future<void> main() async {
await driver.tap(showAlertDialog);
final String status = await driver.getText(find.byValueKey('Status'));
expect(status, 'Success');
});
}, timeout: Timeout.none);
test('Child windows can handle touches', () async {
final SerializableFinder addWindow = find.byValueKey('AddWindow');
......@@ -59,9 +59,8 @@ Future<void> main() async {
await driver.tap(tapWindow);
final String windowClickCount = await driver.getText(
find.byValueKey('WindowClickCount'),
timeout: const Duration(seconds: 5),
);
expect(windowClickCount, 'Click count: 1');
});
}, timeout: Timeout.none);
});
}
......@@ -25,7 +25,7 @@ void main() {
if (status != 'complete') {
fail('Failed at step $step with status $status');
}
}, timeout: const Timeout(Duration(minutes: 1)));
}, timeout: Timeout.none);
tearDownAll(() async {
driver.close();
......
......@@ -59,7 +59,7 @@ Future<void> main() async {
expect(double.parse(matchFast.group(1)!), closeTo(flutterFrameRate * 2.0, 5.0));
expect(double.parse(matchFast.group(2)!), closeTo(flutterFrameRate, 10.0));
expect(int.parse(matchFast.group(3)!), 1);
});
}, timeout: Timeout.none);
tearDownAll(() async {
driver.close();
......
......@@ -17,7 +17,7 @@ void main() {
final SerializableFinder flavorField = find.byValueKey('flavor');
final String flavor = await driver.getText(flavorField);
expect(flavor, 'paid');
});
}, timeout: Timeout.none);
tearDownAll(() async {
driver.close();
......
......@@ -42,6 +42,6 @@ void main() {
final TimelineSummary summary = TimelineSummary.summarize(timeline);
await summary.writeTimelineToFile('home_scroll_perf', pretty: true);
});
}, timeout: Timeout.none);
});
}
......@@ -35,6 +35,6 @@ void main() {
await driver.scroll(demoList, 0.0, 300.0, const Duration(milliseconds: 300));
await Future<void>.delayed(const Duration(milliseconds: 500));
}
});
}, timeout: Timeout.none);
});
}
......@@ -48,7 +48,6 @@ void main([List<String> args = const <String>[]]) {
..removeAll(kProfiledDemos);
await runDemos(unprofiledDemos.toList(), tester);
},
timeout: const Timeout(Duration(minutes: 5)),
semanticsEnabled: withSemantics,
);
});
......
......@@ -5,7 +5,6 @@
import 'package:integration_test/integration_test_driver.dart' as driver;
Future<void> main() => driver.integrationDriver(
timeout: const Duration(minutes: 5),
responseDataCallback: (Map<String, dynamic>? data) async {
await driver.writeResponseData(
data!['performance'] as Map<String, dynamic>,
......
......@@ -130,7 +130,6 @@ Future<void> runDemos(List<String> demos, FlutterDriver driver) async {
await driver.scrollUntilVisible(demoList, demoItem,
dyScroll: -48.0,
alignment: 0.5,
timeout: const Duration(seconds: 30),
);
for (int i = 0; i < 2; i += 1) {
......@@ -181,7 +180,7 @@ void main([List<String> args = const <String>[]]) {
// Assert that we can use semantics related finders in profile mode.
final int id = await driver.getSemanticsId(find.bySemanticsLabel('Material'));
expect(id, greaterThan(-1));
}, skip: !withSemantics);
}, skip: !withSemantics, timeout: Timeout.none);
test('all demos', () async {
// Collect timeline data for just a limited set of demos to avoid OOMs.
......@@ -218,6 +217,6 @@ void main([List<String> args = const <String>[]]) {
await runDemos(unprofiledDemos.toList(), driver);
}
}, timeout: const Timeout(Duration(minutes: 5)));
}, timeout: Timeout.none);
});
}
......@@ -25,7 +25,7 @@ Future<void> main() async {
expect(errorMessage, '');
final SerializableFinder backButton = find.byValueKey('back');
await driver.tap(backButton);
});
}, timeout: Timeout.none);
group('Nested View Event', () {
setUpAll(() async {
......@@ -45,7 +45,7 @@ Future<void> main() async {
await driver.tap(showAlertDialog);
final String status = await driver.getText(find.byValueKey('Status'));
expect(status, 'Success');
});
}, timeout: Timeout.none);
test('Child view can handle touches', () async {
final SerializableFinder addChildView = find.byValueKey('AddChildView');
......@@ -56,7 +56,7 @@ Future<void> main() async {
final String nestedViewClickCount =
await driver.getText(find.byValueKey('NestedViewClickCount'));
expect(nestedViewClickCount, 'Click count: 1');
});
}, timeout: Timeout.none);
});
group('Flutter surface switch', () {
......@@ -107,6 +107,6 @@ Future<void> main() async {
' |-ViewGroup\n'
' |-FlutterImageView\n' // Flutter UI (overlay surface)
);
});
}, timeout: Timeout.none);
});
}
......@@ -40,7 +40,7 @@ void main() {
final Health driverHealth = await driver.checkHealth();
expect(driverHealth.status, HealthStatus.ok);
});
}, timeout: Timeout.none);
test('Merge thread to create and remove platform views should not crash',
() async {
......@@ -65,6 +65,6 @@ void main() {
final Health driverHealth = await driver.checkHealth();
expect(driverHealth.status, HealthStatus.ok);
});
}, timeout: Timeout.none);
});
}
......@@ -25,7 +25,7 @@ void main() {
if (status != 'complete') {
fail('Failed at step $step with status $status');
}
});
}, timeout: Timeout.none);
tearDownAll(() async {
driver.close();
......
......@@ -18,5 +18,5 @@ void main() {
test('check that we are painting in debugPaintSize mode', () async {
expect(await driver.requestData('status'), 'log: paint debugPaintSize');
});
}, timeout: Timeout.none);
}
......@@ -18,6 +18,6 @@ void main() {
test('check that we are in normal mode', () async {
expect(await driver.requestData('status'), 'log: paint');
await driver.waitForAbsent(find.byType('PerformanceOverlay'), timeout: Duration.zero);
});
await driver.waitForAbsent(find.byType('PerformanceOverlay'));
}, timeout: Timeout.none);
}
......@@ -18,6 +18,6 @@ void main() {
test('check that we are showing the performance overlay', () async {
await driver.requestData('status'); // force a reassemble
await driver.waitFor(find.byType('PerformanceOverlay'), timeout: Duration.zero);
});
await driver.waitFor(find.byType('PerformanceOverlay'));
}, timeout: Timeout.none);
}
......@@ -18,5 +18,5 @@ void main() {
test('Can run with --dart-define', () async {
await driver.waitFor(find.text('Example,AValue'));
});
}, timeout: Timeout.none);
}
......@@ -23,7 +23,7 @@ void main() {
test('waitFor should find text "present"', () async {
await driver.waitFor(presentText);
});
}, timeout: Timeout.none);
test('waitForAbsent should time out waiting for text "present" to disappear', () async {
await expectLater(
......@@ -34,7 +34,7 @@ void main() {
contains('Timeout while executing waitForAbsent'),
)),
);
});
}, timeout: Timeout.none);
test('waitForAbsent should resolve when text "present" disappears', () async {
// Begin waiting for it to disappear
......@@ -50,7 +50,7 @@ void main() {
// Ensure waitForAbsent resolves
await whenWaitForAbsentResolves.future;
});
}, timeout: Timeout.none);
test('waitFor times out waiting for "present" to reappear', () async {
await expectLater(
......@@ -61,7 +61,7 @@ void main() {
contains('Timeout while executing waitFor'),
)),
);
});
}, timeout: Timeout.none);
test('waitFor should resolve when text "present" reappears', () async {
// Begin waiting for it to reappear
......@@ -77,11 +77,11 @@ void main() {
// Ensure waitFor resolves
await whenWaitForResolves.future;
});
}, timeout: Timeout.none);
test('waitForAbsent resolves immediately when the element does not exist', () async {
await driver.waitForAbsent(find.text('that does not exist'));
});
}, timeout: Timeout.none);
test('uses hit test to determine tappable elements', () async {
final SerializableFinder a = find.byValueKey('a');
......@@ -97,7 +97,7 @@ void main() {
// Close it again
await driver.tap(a);
await driver.waitForAbsent(menu);
});
}, timeout: Timeout.none);
test('enters text in a text field', () async {
final SerializableFinder textField = find.byValueKey('enter-text-field');
......@@ -106,6 +106,6 @@ void main() {
await driver.waitFor(find.text('Hello!'));
await driver.enterText('World!');
await driver.waitFor(find.text('World!'));
});
}, timeout: Timeout.none);
});
}
......@@ -18,6 +18,6 @@ void main() {
await driver.close();
});
test('empty', () async {}, timeout: const Timeout(Duration(minutes: 1)));
test('empty', () async {}, timeout: Timeout.none);
});
}
......@@ -59,6 +59,6 @@ void main() {
}
}
expect(heightTextDidExpand, isTrue);
});
}, timeout: Timeout.none);
});
}
......@@ -46,6 +46,6 @@ void main() {
// Ensure the scroll offset changed appropriately when TextField scrolled back into view.
expect(scrollOffsetWithKeyboard, greaterThan(scrollOffsetWithoutKeyboard));
});
}, timeout: Timeout.none);
});
}
......@@ -37,7 +37,7 @@ void main() {
final String foundLicense = await driver.getText(find.byValueKey('FlutterLicense'));
expect(foundPackage, equals('flutter'));
expect(foundLicense, equals(license));
});
}, timeout: Timeout.none);
test('engine license', () async {
await driver.waitFor(find.byValueKey('Header'));
......@@ -46,6 +46,6 @@ void main() {
expect(foundPackage, equals('engine'));
// The engine has the same license, but with a different Copyright date.
expect(foundLicense, contains(license.replaceFirst('2014', '2013')));
});
}, timeout: Timeout.none);
});
}
......@@ -22,6 +22,6 @@ void main() {
// This only makes sense if you ran the test as described
// in the test file. It's normally run from devicelab.
expect(await driver.requestData('route'), '/smuggle-it');
});
}, timeout: Timeout.none);
});
}
......@@ -48,6 +48,6 @@ void main() {
imageBefore = imageAfter;
}
}, timeout: const Timeout(Duration(minutes: 2)));
}, timeout: Timeout.none);
});
}
......@@ -20,5 +20,5 @@ Future<void> main() async {
response.allTestsPassed,
false,
);
});
}, timeout: Timeout.none);
}
......@@ -64,7 +64,7 @@ Future<void> writeResponseData(
///
/// ```
Future<void> integrationDriver({
Duration timeout = const Duration(minutes: 1),
Duration timeout = const Duration(minutes: 20),
ResponseDataCallback? responseDataCallback = writeResponseData,
}) async {
final FlutterDriver driver = await FlutterDriver.connect();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment