Unverified Commit 9e86e9fb authored by Jim Graham's avatar Jim Graham Committed by GitHub

report list of detail files so devicelab can save them (#64009)

parent 2d0634eb
......@@ -189,7 +189,10 @@ class _TaskRunner {
/// A result of running a single task.
class TaskResult {
/// Constructs a successful result.
TaskResult.success(this.data, {this.benchmarkScoreKeys = const <String>[]})
TaskResult.success(this.data, {
this.benchmarkScoreKeys = const <String>[],
this.detailFiles,
})
: succeeded = true,
message = 'success' {
const JsonEncoder prettyJson = JsonEncoder.withIndent(' ');
......@@ -219,6 +222,7 @@ class TaskResult {
TaskResult.failure(this.message)
: succeeded = false,
data = null,
detailFiles = null,
benchmarkScoreKeys = const <String>[];
/// Whether the task succeeded.
......@@ -227,6 +231,9 @@ class TaskResult {
/// Task-specific JSON data
final Map<String, dynamic> data;
/// Files containing detail on the run (e.g. timeline trace files)
final List<String> detailFiles;
/// Keys in [data] that store scores that will be submitted to Cocoon.
///
/// Each key is also part of a benchmark's name tracked by Cocoon.
......@@ -245,6 +252,7 @@ class TaskResult {
/// {
/// "success": true|false,
/// "data": arbitrary JSON data valid only for successful results,
/// "detailFiles": list of filenames containing detail on the run
/// "benchmarkScoreKeys": [
/// contains keys into "data" that represent benchmarks scores, which
/// can be uploaded, for example. to golem, valid only for successful
......@@ -259,6 +267,8 @@ class TaskResult {
if (succeeded) {
json['data'] = data;
if (detailFiles != null)
json['detailFiles'] = detailFiles;
json['benchmarkScoreKeys'] = benchmarkScoreKeys;
} else {
json['reason'] = message;
......
......@@ -23,6 +23,7 @@ TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) {
needFullTimeline: false,
timelineSummaryFile: 'e2e_perf_summary',
transitionDurationFile: null,
timelineTraceFile: null,
driverFile: 'transitions_perf_e2e_test',
);
}
......@@ -43,6 +44,7 @@ class GalleryTransitionTest {
this.testFile = 'transitions_perf',
this.needFullTimeline = true,
this.timelineSummaryFile = 'transitions.timeline_summary',
this.timelineTraceFile = 'transitions.timeline',
this.transitionDurationFile = 'transition_durations.timeline',
this.driverFile,
});
......@@ -51,6 +53,7 @@ class GalleryTransitionTest {
final bool needFullTimeline;
final String testFile;
final String timelineSummaryFile;
final String timelineTraceFile;
final String transitionDurationFile;
final String driverFile;
......@@ -95,19 +98,28 @@ class GalleryTransitionTest {
summary['transitions'] = transitions;
summary['missed_transition_count'] = _countMissedTransitions(transitions);
}
return TaskResult.success(summary, benchmarkScoreKeys: <String>[
final List<String> detailFiles = <String>[
if (transitionDurationFile != null)
'missed_transition_count',
'average_frame_build_time_millis',
'worst_frame_build_time_millis',
'90th_percentile_frame_build_time_millis',
'99th_percentile_frame_build_time_millis',
'average_frame_rasterizer_time_millis',
'worst_frame_rasterizer_time_millis',
'90th_percentile_frame_rasterizer_time_millis',
'99th_percentile_frame_rasterizer_time_millis',
]);
'${galleryDirectory.path}/build/$transitionDurationFile.json',
if (timelineTraceFile != null)
'${galleryDirectory.path}/build/$timelineTraceFile.json'
];
return TaskResult.success(summary,
detailFiles: detailFiles.isNotEmpty ? detailFiles : null,
benchmarkScoreKeys: <String>[
if (transitionDurationFile != null)
'missed_transition_count',
'average_frame_build_time_millis',
'worst_frame_build_time_millis',
'90th_percentile_frame_build_time_millis',
'99th_percentile_frame_build_time_millis',
'average_frame_rasterizer_time_millis',
'worst_frame_rasterizer_time_millis',
'90th_percentile_frame_rasterizer_time_millis',
'99th_percentile_frame_rasterizer_time_millis',
],
);
}
}
......
......@@ -116,6 +116,7 @@ TaskFunction createBackdropFilterPerfTest({bool measureCpuGpu = false}) {
'backdrop_filter_perf',
measureCpuGpu: measureCpuGpu,
testDriver: 'test_driver/backdrop_filter_perf_test.dart',
saveTraceFile: true,
).run;
}
......@@ -126,6 +127,7 @@ TaskFunction createPostBackdropFilterPerfTest({bool measureCpuGpu = false}) {
'post_backdrop_filter_perf',
measureCpuGpu: measureCpuGpu,
testDriver: 'test_driver/post_backdrop_filter_perf_test.dart',
saveTraceFile: true,
).run;
}
......@@ -136,6 +138,7 @@ TaskFunction createSimpleAnimationPerfTest({bool measureCpuGpu = false}) {
'simple_animation_perf',
measureCpuGpu: measureCpuGpu,
testDriver: 'test_driver/simple_animation_perf_test.dart',
saveTraceFile: true,
).run;
}
......@@ -247,6 +250,7 @@ TaskFunction createColorFilterAndFadePerfTest() {
'test_driver/run_app.dart',
'color_filter_and_fade_perf',
testDriver: 'test_driver/color_filter_and_fade_perf_test.dart',
saveTraceFile: true,
).run;
}
......@@ -256,6 +260,7 @@ TaskFunction createFadingChildAnimationPerfTest() {
'test_driver/run_app.dart',
'fading_child_animation_perf',
testDriver: 'test_driver/fading_child_animation_perf_test.dart',
saveTraceFile: true,
).run;
}
......@@ -265,6 +270,7 @@ TaskFunction createImageFilteredTransformAnimationPerfTest() {
'test_driver/run_app.dart',
'imagefiltered_transform_animation_perf',
testDriver: 'test_driver/imagefiltered_transform_animation_perf_test.dart',
saveTraceFile: true,
).run;
}
......@@ -369,6 +375,7 @@ class PerfTest {
this.testTarget,
this.timelineFileName, {
this.measureCpuGpu = false,
this.saveTraceFile = false,
this.testDriver,
this.needsFullTimeline = true,
this.benchmarkScoreKeys,
......@@ -382,12 +389,15 @@ class PerfTest {
// The prefix name of the filename such as `<timelineFileName>.timeline_summary.json`.
final String timelineFileName;
String get resultFilename => '$timelineFileName.timeline_summary';
String get traceFilename => '$timelineFileName.timeline';
/// The test file to run on the host.
final String testDriver;
/// Whether to collect CPU and GPU metrics.
final bool measureCpuGpu;
/// Whether to collect full timeline, meaning if `--trace-startup` flag is needed.
final bool needsFullTimeline;
/// Whether to save the trace timeline file `*.timeline.json`.
final bool saveTraceFile;
/// The keys of the values that need to be reported.
///
......@@ -454,6 +464,10 @@ class PerfTest {
final Map<String, dynamic> data = json.decode(
file('$testDirectory/build/$resultFilename.json').readAsStringSync(),
) as Map<String, dynamic>;
final List<String> detailFiles = <String>[
if (saveTraceFile)
'$testDirectory/build/$traceFilename.json',
];
if (data['frame_count'] as int < 5) {
return TaskResult.failure(
......@@ -464,6 +478,7 @@ class PerfTest {
return TaskResult.success(
data,
detailFiles: detailFiles.isNotEmpty ? detailFiles : null,
benchmarkScoreKeys: benchmarkScoreKeys ?? <String>[
'average_frame_build_time_millis',
'worst_frame_build_time_millis',
......
......@@ -204,6 +204,7 @@ void main([List<String> args = const <String>[]]) {
// 'Start Transition' event when a demo is launched (see GalleryItem).
final TimelineSummary summary = TimelineSummary.summarize(timeline);
await summary.writeSummaryToFile('transitions', pretty: true);
await summary.writeTimelineToFile('transitions', pretty: true);
final String histogramPath = path.join(testOutputsDirectory, 'transition_durations.timeline.json');
await saveDurationsHistogram(
List<Map<String, dynamic>>.from(timeline.json['traceEvents'] as List<dynamic>),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment