coverage_collector.dart 12.5 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';

7
import 'package:coverage/coverage.dart' as coverage;
8
import 'package:vm_service/vm_service.dart' as vm_service;
9

10 11
import '../base/file_system.dart';
import '../base/io.dart';
12
import '../base/process.dart';
13
import '../base/utils.dart';
14
import '../dart/package_map.dart';
15
import '../globals.dart' as globals;
16
import '../vmservice.dart';
17

18 19
import 'watcher.dart';

20
/// A class that's used to collect coverage data during tests.
21
class CoverageCollector extends TestWatcher {
22
  CoverageCollector({this.libraryPredicate, this.verbose = true});
23

24
  final bool verbose;
25
  Map<String, Map<int, int>> _globalHitmap;
26
  bool Function(String) libraryPredicate;
27

28
  @override
29
  Future<void> handleFinishedTest(ProcessEvent event) async {
30
    _logMessage('test ${event.childIndex}: collecting coverage');
31 32 33
    await collectCoverage(event.process, event.observatoryUri);
  }

34 35 36 37 38 39 40 41 42 43 44
  void _logMessage(String line, { bool error = false }) {
    if (!verbose) {
      return;
    }
    if (error) {
      globals.printError(line);
    } else {
      globals.printTrace(line);
    }
  }

45
  void _addHitmap(Map<String, Map<int, int>> hitmap) {
46
    if (_globalHitmap == null) {
47
      _globalHitmap = hitmap;
48
    } else {
49
      coverage.mergeHitmaps(hitmap, _globalHitmap);
50
    }
51 52
  }

53 54 55 56 57 58
  /// Collects coverage for an isolate using the given `port`.
  ///
  /// This should be called when the code whose coverage data is being collected
  /// has been run to completion so that all coverage data has been recorded.
  ///
  /// The returned [Future] completes when the coverage is collected.
59
  Future<void> collectCoverageIsolate(Uri observatoryUri) async {
60
    assert(observatoryUri != null);
61
    _logMessage('collecting coverage data from $observatoryUri...');
62
    final Map<String, dynamic> data = await collect(observatoryUri, libraryPredicate);
63 64 65 66 67
    if (data == null) {
      throw Exception('Failed to collect coverage.');
    }
    assert(data != null);

68
    _logMessage('($observatoryUri): collected coverage data; merging...');
69
    _addHitmap(coverage.createHitmap(data['coverage'] as List<Map<String, dynamic>>));
70
    _logMessage('($observatoryUri): done merging coverage data into global coverage map.');
71 72
  }

73
  /// Collects coverage for the given [Process] using the given `port`.
74
  ///
75 76 77 78
  /// This should be called when the code whose coverage data is being collected
  /// has been run to completion so that all coverage data has been recorded.
  ///
  /// The returned [Future] completes when the coverage is collected.
79
  Future<void> collectCoverage(Process process, Uri observatoryUri) async {
80
    assert(process != null);
81
    assert(observatoryUri != null);
82
    final int pid = process.pid;
83
    _logMessage('pid $pid: collecting coverage data from $observatoryUri...');
84 85 86 87

    Map<String, dynamic> data;
    final Future<void> processComplete = process.exitCode
      .then<void>((int code) {
88
        throw Exception('Failed to collect coverage, process terminated prematurely with exit code $code.');
89
      });
90
    final Future<void> collectionComplete = collect(observatoryUri, libraryPredicate)
91
      .then<void>((Map<String, dynamic> result) {
92
        if (result == null) {
93
          throw Exception('Failed to collect coverage.');
94
        }
95
        data = result;
96
      });
97 98 99
    await Future.any<void>(<Future<void>>[ processComplete, collectionComplete ]);
    assert(data != null);

100
    _logMessage('pid $pid ($observatoryUri): collected coverage data; merging...');
101
    _addHitmap(coverage.createHitmap(data['coverage'] as List<Map<String, dynamic>>));
102
    _logMessage('pid $pid ($observatoryUri): done merging coverage data into global coverage map.');
103 104
  }

105 106 107
  /// Returns a future that will complete with the formatted coverage data
  /// (using [formatter]) once all coverage data has been collected.
  ///
108 109
  /// This will not start any collection tasks. It us up to the caller of to
  /// call [collectCoverage] for each process first.
110
  Future<String> finalizeCoverage({
111
    coverage.Formatter formatter,
112
    Directory coverageDirectory,
113
  }) async {
114
    if (_globalHitmap == null) {
115
      return null;
116
    }
117
    if (formatter == null) {
118
      final coverage.Resolver resolver = coverage.Resolver(packagesPath: globalPackagesPath);
119
      final String packagePath = globals.fs.currentDirectory.path;
120
      final List<String> reportOn = coverageDirectory == null
121
        ? <String>[globals.fs.path.join(packagePath, 'lib')]
122
        : <String>[coverageDirectory.path];
123
      formatter = coverage.LcovFormatter(resolver, reportOn: reportOn, basePath: packagePath);
124
    }
125
    final String result = await formatter.format(_globalHitmap);
126 127
    _globalHitmap = null;
    return result;
128
  }
129

130
  Future<bool> collectCoverageData(String coveragePath, { bool mergeCoverageData = false, Directory coverageDirectory }) async {
131
    final String coverageData = await finalizeCoverage(
132
      coverageDirectory: coverageDirectory,
133
    );
134
    _logMessage('coverage information collection complete');
135
    if (coverageData == null) {
136
      return false;
137
    }
138

139
    final File coverageFile = globals.fs.file(coveragePath)
140 141
      ..createSync(recursive: true)
      ..writeAsStringSync(coverageData, flush: true);
142
    _logMessage('wrote coverage data to $coveragePath (size=${coverageData.length})');
143 144 145

    const String baseCoverageData = 'coverage/lcov.base.info';
    if (mergeCoverageData) {
146
      if (!globals.fs.isFileSync(baseCoverageData)) {
147
        _logMessage('Missing "$baseCoverageData". Unable to merge coverage data.', error: true);
148 149 150
        return false;
      }

151
      if (globals.os.which('lcov') == null) {
152
        String installMessage = 'Please install lcov.';
153
        if (globals.platform.isLinux) {
154
          installMessage = 'Consider running "sudo apt-get install lcov".';
155
        } else if (globals.platform.isMacOS) {
156
          installMessage = 'Consider running "brew install lcov".';
157
        }
158
        _logMessage('Missing "lcov" tool. Unable to merge coverage data.\n$installMessage', error: true);
159 160 161
        return false;
      }

162
      final Directory tempDir = globals.fs.systemTempDirectory.createTempSync('flutter_tools_test_coverage.');
163
      try {
164
        final File sourceFile = coverageFile.copySync(globals.fs.path.join(tempDir.path, 'lcov.source.info'));
165
        final RunResult result = processUtils.runSync(<String>[
166 167 168 169 170
          'lcov',
          '--add-tracefile', baseCoverageData,
          '--add-tracefile', sourceFile.path,
          '--output-file', coverageFile.path,
        ]);
171
        if (result.exitCode != 0) {
172
          return false;
173
        }
174 175 176 177 178 179
      } finally {
        tempDir.deleteSync(recursive: true);
      }
    }
    return true;
  }
180 181 182 183 184 185

  @override
  Future<void> handleTestCrashed(ProcessEvent event) async { }

  @override
  Future<void> handleTestTimedOut(ProcessEvent event) async { }
186
}
187

188 189
Future<vm_service.VmService> _defaultConnect(Uri serviceUri) {
  return connectToVmService(
190
      serviceUri, compression: CompressionOptions.compressionOff);
191 192
}

193 194 195
Future<Map<String, dynamic>> collect(Uri serviceUri, bool Function(String) libraryPredicate, {
  bool waitPaused = false,
  String debugName,
196
  Future<vm_service.VmService> Function(Uri) connector = _defaultConnect,
197
}) async {
198
  final vm_service.VmService vmService = await connector(serviceUri);
199 200
  final Map<String, dynamic> result = await _getAllCoverage(
      vmService, libraryPredicate);
201
  vmService.dispose();
202
  return result;
203 204
}

205 206
Future<Map<String, dynamic>> _getAllCoverage(vm_service.VmService service, bool Function(String) libraryPredicate) async {
  final vm_service.VM vm = await service.getVM();
207
  final List<Map<String, dynamic>> coverage = <Map<String, dynamic>>[];
208
  for (final vm_service.IsolateRef isolateRef in vm.isolates) {
209 210 211 212 213 214 215
    Map<String, Object> scriptList;
    try {
      final vm_service.ScriptList actualScriptList = await service.getScripts(isolateRef.id);
      scriptList = actualScriptList.json;
    } on vm_service.SentinelException {
      continue;
    }
216 217 218 219 220 221
    final List<Future<void>> futures = <Future<void>>[];

    final Map<String, Map<String, dynamic>> scripts = <String, Map<String, dynamic>>{};
    final Map<String, Map<String, dynamic>> sourceReports = <String, Map<String, dynamic>>{};
    // For each ScriptRef loaded into the VM, load the corresponding Script and
    // SourceReport object.
222

223
    for (final Map<String, dynamic> script in (scriptList['scripts'] as List<dynamic>).cast<Map<String, dynamic>>()) {
224
      if (!libraryPredicate(script['uri'] as String)) {
225 226
        continue;
      }
227
      final String scriptId = script['id'] as String;
228
      futures.add(
229 230 231 232 233 234 235 236
        service.getSourceReport(
          isolateRef.id,
          <String>['Coverage'],
          scriptId: scriptId,
          forceCompile: true,
        )
        .then((vm_service.SourceReport report) {
          sourceReports[scriptId] = report.json;
237 238 239
        })
      );
      futures.add(
240 241 242 243 244
        service
          .getObject(isolateRef.id, scriptId)
          .then((vm_service.Obj script) {
            scripts[scriptId] = script.json;
          })
245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
      );
    }
    await Future.wait(futures);
    _buildCoverageMap(scripts, sourceReports, coverage);
  }
  return <String, dynamic>{'type': 'CodeCoverage', 'coverage': coverage};
}

// Build a hitmap of Uri -> Line -> Hit Count for each script object.
void _buildCoverageMap(
  Map<String, Map<String, dynamic>> scripts,
  Map<String, Map<String, dynamic>> sourceReports,
  List<Map<String, dynamic>> coverage,
) {
  final Map<String, Map<int, int>> hitMaps = <String, Map<int, int>>{};
260
  for (final String scriptId in scripts.keys) {
261
    final Map<String, dynamic> sourceReport = sourceReports[scriptId];
262
    for (final Map<String, dynamic> range in (sourceReport['ranges'] as List<dynamic>).cast<Map<String, dynamic>>()) {
263
      final Map<String, dynamic> coverage = castStringKeyedMap(range['coverage']);
264
      // Coverage reports may sometimes be null for a Script.
265
      if (coverage == null) {
266 267
        continue;
      }
268 269
      final Map<String, dynamic> scriptRef = castStringKeyedMap(sourceReport['scripts'][range['scriptIndex']]);
      final String uri = scriptRef['uri'] as String;
270

271 272
      hitMaps[uri] ??= <int, int>{};
      final Map<int, int> hitMap = hitMaps[uri];
273 274 275
      final List<int> hits = (coverage['hits'] as List<dynamic>).cast<int>();
      final List<int> misses = (coverage['misses'] as List<dynamic>).cast<int>();
      final List<dynamic> tokenPositions = scripts[scriptRef['id']]['tokenPosTable'] as List<dynamic>;
276 277 278 279
      // The token positions can be null if the script has no coverable lines.
      if (tokenPositions == null) {
        continue;
      }
280
      if (hits != null) {
281
        for (final int hit in hits) {
282 283 284 285 286 287
          final int line = _lineAndColumn(hit, tokenPositions)[0];
          final int current = hitMap[line] ?? 0;
          hitMap[line] = current + 1;
        }
      }
      if (misses != null) {
288
        for (final int miss in misses) {
289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307
          final int line = _lineAndColumn(miss, tokenPositions)[0];
          hitMap[line] ??= 0;
        }
      }
    }
  }
  hitMaps.forEach((String uri, Map<int, int> hitMap) {
    coverage.add(_toScriptCoverageJson(uri, hitMap));
  });
}

// Binary search the token position table for the line and column which
// corresponds to each token position.
// The format of this table is described in https://github.com/dart-lang/sdk/blob/master/runtime/vm/service/service.md#script
List<int> _lineAndColumn(int position, List<dynamic> tokenPositions) {
  int min = 0;
  int max = tokenPositions.length;
  while (min < max) {
    final int mid = min + ((max - min) >> 1);
308
    final List<int> row = (tokenPositions[mid] as List<dynamic>).cast<int>();
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
    if (row[1] > position) {
      max = mid;
    } else {
      for (int i = 1; i < row.length; i += 2) {
        if (row[i] == position) {
          return <int>[row.first, row[i + 1]];
        }
      }
      min = mid + 1;
    }
  }
  throw StateError('Unreachable');
}

// Returns a JSON hit map backward-compatible with pre-1.16.0 SDKs.
Map<String, dynamic> _toScriptCoverageJson(String scriptUri, Map<int, int> hitMap) {
  final Map<String, dynamic> json = <String, dynamic>{};
  final List<int> hits = <int>[];
  hitMap.forEach((int line, int hitCount) {
    hits.add(line);
    hits.add(hitCount);
  });
  json['source'] = scriptUri;
  json['script'] = <String, dynamic>{
    'type': '@Script',
    'fixedId': true,
    'id': 'libraries/1/scripts/${Uri.encodeComponent(scriptUri)}',
    'uri': scriptUri,
    '_kind': 'library',
  };
  json['hits'] = hits;
  return json;
}