coverage_collector.dart 12.6 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
import 'package:coverage/coverage.dart' as coverage;
6
import 'package:vm_service/vm_service.dart' as vm_service;
7

8 9
import '../base/file_system.dart';
import '../base/io.dart';
10
import '../base/process.dart';
11
import '../base/utils.dart';
12
import '../dart/package_map.dart';
13
import '../globals.dart' as globals;
14
import '../vmservice.dart';
15

16 17
import 'watcher.dart';

18
/// A class that's used to collect coverage data during tests.
19
class CoverageCollector extends TestWatcher {
20
  CoverageCollector({this.libraryPredicate, this.verbose = true});
21

22
  final bool verbose;
23
  Map<String, Map<int, int>> _globalHitmap;
24
  bool Function(String) libraryPredicate;
25

26
  @override
27
  Future<void> handleFinishedTest(ProcessEvent event) async {
28
    _logMessage('test ${event.childIndex}: collecting coverage');
29 30 31
    await collectCoverage(event.process, event.observatoryUri);
  }

32 33 34 35 36 37 38 39 40 41 42
  void _logMessage(String line, { bool error = false }) {
    if (!verbose) {
      return;
    }
    if (error) {
      globals.printError(line);
    } else {
      globals.printTrace(line);
    }
  }

43
  void _addHitmap(Map<String, Map<int, int>> hitmap) {
44
    if (_globalHitmap == null) {
45
      _globalHitmap = hitmap;
46
    } else {
47
      coverage.mergeHitmaps(hitmap, _globalHitmap);
48
    }
49 50
  }

51 52 53 54 55 56
  /// Collects coverage for an isolate using the given `port`.
  ///
  /// This should be called when the code whose coverage data is being collected
  /// has been run to completion so that all coverage data has been recorded.
  ///
  /// The returned [Future] completes when the coverage is collected.
57
  Future<void> collectCoverageIsolate(Uri observatoryUri) async {
58
    assert(observatoryUri != null);
59
    _logMessage('collecting coverage data from $observatoryUri...');
60
    final Map<String, dynamic> data = await collect(observatoryUri, libraryPredicate);
61 62 63 64 65
    if (data == null) {
      throw Exception('Failed to collect coverage.');
    }
    assert(data != null);

66
    _logMessage('($observatoryUri): collected coverage data; merging...');
67 68 69 70 71
    _addHitmap(await coverage.createHitmap(
      data['coverage'] as List<Map<String, dynamic>>,
      packagesPath: globalPackagesPath,
      checkIgnoredLines: true,
    ));
72
    _logMessage('($observatoryUri): done merging coverage data into global coverage map.');
73 74
  }

75
  /// Collects coverage for the given [Process] using the given `port`.
76
  ///
77 78 79 80
  /// This should be called when the code whose coverage data is being collected
  /// has been run to completion so that all coverage data has been recorded.
  ///
  /// The returned [Future] completes when the coverage is collected.
81
  Future<void> collectCoverage(Process process, Uri observatoryUri) async {
82
    assert(process != null);
83
    assert(observatoryUri != null);
84
    final int pid = process.pid;
85
    _logMessage('pid $pid: collecting coverage data from $observatoryUri...');
86 87 88 89

    Map<String, dynamic> data;
    final Future<void> processComplete = process.exitCode
      .then<void>((int code) {
90
        throw Exception('Failed to collect coverage, process terminated prematurely with exit code $code.');
91
      });
92
    final Future<void> collectionComplete = collect(observatoryUri, libraryPredicate)
93
      .then<void>((Map<String, dynamic> result) {
94
        if (result == null) {
95
          throw Exception('Failed to collect coverage.');
96
        }
97
        data = result;
98
      });
99 100 101
    await Future.any<void>(<Future<void>>[ processComplete, collectionComplete ]);
    assert(data != null);

102
    _logMessage('pid $pid ($observatoryUri): collected coverage data; merging...');
103 104 105 106 107
    _addHitmap(await coverage.createHitmap(
      data['coverage'] as List<Map<String, dynamic>>,
      packagesPath: globalPackagesPath,
      checkIgnoredLines: true,
    ));
108
    _logMessage('pid $pid ($observatoryUri): done merging coverage data into global coverage map.');
109 110
  }

111 112 113
  /// Returns a future that will complete with the formatted coverage data
  /// (using [formatter]) once all coverage data has been collected.
  ///
114 115
  /// This will not start any collection tasks. It us up to the caller of to
  /// call [collectCoverage] for each process first.
116
  Future<String> finalizeCoverage({
117
    coverage.Formatter formatter,
118
    Directory coverageDirectory,
119
  }) async {
120
    if (_globalHitmap == null) {
121
      return null;
122
    }
123
    if (formatter == null) {
124
      final coverage.Resolver resolver = coverage.Resolver(packagesPath: globalPackagesPath);
125
      final String packagePath = globals.fs.currentDirectory.path;
126
      final List<String> reportOn = coverageDirectory == null
127
        ? <String>[globals.fs.path.join(packagePath, 'lib')]
128
        : <String>[coverageDirectory.path];
129
      formatter = coverage.LcovFormatter(resolver, reportOn: reportOn, basePath: packagePath);
130
    }
131
    final String result = await formatter.format(_globalHitmap);
132 133
    _globalHitmap = null;
    return result;
134
  }
135

136
  Future<bool> collectCoverageData(String coveragePath, { bool mergeCoverageData = false, Directory coverageDirectory }) async {
137
    final String coverageData = await finalizeCoverage(
138
      coverageDirectory: coverageDirectory,
139
    );
140
    _logMessage('coverage information collection complete');
141
    if (coverageData == null) {
142
      return false;
143
    }
144

145
    final File coverageFile = globals.fs.file(coveragePath)
146 147
      ..createSync(recursive: true)
      ..writeAsStringSync(coverageData, flush: true);
148
    _logMessage('wrote coverage data to $coveragePath (size=${coverageData.length})');
149 150 151

    const String baseCoverageData = 'coverage/lcov.base.info';
    if (mergeCoverageData) {
152
      if (!globals.fs.isFileSync(baseCoverageData)) {
153
        _logMessage('Missing "$baseCoverageData". Unable to merge coverage data.', error: true);
154 155 156
        return false;
      }

157
      if (globals.os.which('lcov') == null) {
158
        String installMessage = 'Please install lcov.';
159
        if (globals.platform.isLinux) {
160
          installMessage = 'Consider running "sudo apt-get install lcov".';
161
        } else if (globals.platform.isMacOS) {
162
          installMessage = 'Consider running "brew install lcov".';
163
        }
164
        _logMessage('Missing "lcov" tool. Unable to merge coverage data.\n$installMessage', error: true);
165 166 167
        return false;
      }

168
      final Directory tempDir = globals.fs.systemTempDirectory.createTempSync('flutter_tools_test_coverage.');
169
      try {
170
        final File sourceFile = coverageFile.copySync(globals.fs.path.join(tempDir.path, 'lcov.source.info'));
171
        final RunResult result = globals.processUtils.runSync(<String>[
172 173 174 175 176
          'lcov',
          '--add-tracefile', baseCoverageData,
          '--add-tracefile', sourceFile.path,
          '--output-file', coverageFile.path,
        ]);
177
        if (result.exitCode != 0) {
178
          return false;
179
        }
180 181 182 183 184 185
      } finally {
        tempDir.deleteSync(recursive: true);
      }
    }
    return true;
  }
186 187 188 189 190 191

  @override
  Future<void> handleTestCrashed(ProcessEvent event) async { }

  @override
  Future<void> handleTestTimedOut(ProcessEvent event) async { }
192
}
193

194 195
Future<vm_service.VmService> _defaultConnect(Uri serviceUri) {
  return connectToVmService(
196
      serviceUri, compression: CompressionOptions.compressionOff);
197 198
}

199 200 201
Future<Map<String, dynamic>> collect(Uri serviceUri, bool Function(String) libraryPredicate, {
  bool waitPaused = false,
  String debugName,
202
  Future<vm_service.VmService> Function(Uri) connector = _defaultConnect,
203
}) async {
204
  final vm_service.VmService vmService = await connector(serviceUri);
205 206
  final Map<String, dynamic> result = await _getAllCoverage(
      vmService, libraryPredicate);
207
  vmService.dispose();
208
  return result;
209 210
}

211 212
Future<Map<String, dynamic>> _getAllCoverage(vm_service.VmService service, bool Function(String) libraryPredicate) async {
  final vm_service.VM vm = await service.getVM();
213
  final List<Map<String, dynamic>> coverage = <Map<String, dynamic>>[];
214
  for (final vm_service.IsolateRef isolateRef in vm.isolates) {
215 216 217 218 219 220 221
    Map<String, Object> scriptList;
    try {
      final vm_service.ScriptList actualScriptList = await service.getScripts(isolateRef.id);
      scriptList = actualScriptList.json;
    } on vm_service.SentinelException {
      continue;
    }
222 223 224 225 226 227
    final List<Future<void>> futures = <Future<void>>[];

    final Map<String, Map<String, dynamic>> scripts = <String, Map<String, dynamic>>{};
    final Map<String, Map<String, dynamic>> sourceReports = <String, Map<String, dynamic>>{};
    // For each ScriptRef loaded into the VM, load the corresponding Script and
    // SourceReport object.
228

229
    for (final Map<String, dynamic> script in (scriptList['scripts'] as List<dynamic>).cast<Map<String, dynamic>>()) {
230
      if (!libraryPredicate(script['uri'] as String)) {
231 232
        continue;
      }
233
      final String scriptId = script['id'] as String;
234
      futures.add(
235 236 237 238 239 240 241 242
        service.getSourceReport(
          isolateRef.id,
          <String>['Coverage'],
          scriptId: scriptId,
          forceCompile: true,
        )
        .then((vm_service.SourceReport report) {
          sourceReports[scriptId] = report.json;
243 244 245
        })
      );
      futures.add(
246 247 248 249 250
        service
          .getObject(isolateRef.id, scriptId)
          .then((vm_service.Obj script) {
            scripts[scriptId] = script.json;
          })
251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
      );
    }
    await Future.wait(futures);
    _buildCoverageMap(scripts, sourceReports, coverage);
  }
  return <String, dynamic>{'type': 'CodeCoverage', 'coverage': coverage};
}

// Build a hitmap of Uri -> Line -> Hit Count for each script object.
void _buildCoverageMap(
  Map<String, Map<String, dynamic>> scripts,
  Map<String, Map<String, dynamic>> sourceReports,
  List<Map<String, dynamic>> coverage,
) {
  final Map<String, Map<int, int>> hitMaps = <String, Map<int, int>>{};
266
  for (final String scriptId in scripts.keys) {
267
    final Map<String, dynamic> sourceReport = sourceReports[scriptId];
268
    for (final Map<String, dynamic> range in (sourceReport['ranges'] as List<dynamic>).cast<Map<String, dynamic>>()) {
269
      final Map<String, dynamic> coverage = castStringKeyedMap(range['coverage']);
270
      // Coverage reports may sometimes be null for a Script.
271
      if (coverage == null) {
272 273
        continue;
      }
274 275
      final Map<String, dynamic> scriptRef = castStringKeyedMap(sourceReport['scripts'][range['scriptIndex']]);
      final String uri = scriptRef['uri'] as String;
276

277 278
      hitMaps[uri] ??= <int, int>{};
      final Map<int, int> hitMap = hitMaps[uri];
279 280 281
      final List<int> hits = (coverage['hits'] as List<dynamic>).cast<int>();
      final List<int> misses = (coverage['misses'] as List<dynamic>).cast<int>();
      final List<dynamic> tokenPositions = scripts[scriptRef['id']]['tokenPosTable'] as List<dynamic>;
282
      // The token positions can be null if the script has no lines that may be covered.
283 284 285
      if (tokenPositions == null) {
        continue;
      }
286
      if (hits != null) {
287
        for (final int hit in hits) {
288 289 290 291 292 293
          final int line = _lineAndColumn(hit, tokenPositions)[0];
          final int current = hitMap[line] ?? 0;
          hitMap[line] = current + 1;
        }
      }
      if (misses != null) {
294
        for (final int miss in misses) {
295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313
          final int line = _lineAndColumn(miss, tokenPositions)[0];
          hitMap[line] ??= 0;
        }
      }
    }
  }
  hitMaps.forEach((String uri, Map<int, int> hitMap) {
    coverage.add(_toScriptCoverageJson(uri, hitMap));
  });
}

// Binary search the token position table for the line and column which
// corresponds to each token position.
// The format of this table is described in https://github.com/dart-lang/sdk/blob/master/runtime/vm/service/service.md#script
List<int> _lineAndColumn(int position, List<dynamic> tokenPositions) {
  int min = 0;
  int max = tokenPositions.length;
  while (min < max) {
    final int mid = min + ((max - min) >> 1);
314
    final List<int> row = (tokenPositions[mid] as List<dynamic>).cast<int>();
315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
    if (row[1] > position) {
      max = mid;
    } else {
      for (int i = 1; i < row.length; i += 2) {
        if (row[i] == position) {
          return <int>[row.first, row[i + 1]];
        }
      }
      min = mid + 1;
    }
  }
  throw StateError('Unreachable');
}

// Returns a JSON hit map backward-compatible with pre-1.16.0 SDKs.
Map<String, dynamic> _toScriptCoverageJson(String scriptUri, Map<int, int> hitMap) {
  final Map<String, dynamic> json = <String, dynamic>{};
  final List<int> hits = <int>[];
  hitMap.forEach((int line, int hitCount) {
    hits.add(line);
    hits.add(hitCount);
  });
  json['source'] = scriptUri;
  json['script'] = <String, dynamic>{
    'type': '@Script',
    'fixedId': true,
    'id': 'libraries/1/scripts/${Uri.encodeComponent(scriptUri)}',
    'uri': scriptUri,
    '_kind': 'library',
  };
  json['hits'] = hits;
  return json;
}