Bug 1473313 - Part 3: Extend CodeCoverageMixin to handle java code coverage tools. r=marco,gbrown
MozReview-Commit-ID: Lld5XLUMb3Y
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -721,16 +721,22 @@ def handle_suite_category(config, tests)
def enable_code_coverage(config, tests):
"""Enable code coverage for the ccov and jsdcov build-platforms"""
for test in tests:
if 'ccov' in test['build-platform']:
# do not run tests on fuzzing or opt build
if 'opt' in test['build-platform'] or 'fuzzing' in test['build-platform']:
test['run-on-projects'] = []
continue
+ # Skip this transform for android code coverage builds.
+ if 'android' in test['build-platform']:
+ test.setdefault('fetches', {}).setdefault('fetch', []).append('grcov-linux-x86_64')
+ test['mozharness'].setdefault('extra-options', []).append('--java-code-coverage')
+ yield test
+ continue
test['mozharness'].setdefault('extra-options', []).append('--code-coverage')
test['instance-size'] = 'xlarge'
# Ensure we always run on the projects defined by the build, unless the test
# is try only or shouldn't run at all.
if test['run-on-projects'] not in [[], ['try']]:
test['run-on-projects'] = 'built-projects'
# Ensure we don't optimize test suites out.
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
@@ -40,16 +40,22 @@ code_coverage_config_options = [
"help": "Whether test run should package and upload code coverage data."
}],
[["--jsd-code-coverage"],
{"action": "store_true",
"dest": "jsd_code_coverage",
"default": False,
"help": "Whether JSDebugger code coverage should be run."
}],
+ [["--java-code-coverage"],
+ {"action": "store_true",
+ "dest": "java_code_coverage",
+ "default": False,
+ "help": "Whether Java code coverage should be run."
+ }],
]
class CodeCoverageMixin(SingleTestMixin):
"""
Mixin for setting GCOV_PREFIX during test execution, packaging up
the resulting .gcda files and uploading them to blobber.
"""
@@ -84,16 +90,23 @@ class CodeCoverageMixin(SingleTestMixin)
@property
def jsd_code_coverage_enabled(self):
try:
return bool(self.config.get('jsd_code_coverage'))
except (AttributeError, KeyError, TypeError):
return False
+ @property
+ def java_code_coverage_enabled(self):
+ try:
+ return bool(self.config.get('java_code_coverage'))
+ except (AttributeError, KeyError, TypeError):
+ return False
+
@PostScriptAction('download-and-extract')
def setup_coverage_tools(self, action, success=None):
if not self.code_coverage_enabled:
return
if mozinfo.os == 'linux' or mozinfo.os == 'mac':
self.prefix = '/builds/worker/workspace/build/src/'
strip_count = self.prefix.count('/')
@@ -108,35 +121,39 @@ class CodeCoverageMixin(SingleTestMixin)
os.environ['GCOV_PREFIX_STRIP'] = str(strip_count)
# Install grcov on the test machine
# Get the path to the build machines gcno files.
self.url_to_gcno = self.query_build_dir_url('target.code-coverage-gcno.zip')
self.url_to_chrome_map = self.query_build_dir_url('chrome-map.json')
# Create the grcov directory, then download it.
- # TODO: use the fetch-content script to download artifacts.
+ # Note: We assume grcov is our only artifact.
self.grcov_dir = tempfile.mkdtemp()
- ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{task}/artifacts/{artifact}'
- for word in os.getenv('MOZ_FETCHES').split():
- artifact, task = word.split('@', 1)
- filename = os.path.basename(artifact)
- url = ARTIFACT_URL.format(artifact=artifact, task=task)
- self.download_file(url, parent_dir=self.grcov_dir)
-
- with tarfile.open(os.path.join(self.grcov_dir, filename), 'r') as tar:
- tar.extractall(self.grcov_dir)
- os.remove(os.path.join(self.grcov_dir, filename))
+ self._download_artifacts(self.grcov_dir)
# Download the gcno archive from the build machine.
self.download_file(self.url_to_gcno, parent_dir=self.grcov_dir)
# Download the chrome-map.json file from the build machine.
self.download_file(self.url_to_chrome_map, parent_dir=self.grcov_dir)
+ def _download_artifacts(self, destination):
+ # TODO: use the fetch-content script to download artifacts.
+ ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{task}/artifacts/{artifact}'
+ for word in os.getenv('MOZ_FETCHES').split():
+ artifact, task = word.split('@', 1)
+ filename = os.path.basename(artifact)
+ url = ARTIFACT_URL.format(artifact=artifact, task=task)
+ self.download_file(url, parent_dir=destination)
+
+ with tarfile.open(os.path.join(destination, filename), 'r') as tar:
+ tar.extractall(destination)
+ os.remove(os.path.join(destination, filename))
+
@PostScriptAction('download-and-extract')
def find_tests_for_coverage(self, action, success=None):
"""
For each file modified on this push, determine if the modified file
is a test, by searching test manifests. Populate self.verify_suites
with test files, organized by suite.
This depends on test manifests, so can only run after test zips have
@@ -195,16 +212,43 @@ class CodeCoverageMixin(SingleTestMixin)
# Add all baseline tests needed
for suite in tests_to_add:
for test in tests_to_add[suite]:
if suite not in self.suites:
self.suites[suite] = []
if test not in self.suites[suite]:
self.suites[suite].append(test)
+ @PostScriptAction('download-and-extract')
+ def setup_java_coverage_tools(self, action, success=None):
+ '''
+ Downloads grcov, jacoco cli and the geckoview classfiles generated in the build.
+ '''
+ if not self.java_code_coverage_enabled:
+ return
+
+ self.grcov_dir = tempfile.mkdtemp()
+ self._download_artifacts(self.grcov_dir)
+
+ url_to_classfiles = self.query_build_dir_url('target.geckoview_classfiles.zip')
+ self.classfiles_dir = tempfile.mkdtemp()
+ classfiles_zip_path = os.path.join(self.classfiles_dir, 'target.geckoview_classfiles.zip')
+ self.download_file(url_to_classfiles, classfiles_zip_path)
+ with zipfile.ZipFile(classfiles_zip_path, 'r') as z:
+ z.extractall(self.classfiles_dir)
+ os.remove(classfiles_zip_path)
+
+ # Create the directory where the emulator coverage file will be placed.
+ self.java_coverage_output_path = os.path.join(tempfile.mkdtemp(),
+ 'geckoview-junit-coverage.ec')
+
+ # TODO: set up jacoco as a fetch task from the android build
+ self.jacoco_jar = os.path.join(tempfile.mkdtemp(), 'org.jacoco.cli-0.8.1-nodeps.jar')
+ self.download_file('http://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.1/org.jacoco.cli-0.8.1-nodeps.jar', self.jacoco_jar) # NOQA: E501
+
@property
def coverage_args(self):
return []
def set_coverage_env(self, env, is_baseline_test=False):
# Set the GCOV directory.
self.gcov_dir = tempfile.mkdtemp()
env['GCOV_PREFIX'] = self.gcov_dir
@@ -431,16 +475,53 @@ class CodeCoverageMixin(SingleTestMixin)
# Zip the JSVM coverage data and upload it.
jsvm_zip_path = os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-jsvm.zip')
with zipfile.ZipFile(jsvm_zip_path, 'w', zipfile.ZIP_DEFLATED) as z:
z.write(jsvm_output_file)
shutil.rmtree(self.grcov_dir)
+ @PostScriptAction('run-tests')
+ def process_java_coverage_data(self, action, success=None):
+ '''
+ Run JaCoCo on the coverage.exec file in order to get a XML report.
+ After that, run grcov on the XML report to get a lcov report.
+ Finally, archive the lcov file and upload it, as process_coverage_data is doing.
+ '''
+ if not self.java_code_coverage_enabled:
+ return
+
+ dirs = self.query_abs_dirs()
+ xml_path = tempfile.mkdtemp()
+ jacoco_command = ['java', '-jar', self.jacoco_jar, 'report',
+ self.java_coverage_output_path,
+ '--classfiles', self.classfiles_dir,
+ '--name', 'geckoview-junit',
+ '--xml', os.path.join(xml_path, 'geckoview-junit.xml')]
+ self.run_command(jacoco_command, halt_on_failure=True)
+
+ grcov_command = [
+ os.path.join(self.grcov_dir, 'grcov'),
+ '-t', 'lcov',
+ xml_path,
+ ]
+ tmp_output_file, _ = self.get_output_from_command(
+ grcov_command,
+ silent=True,
+ save_tmpfiles=True,
+ return_type='files',
+ throw_exception=True,
+ )
+
+ if not self.ccov_upload_disabled:
+ grcov_zip_path = os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-grcov.zip')
+ with zipfile.ZipFile(grcov_zip_path, 'w', zipfile.ZIP_DEFLATED) as z:
+ z.write(tmp_output_file, 'grcov_lcov_output.info')
+
def rm_baseline_cov(baseline_coverage, test_coverage):
'''
Returns the difference between test_coverage and
baseline_coverage, such that what is returned
is the unique coverage for the test in question.
'''
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -21,17 +21,20 @@ sys.path.insert(1, os.path.dirname(sys.p
from mozprocess import ProcessHandler
from mozharness.base.log import FATAL
from mozharness.base.script import BaseScript, PreScriptAction, PostScriptAction
from mozharness.mozilla.automation import TBPL_RETRY, EXIT_STATUS_DICT
from mozharness.mozilla.mozbase import MozbaseMixin
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
-from mozharness.mozilla.testing.codecoverage import CodeCoverageMixin
+from mozharness.mozilla.testing.codecoverage import (
+ CodeCoverageMixin,
+ code_coverage_config_options
+)
class AndroidEmulatorTest(TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMixin):
"""
A mozharness script for Android functional tests (like mochitests and reftests)
run on an Android emulator. This script starts and manages an Android emulator
for the duration of the required tests. This is like desktop_unittest.py, but
for Android emulator test platforms.
@@ -72,17 +75,18 @@ class AndroidEmulatorTest(TestingMixin,
}
], [
["--log-tbpl-level"],
{"action": "store",
"dest": "log_tbpl_level",
"default": "info",
"help": "Set log level (debug|info|warning|error|critical|fatal)",
}
- ]] + copy.deepcopy(testing_config_options)
+ ]] + copy.deepcopy(testing_config_options) + \
+ copy.deepcopy(code_coverage_config_options)
app_name = None
def __init__(self, require_config_file=False):
super(AndroidEmulatorTest, self).__init__(
config_options=self.config_options,
all_actions=['clobber',
'setup-avds',
@@ -482,16 +486,20 @@ class AndroidEmulatorTest(TestingMixin,
try_options, try_tests = self.try_args(self.test_suite)
cmd.extend(try_options)
if not self.verify_enabled and not self.per_test_coverage:
cmd.extend(self.query_tests_args(
self.config["suite_definitions"][self.test_suite].get("tests"),
None,
try_tests))
+ if self.java_code_coverage_enabled:
+ cmd.extend(['--enable-coverage',
+ '--coverage-output-path', self.java_coverage_output_path])
+
return cmd
def _get_repo_url(self, path):
"""
Return a url for a file (typically a tooltool manifest) in this hg repo
and using this revision (or mozilla-central/default if repo/rev cannot
be determined).