Bug 1473313 - Part 3: Extend CodeCoverageMixin to handle java code coverage tools. r=marco,gbrown
MozReview-Commit-ID: Lld5XLUMb3Y
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -721,16 +721,22 @@ def handle_suite_category(config, tests)
def enable_code_coverage(config, tests):
"""Enable code coverage for the ccov and jsdcov build-platforms"""
for test in tests:
if 'ccov' in test['build-platform']:
# do not run tests on fuzzing or opt build
if 'opt' in test['build-platform'] or 'fuzzing' in test['build-platform']:
test['run-on-projects'] = []
continue
+ # Skip this transform for android code coverage builds.
+ if 'android' in test['build-platform']:
+ test.setdefault('fetches', {}).setdefault('fetch', []).append('grcov-linux-x86_64')
+ test['mozharness'].setdefault('extra-options', []).append('--java-code-coverage')
+ yield test
+ continue
test['mozharness'].setdefault('extra-options', []).append('--code-coverage')
test['instance-size'] = 'xlarge'
# Ensure we always run on the projects defined by the build, unless the test
# is try only or shouldn't run at all.
if test['run-on-projects'] not in [[], ['try']]:
test['run-on-projects'] = 'built-projects'
# Ensure we don't optimize test suites out.
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
@@ -40,16 +40,22 @@ code_coverage_config_options = [
"help": "Whether test run should package and upload code coverage data."
}],
[["--jsd-code-coverage"],
{"action": "store_true",
"dest": "jsd_code_coverage",
"default": False,
"help": "Whether JSDebugger code coverage should be run."
}],
+ [["--java-code-coverage"],
+ {"action": "store_true",
+ "dest": "java_code_coverage",
+ "default": False,
+ "help": "Whether Java code coverage should be run."
+ }],
]
class CodeCoverageMixin(SingleTestMixin):
"""
Mixin for setting GCOV_PREFIX during test execution, packaging up
the resulting .gcda files and uploading them to blobber.
"""
@@ -84,39 +90,65 @@ class CodeCoverageMixin(SingleTestMixin)
@property
def jsd_code_coverage_enabled(self):
try:
return bool(self.config.get('jsd_code_coverage'))
except (AttributeError, KeyError, TypeError):
return False
- @PostScriptAction('download-and-extract')
- def setup_coverage_tools(self, action, success=None):
- if not self.code_coverage_enabled:
- return
+ @property
+ def java_code_coverage_enabled(self):
+ try:
+ return bool(self.config.get('java_code_coverage'))
+ except (AttributeError, KeyError, TypeError):
+ return False
+ def _setup_cpp_js_coverage_tools(self):
if mozinfo.os == 'linux' or mozinfo.os == 'mac':
self.prefix = '/builds/worker/workspace/build/src/'
strip_count = self.prefix.count('/')
elif mozinfo.os == 'win':
self.prefix = 'z:/build/build/src/'
# Add 1 as on Windows the path where the compiler tries to write the
# gcda files has an additional 'obj-firefox' component.
strip_count = self.prefix.count('/') + 1
else:
raise Exception('Unexpected OS: {}'.format(mozinfo.os))
os.environ['GCOV_PREFIX_STRIP'] = str(strip_count)
- # Install grcov on the test machine
- # Get the path to the build machines gcno files.
- self.url_to_gcno = self.query_build_dir_url('target.code-coverage-gcno.zip')
- self.url_to_chrome_map = self.query_build_dir_url('chrome-map.json')
+ # Download the gcno archive from the build machine.
+ url_to_gcno = self.query_build_dir_url('target.code-coverage-gcno.zip')
+ self.download_file(url_to_gcno, parent_dir=self.grcov_dir)
+
+ # Download the chrome-map.json file from the build machine.
+ url_to_chrome_map = self.query_build_dir_url('chrome-map.json')
+ self.download_file(url_to_chrome_map, parent_dir=self.grcov_dir)
+
+ def _setup_java_coverage_tools(self):
+ # Download and extract jacoco-cli from the build task.
+ url_to_jacoco = self.query_build_dir_url('target.jacoco-cli.jar')
+ self.jacoco_jar = os.path.join(tempfile.mkdtemp(), 'target.jacoco-cli.jar')
+ self.download_file(url_to_jacoco, self.jacoco_jar)
+ # Download and extract class files from the build task.
+ self.classfiles_dir = tempfile.mkdtemp()
+ url_to_classfiles = self.query_build_dir_url('target.geckoview_classfiles.zip')
+ classfiles_zip_path = os.path.join(self.classfiles_dir, 'target.geckoview_classfiles.zip')
+ self.download_file(url_to_classfiles, classfiles_zip_path)
+ with zipfile.ZipFile(classfiles_zip_path, 'r') as z:
+ z.extractall(self.classfiles_dir)
+ os.remove(classfiles_zip_path)
+
+ # Create the directory where the emulator coverage file will be placed.
+ self.java_coverage_output_path = os.path.join(tempfile.mkdtemp(),
+ 'junit-coverage.ec')
+
+ def _download_grcov(self):
fetches_dir = os.environ.get('MOZ_FETCHES_DIR')
if fetches_dir and os.path.isfile(os.path.join(fetches_dir, 'grcov')):
self.grcov_dir = fetches_dir
else:
# Create the grcov directory, then download it.
# TODO: use the fetch-content script to download artifacts.
self.grcov_dir = tempfile.mkdtemp()
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{task}/artifacts/{artifact}'
@@ -125,21 +157,28 @@ class CodeCoverageMixin(SingleTestMixin)
filename = os.path.basename(artifact)
url = ARTIFACT_URL.format(artifact=artifact, task=task)
self.download_file(url, parent_dir=self.grcov_dir)
with tarfile.open(os.path.join(self.grcov_dir, filename), 'r') as tar:
tar.extractall(self.grcov_dir)
os.remove(os.path.join(self.grcov_dir, filename))
- # Download the gcno archive from the build machine.
- self.download_file(self.url_to_gcno, parent_dir=self.grcov_dir)
+ @PostScriptAction('download-and-extract')
+ def setup_coverage_tools(self, action, success=None):
+ if not self.code_coverage_enabled and not self.java_code_coverage_enabled:
+ return
- # Download the chrome-map.json file from the build machine.
- self.download_file(self.url_to_chrome_map, parent_dir=self.grcov_dir)
+ self._download_grcov()
+
+ if self.code_coverage_enabled:
+ self._setup_cpp_js_coverage_tools()
+
+ if self.java_code_coverage_enabled:
+ self._setup_java_coverage_tools()
@PostScriptAction('download-and-extract')
def find_tests_for_coverage(self, action, success=None):
"""
For each file modified on this push, determine if the modified file
is a test, by searching test manifests. Populate self.verify_suites
with test files, organized by suite.
@@ -435,16 +474,59 @@ class CodeCoverageMixin(SingleTestMixin)
# Zip the JSVM coverage data and upload it.
jsvm_zip_path = os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-jsvm.zip')
with zipfile.ZipFile(jsvm_zip_path, 'w', zipfile.ZIP_DEFLATED) as z:
z.write(jsvm_output_file)
shutil.rmtree(self.grcov_dir)
+ @PostScriptAction('run-tests')
+ def process_java_coverage_data(self, action, success=None):
+ '''
+ Run JaCoCo on the coverage.ec file in order to get a XML report.
+ After that, run grcov on the XML report to get a lcov report.
+ Finally, archive the lcov file and upload it, as process_coverage_data is doing.
+ '''
+ if not self.java_code_coverage_enabled:
+ return
+
+ # If the emulator became unresponsive, the task has failed and we don't
+ # have the coverage report file, so stop running this function and
+ # allow the task to be retried automatically.
+ if not success and not os.path.exists(self.java_coverage_output_path):
+ return
+
+ dirs = self.query_abs_dirs()
+ xml_path = tempfile.mkdtemp()
+ jacoco_command = ['java', '-jar', self.jacoco_jar, 'report',
+ self.java_coverage_output_path,
+ '--classfiles', self.classfiles_dir,
+ '--name', 'geckoview-junit',
+ '--xml', os.path.join(xml_path, 'geckoview-junit.xml')]
+ self.run_command(jacoco_command, halt_on_failure=True)
+
+ grcov_command = [
+ os.path.join(self.grcov_dir, 'grcov'),
+ '-t', 'lcov',
+ xml_path,
+ ]
+ tmp_output_file, _ = self.get_output_from_command(
+ grcov_command,
+ silent=True,
+ save_tmpfiles=True,
+ return_type='files',
+ throw_exception=True,
+ )
+
+ if not self.ccov_upload_disabled:
+ grcov_zip_path = os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-grcov.zip')
+ with zipfile.ZipFile(grcov_zip_path, 'w', zipfile.ZIP_DEFLATED) as z:
+ z.write(tmp_output_file, 'grcov_lcov_output.info')
+
def rm_baseline_cov(baseline_coverage, test_coverage):
'''
Returns the difference between test_coverage and
baseline_coverage, such that what is returned
is the unique coverage for the test in question.
'''
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -21,17 +21,20 @@ sys.path.insert(1, os.path.dirname(sys.p
from mozprocess import ProcessHandler
from mozharness.base.log import FATAL
from mozharness.base.script import BaseScript, PreScriptAction, PostScriptAction
from mozharness.mozilla.automation import TBPL_RETRY, EXIT_STATUS_DICT
from mozharness.mozilla.mozbase import MozbaseMixin
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
-from mozharness.mozilla.testing.codecoverage import CodeCoverageMixin
+from mozharness.mozilla.testing.codecoverage import (
+ CodeCoverageMixin,
+ code_coverage_config_options
+)
class AndroidEmulatorTest(TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMixin):
"""
A mozharness script for Android functional tests (like mochitests and reftests)
run on an Android emulator. This script starts and manages an Android emulator
for the duration of the required tests. This is like desktop_unittest.py, but
for Android emulator test platforms.
@@ -72,17 +75,18 @@ class AndroidEmulatorTest(TestingMixin,
}
], [
["--log-tbpl-level"],
{"action": "store",
"dest": "log_tbpl_level",
"default": "info",
"help": "Set log level (debug|info|warning|error|critical|fatal)",
}
- ]] + copy.deepcopy(testing_config_options)
+ ]] + copy.deepcopy(testing_config_options) + \
+ copy.deepcopy(code_coverage_config_options)
app_name = None
def __init__(self, require_config_file=False):
super(AndroidEmulatorTest, self).__init__(
config_options=self.config_options,
all_actions=['clobber',
'setup-avds',
@@ -482,16 +486,20 @@ class AndroidEmulatorTest(TestingMixin,
try_options, try_tests = self.try_args(self.test_suite)
cmd.extend(try_options)
if not self.verify_enabled and not self.per_test_coverage:
cmd.extend(self.query_tests_args(
self.config["suite_definitions"][self.test_suite].get("tests"),
None,
try_tests))
+ if self.java_code_coverage_enabled:
+ cmd.extend(['--enable-coverage',
+ '--coverage-output-path', self.java_coverage_output_path])
+
return cmd
def _get_repo_url(self, path):
"""
Return a url for a file (typically a tooltool manifest) in this hg repo
and using this revision (or mozilla-central/default if repo/rev cannot
be determined).