Bug 1473313 - Part 3: Extend CodeCoverageMixin to handle java code coverage tools. r=marco,gbrown draft
authorTudor-Gabriel Vîjială <tvijiala@mozilla.com>
Tue, 24 Jul 2018 11:51:54 +0100
changeset 821934 08534f5b87009d4a22355cffc7c4d1bcfde80080
parent 821933 e3cfbba95667d455f0147bd9c0b5181c8d87e6bd
child 821935 b84d40026124394aa5d2ff95a67241033a002b76
push id117232
push userbmo:tvijiala@mozilla.com
push dateTue, 24 Jul 2018 12:32:30 +0000
reviewersmarco, gbrown
bugs1473313
milestone63.0a1
Bug 1473313 - Part 3: Extend CodeCoverageMixin to handle java code coverage tools. r=marco,gbrown MozReview-Commit-ID: Lld5XLUMb3Y
taskcluster/taskgraph/transforms/tests.py
testing/mozharness/mozharness/mozilla/testing/codecoverage.py
testing/mozharness/scripts/android_emulator_unittest.py
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -721,16 +721,22 @@ def handle_suite_category(config, tests)
 def enable_code_coverage(config, tests):
     """Enable code coverage for the ccov and jsdcov build-platforms"""
     for test in tests:
         if 'ccov' in test['build-platform']:
             # do not run tests on fuzzing or opt build
             if 'opt' in test['build-platform'] or 'fuzzing' in test['build-platform']:
                 test['run-on-projects'] = []
                 continue
+            # Skip this transform for android code coverage builds.
+            if 'android' in test['build-platform']:
+                test.setdefault('fetches', {}).setdefault('fetch', []).append('grcov-linux-x86_64')
+                test['mozharness'].setdefault('extra-options', []).append('--java-code-coverage')
+                yield test
+                continue
             test['mozharness'].setdefault('extra-options', []).append('--code-coverage')
             test['instance-size'] = 'xlarge'
             # Ensure we always run on the projects defined by the build, unless the test
             # is try only or shouldn't run at all.
             if test['run-on-projects'] not in [[], ['try']]:
                 test['run-on-projects'] = 'built-projects'
 
             # Ensure we don't optimize test suites out.
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
@@ -40,16 +40,22 @@ code_coverage_config_options = [
       "help": "Whether test run should package and upload code coverage data."
       }],
     [["--jsd-code-coverage"],
      {"action": "store_true",
       "dest": "jsd_code_coverage",
       "default": False,
       "help": "Whether JSDebugger code coverage should be run."
       }],
+    [["--java-code-coverage"],
+     {"action": "store_true",
+      "dest": "java_code_coverage",
+      "default": False,
+      "help": "Whether Java code coverage should be run."
+      }],
 ]
 
 
 class CodeCoverageMixin(SingleTestMixin):
     """
     Mixin for setting GCOV_PREFIX during test execution, packaging up
     the resulting .gcda files and uploading them to blobber.
     """
@@ -84,16 +90,23 @@ class CodeCoverageMixin(SingleTestMixin)
 
     @property
     def jsd_code_coverage_enabled(self):
         try:
             return bool(self.config.get('jsd_code_coverage'))
         except (AttributeError, KeyError, TypeError):
             return False
 
+    @property
+    def java_code_coverage_enabled(self):
+        try:
+            return bool(self.config.get('java_code_coverage'))
+        except (AttributeError, KeyError, TypeError):
+            return False
+
     @PostScriptAction('download-and-extract')
     def setup_coverage_tools(self, action, success=None):
         if not self.code_coverage_enabled:
             return
 
         if mozinfo.os == 'linux' or mozinfo.os == 'mac':
             self.prefix = '/builds/worker/workspace/build/src/'
             strip_count = self.prefix.count('/')
@@ -108,35 +121,39 @@ class CodeCoverageMixin(SingleTestMixin)
         os.environ['GCOV_PREFIX_STRIP'] = str(strip_count)
 
         # Install grcov on the test machine
         # Get the path to the build machines gcno files.
         self.url_to_gcno = self.query_build_dir_url('target.code-coverage-gcno.zip')
         self.url_to_chrome_map = self.query_build_dir_url('chrome-map.json')
 
         # Create the grcov directory, then download it.
-        # TODO: use the fetch-content script to download artifacts.
+        # Note: We assume grcov is our only artifact.
         self.grcov_dir = tempfile.mkdtemp()
-        ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{task}/artifacts/{artifact}'
-        for word in os.getenv('MOZ_FETCHES').split():
-            artifact, task = word.split('@', 1)
-            filename = os.path.basename(artifact)
-            url = ARTIFACT_URL.format(artifact=artifact, task=task)
-            self.download_file(url, parent_dir=self.grcov_dir)
-
-            with tarfile.open(os.path.join(self.grcov_dir, filename), 'r') as tar:
-                tar.extractall(self.grcov_dir)
-            os.remove(os.path.join(self.grcov_dir, filename))
+        self._download_artifacts(self.grcov_dir)
 
         # Download the gcno archive from the build machine.
         self.download_file(self.url_to_gcno, parent_dir=self.grcov_dir)
 
         # Download the chrome-map.json file from the build machine.
         self.download_file(self.url_to_chrome_map, parent_dir=self.grcov_dir)
 
+    def _download_artifacts(self, destination):
+        # TODO: use the fetch-content script to download artifacts.
+        ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{task}/artifacts/{artifact}'
+        for word in os.getenv('MOZ_FETCHES').split():
+            artifact, task = word.split('@', 1)
+            filename = os.path.basename(artifact)
+            url = ARTIFACT_URL.format(artifact=artifact, task=task)
+            self.download_file(url, parent_dir=destination)
+
+            with tarfile.open(os.path.join(destination, filename), 'r') as tar:
+                tar.extractall(destination)
+            os.remove(os.path.join(destination, filename))
+
     @PostScriptAction('download-and-extract')
     def find_tests_for_coverage(self, action, success=None):
         """
            For each file modified on this push, determine if the modified file
            is a test, by searching test manifests. Populate self.verify_suites
            with test files, organized by suite.
 
            This depends on test manifests, so can only run after test zips have
@@ -195,16 +212,43 @@ class CodeCoverageMixin(SingleTestMixin)
         # Add all baseline tests needed
         for suite in tests_to_add:
             for test in tests_to_add[suite]:
                 if suite not in self.suites:
                     self.suites[suite] = []
                 if test not in self.suites[suite]:
                     self.suites[suite].append(test)
 
+    @PostScriptAction('download-and-extract')
+    def setup_java_coverage_tools(self, action, success=None):
+        '''
+        Downloads grcov, jacoco cli and the geckoview classfiles generated in the build.
+        '''
+        if not self.java_code_coverage_enabled:
+            return
+
+        self.grcov_dir = tempfile.mkdtemp()
+        self._download_artifacts(self.grcov_dir)
+
+        url_to_classfiles = self.query_build_dir_url('target.geckoview_classfiles.zip')
+        self.classfiles_dir = tempfile.mkdtemp()
+        classfiles_zip_path = os.path.join(self.classfiles_dir, 'target.geckoview_classfiles.zip')
+        self.download_file(url_to_classfiles, classfiles_zip_path)
+        with zipfile.ZipFile(classfiles_zip_path, 'r') as z:
+            z.extractall(self.classfiles_dir)
+        os.remove(classfiles_zip_path)
+
+        # Create the directory where the emulator coverage file will be placed.
+        self.java_coverage_output_path = os.path.join(tempfile.mkdtemp(),
+                                                      'geckoview-junit-coverage.ec')
+
+        # TODO: set up jacoco as a fetch task from the android build
+        self.jacoco_jar = os.path.join(tempfile.mkdtemp(), 'org.jacoco.cli-0.8.1-nodeps.jar')
+        self.download_file('http://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.1/org.jacoco.cli-0.8.1-nodeps.jar', self.jacoco_jar)  # NOQA: E501
+
     @property
     def coverage_args(self):
         return []
 
     def set_coverage_env(self, env, is_baseline_test=False):
         # Set the GCOV directory.
         self.gcov_dir = tempfile.mkdtemp()
         env['GCOV_PREFIX'] = self.gcov_dir
@@ -431,16 +475,53 @@ class CodeCoverageMixin(SingleTestMixin)
 
             # Zip the JSVM coverage data and upload it.
             jsvm_zip_path = os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-jsvm.zip')
             with zipfile.ZipFile(jsvm_zip_path, 'w', zipfile.ZIP_DEFLATED) as z:
                 z.write(jsvm_output_file)
 
         shutil.rmtree(self.grcov_dir)
 
+    @PostScriptAction('run-tests')
+    def process_java_coverage_data(self, action, success=None):
+        '''
+        Run JaCoCo on the coverage.exec file in order to get a XML report.
+        After that, run grcov on the XML report to get a lcov report.
+        Finally, archive the lcov file and upload it, as process_coverage_data is doing.
+        '''
+        if not self.java_code_coverage_enabled:
+            return
+
+        dirs = self.query_abs_dirs()
+        xml_path = tempfile.mkdtemp()
+        jacoco_command = ['java', '-jar', self.jacoco_jar, 'report',
+                          self.java_coverage_output_path,
+                          '--classfiles', self.classfiles_dir,
+                          '--name', 'geckoview-junit',
+                          '--xml', os.path.join(xml_path, 'geckoview-junit.xml')]
+        self.run_command(jacoco_command, halt_on_failure=True)
+
+        grcov_command = [
+            os.path.join(self.grcov_dir, 'grcov'),
+            '-t', 'lcov',
+            xml_path,
+        ]
+        tmp_output_file, _ = self.get_output_from_command(
+            grcov_command,
+            silent=True,
+            save_tmpfiles=True,
+            return_type='files',
+            throw_exception=True,
+        )
+
+        if not self.ccov_upload_disabled:
+            grcov_zip_path = os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-grcov.zip')
+            with zipfile.ZipFile(grcov_zip_path, 'w', zipfile.ZIP_DEFLATED) as z:
+                z.write(tmp_output_file, 'grcov_lcov_output.info')
+
 
 def rm_baseline_cov(baseline_coverage, test_coverage):
     '''
     Returns the difference between test_coverage and
     baseline_coverage, such that what is returned
     is the unique coverage for the test in question.
     '''
 
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -21,17 +21,20 @@ sys.path.insert(1, os.path.dirname(sys.p
 
 from mozprocess import ProcessHandler
 
 from mozharness.base.log import FATAL
 from mozharness.base.script import BaseScript, PreScriptAction, PostScriptAction
 from mozharness.mozilla.automation import TBPL_RETRY, EXIT_STATUS_DICT
 from mozharness.mozilla.mozbase import MozbaseMixin
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
-from mozharness.mozilla.testing.codecoverage import CodeCoverageMixin
+from mozharness.mozilla.testing.codecoverage import (
+    CodeCoverageMixin,
+    code_coverage_config_options
+)
 
 
 class AndroidEmulatorTest(TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMixin):
     """
        A mozharness script for Android functional tests (like mochitests and reftests)
        run on an Android emulator. This script starts and manages an Android emulator
        for the duration of the required tests. This is like desktop_unittest.py, but
        for Android emulator test platforms.
@@ -72,17 +75,18 @@ class AndroidEmulatorTest(TestingMixin, 
          }
     ], [
         ["--log-tbpl-level"],
         {"action": "store",
          "dest": "log_tbpl_level",
          "default": "info",
          "help": "Set log level (debug|info|warning|error|critical|fatal)",
          }
-    ]] + copy.deepcopy(testing_config_options)
+    ]] + copy.deepcopy(testing_config_options) + \
+        copy.deepcopy(code_coverage_config_options)
 
     app_name = None
 
     def __init__(self, require_config_file=False):
         super(AndroidEmulatorTest, self).__init__(
             config_options=self.config_options,
             all_actions=['clobber',
                          'setup-avds',
@@ -482,16 +486,20 @@ class AndroidEmulatorTest(TestingMixin, 
         try_options, try_tests = self.try_args(self.test_suite)
         cmd.extend(try_options)
         if not self.verify_enabled and not self.per_test_coverage:
             cmd.extend(self.query_tests_args(
                 self.config["suite_definitions"][self.test_suite].get("tests"),
                 None,
                 try_tests))
 
+        if self.java_code_coverage_enabled:
+            cmd.extend(['--enable-coverage',
+                        '--coverage-output-path', self.java_coverage_output_path])
+
         return cmd
 
     def _get_repo_url(self, path):
         """
            Return a url for a file (typically a tooltool manifest) in this hg repo
            and using this revision (or mozilla-central/default if repo/rev cannot
            be determined).