Bug 1471660 - Integrate code coverage for A(test) junit test suite via JaCoCo plugin. r=nalexander,marco draft
authorTudor-Gabriel Vîjială <tvijiala@mozilla.com>
Mon, 02 Jul 2018 11:32:35 +0100
changeset 815711 bd695e01ebd73c143f34e9b47b66336522ce197c
parent 814904 fa376bf17cc95539f5e37186977d760296fb5093
child 815991 fbd419beaf1fc33cca3f7ed5133702bdcaec7b23
child 815992 32c4f91a09b221d00828ba238b45976c79148c16
push id115616
push userbmo:tvijiala@mozilla.com
push dateMon, 09 Jul 2018 18:23:54 +0000
reviewersnalexander, marco
bugs1471660
milestone63.0a1
Bug 1471660 - Integrate code coverage for A(test) junit test suite via JaCoCo plugin. r=nalexander,marco MozReview-Commit-ID: ElYGmF6zoYg
mobile/android/app/build.gradle
mobile/android/geckoview/build.gradle
mobile/android/gradle.configure
mobile/android/gradle/jacoco_dependencies.gradle
mobile/android/gradle/jacoco_for_junit.gradle
mobile/android/mach_commands.py
taskcluster/ci/build/android-stuff.yml
testing/mozharness/configs/builds/releng_sub_android_configs/64_test_ccov.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
--- a/mobile/android/app/build.gradle
+++ b/mobile/android/app/build.gradle
@@ -501,8 +501,13 @@ android.applicationVariants.all { varian
 android.applicationVariants.all { variant ->
     def productFlavor = ""
     variant.productFlavors.each {
         productFlavor += "${it.name.capitalize()}"
     }
     def buildType = "${variant.buildType.name.capitalize()}"
     tasks["compile${productFlavor}${buildType}UnitTestSources"].dependsOn(tasks["merge${productFlavor}${buildType}Assets"])
 }
+
+apply from: "${topsrcdir}/mobile/android/gradle/jacoco_dependencies.gradle"
+if (project.hasProperty('enable_code_coverage')) {
+    apply from: "${topsrcdir}/mobile/android/gradle/jacoco_for_junit.gradle"
+}
--- a/mobile/android/geckoview/build.gradle
+++ b/mobile/android/geckoview/build.gradle
@@ -360,8 +360,13 @@ task("generateSDKBindings", type: JavaEx
         // From -Pgenerate_sdk_bindings_args=... on command line.
         args project.generate_sdk_bindings_args.split(':')
     }
 
     workingDir "${topsrcdir}/widget/android/bindings"
 
     dependsOn project(':annotations').jar
 }
+
+apply from: "${topsrcdir}/mobile/android/gradle/jacoco_dependencies.gradle"
+if (project.hasProperty('enable_code_coverage')) {
+    apply from: "${topsrcdir}/mobile/android/gradle/jacoco_for_junit.gradle"
+}
--- a/mobile/android/gradle.configure
+++ b/mobile/android/gradle.configure
@@ -174,17 +174,27 @@ set_config('GRADLE_ANDROID_APP_ANDROIDTE
 def gradle_android_test_tasks(build_config):
     '''Gradle tasks run by |mach android test|.'''
     return [
         'app:test{app.variant.name}UnitTest'.format(app=build_config.app),
         'geckoview:test{geckoview.variant.name}UnitTest'.format(
             geckoview=build_config.geckoview),
     ]
 
+
+@dependable
+def gradle_android_test_ccov_report_tasks():
+    '''Additional gradle tasks run by |mach android test-ccov|.'''
+    return [
+        'app:jacocoTestReport',
+        'geckoview:jacocoTestReport',
+    ]
+
 set_config('GRADLE_ANDROID_TEST_TASKS', gradle_android_test_tasks)
+set_config('GRADLE_ANDROID_TEST_CCOV_REPORT_TASKS', gradle_android_test_ccov_report_tasks)
 
 
 @depends(gradle_android_build_config)
 def gradle_android_lint_tasks(build_config):
     '''Gradle tasks run by |mach android lint|.'''
     return [
         'app:lint{app.variant.name}'.format(app=build_config.app),
     ]
new file mode 100644
--- /dev/null
+++ b/mobile/android/gradle/jacoco_dependencies.gradle
@@ -0,0 +1,13 @@
+/* -*- Mode: Groovy; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+project.ext.jacoco_version = "0.7.8"
+
+dependencies {
+    testImplementation "org.jacoco:org.jacoco.agent:${project.jacoco_version}"
+    testImplementation "org.jacoco:org.jacoco.ant:${project.jacoco_version}"
+    testImplementation "org.jacoco:org.jacoco.core:${project.jacoco_version}"
+    testImplementation "org.jacoco:org.jacoco.report:${project.jacoco_version}"
+}
new file mode 100644
--- /dev/null
+++ b/mobile/android/gradle/jacoco_for_junit.gradle
@@ -0,0 +1,41 @@
+/* -*- Mode: Groovy; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+apply plugin: "jacoco"
+
+jacoco {
+    toolVersion = "${project.jacoco_version}"
+}
+
+android {
+    testOptions {
+        unitTests.all {
+            jacoco {
+                includeNoLocationClasses = true
+            }
+        }
+    }
+}
+
+dependencies {
+    jacocoAgent "org.jacoco:org.jacoco.agent:${project.jacoco_version}"
+    jacocoAnt "org.jacoco:org.jacoco.ant:${project.jacoco_version}"
+}
+
+task jacocoTestReport(type: JacocoReport) {
+    reports {
+        xml.enabled = true
+        html.enabled = false
+        csv.enabled = false
+    }
+    def fileFilter = ['**/androidTest/**', '**/test/**', '**/R.class', '**/R$*.class',
+            '**/BuildConfig.*', '**/Manifest*.*', '**/*Test*.*', 'android/**/*.*']
+    def debugTree = fileTree(dir: "${buildDir}/intermediates/classes", excludes: fileFilter)
+    def mainSrc = "${project.projectDir}/src/main/java"
+
+    sourceDirectories = files([mainSrc])
+    classDirectories = files([debugTree])
+    executionData = fileTree("${buildDir}/jacoco")
+}
--- a/mobile/android/mach_commands.py
+++ b/mobile/android/mach_commands.py
@@ -2,19 +2,26 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
 import logging
 import os
+import shutil
+import subprocess
+import tarfile
+import urllib
+import zipfile
 
 import mozpack.path as mozpath
 
+from mozfile import TemporaryDirectory
+
 from mozbuild.base import (
     MachCommandBase,
     MachCommandConditions as conditions,
 )
 
 from mozbuild.shellutil import (
     split as shell_split,
 )
@@ -195,16 +202,60 @@ class MachCommands(MachCommandBase):
                 print('SUITE-END | android-test | {} {}'.format(report, root.get('name')))
 
         if not found_reports:
             print('TEST-UNEXPECTED-FAIL | android-test | No reports found under {}'.format(gradledir))  # NOQA: E501
             return 1
 
         return ret
 
+    @SubCommand('android', 'test-ccov',
+                """Run Android local unit tests in order to get a code coverage report.
+                See https://developer.mozilla.org/en-US/docs/Mozilla/Android-specific_test_suites#android-test""")  # NOQA: E501
+    @CommandArgument('args', nargs=argparse.REMAINDER)
+    def android_test_ccov(self, args):
+        enable_ccov = '-Penable_code_coverage'
+
+        # Don't care if the tests are failing, we only want the coverage information.
+        self.android_test([enable_ccov])
+
+        self.gradle(self.substs['GRADLE_ANDROID_TEST_CCOV_REPORT_TASKS'] +
+                    ['--continue', enable_ccov] + args, verbose=True)
+        self._process_jacoco_reports()
+        return 0
+
+    def _process_jacoco_reports(self):
+        def download_grcov(parent_dir):
+            # TODO: Bug 1472236 - Remove this and use fetch tasks to download grcov.
+            grcov_version = "v0.2.1"
+            tar_name = 'grcov.tar.bz2'
+            tar_path = os.path.join(parent_dir, tar_name)
+            url = 'https://github.com/mozilla/grcov/releases/download/%s/grcov-linux-x86_64.tar.bz2' % grcov_version  # NOQA: E501
+            urllib.urlretrieve(url, tar_path)
+            with tarfile.open(tar_path) as tar:
+                tar.extractall(parent_dir)
+            return os.path.join(parent_dir, 'grcov')
+
+        def run_grcov(grcov_path, input_path):
+            args = [grcov_path, input_path, '-t', 'lcov']
+            return subprocess.check_output(args)
+
+        with TemporaryDirectory() as xml_dir, TemporaryDirectory() as grcov_dir:
+            grcov = download_grcov(grcov_dir)
+
+            report_xml_template = self.topobjdir + '/gradle/build/mobile/android/%s/reports/jacoco/jacocoTestReport/jacocoTestReport.xml'  # NOQA: E501
+            shutil.copy(report_xml_template % 'app', os.path.join(xml_dir, 'app.xml'))
+            shutil.copy(report_xml_template % 'geckoview', os.path.join(xml_dir, 'geckoview.xml'))
+
+            # Parse output files with grcov.
+            grcov_output = run_grcov(grcov, xml_dir)
+            grcov_zip_path = os.path.join(self.topobjdir, 'code-coverage-grcov.zip')
+            with zipfile.ZipFile(grcov_zip_path, 'w', zipfile.ZIP_DEFLATED) as z:
+                z.writestr('grcov_lcov_output.info', grcov_output)
+
     @SubCommand('android', 'lint',
                 """Run Android lint.
                 See https://developer.mozilla.org/en-US/docs/Mozilla/Android-specific_test_suites#android-lint""")  # NOQA: E501
     @CommandArgument('args', nargs=argparse.REMAINDER)
     def android_lint(self, args):
         ret = self.gradle(self.substs['GRADLE_ANDROID_LINT_TASKS'] +
                           ["--continue"] + args, verbose=True)
 
--- a/taskcluster/ci/build/android-stuff.yml
+++ b/taskcluster/ci/build/android-stuff.yml
@@ -40,16 +40,50 @@ android-test/opt:
     optimization:
         skip-unless-changed:
             - "mobile/android/base/**"
             - "mobile/android/config/**"
             - "mobile/android/gradle.configure"
             - "mobile/android/tests/background/junit4/**"
             - "**/*.gradle"
 
+android-test-ccov/opt:
+    description: "Android armv7 unit test coverage report"
+    index:
+        product: mobile
+        job-name: android-test-ccov
+    treeherder:
+        platform: android-4-0-armv7-api16/opt
+        kind: build
+        tier: 1
+        symbol: A(test-ccov)
+    worker-type: aws-provisioner-v1/gecko-{level}-b-android
+    worker:
+        docker-image: {in-tree: android-build}
+        env:
+            GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-offline"
+            PERFHERDER_EXTRA_OPTIONS: android-test-ccov
+        artifacts:
+            - name: public/code-coverage-grcov.zip
+              path: /builds/worker/workspace/build/src/obj-firefox/code-coverage-grcov.zip
+              type: file
+        max-run-time: 7200
+    run:
+        using: mozharness
+        actions: [get-secrets build]
+        config:
+            - builds/releng_base_android_64_builds.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        custom-build-variant-cfg: android-test-ccov
+        tooltool-downloads: internal
+    toolchains:
+        - android-gradle-dependencies
+        - android-sdk-linux
+
 android-lint/opt:
     description: "Android lint"
     index:
         product: mobile
         job-name: android-lint
     treeherder:
         platform: android-4-0-armv7-api16/opt
         kind: build
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/releng_sub_android_configs/64_test_ccov.py
@@ -0,0 +1,15 @@
+config = {
+    'base_name': 'Android armv7 unit test code coverage %(branch)s',
+    'stage_platform': 'android-test-ccov',
+    'src_mozconfig': 'mobile/android/config/mozconfigs/android-api-16-frontend/nightly',
+    'multi_locale_config_platform': 'android',
+    # unit tests don't produce a package. So don't collect package metrics.
+    'disable_package_metrics': True,
+    'postflight_build_mach_commands': [
+        ['android',
+         'test-ccov',
+        ],
+    ],
+    'artifact_flag_build_variant_in_try': None, # There's no artifact equivalent.
+    'max_build_output_timeout': 0,
+}
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -434,16 +434,17 @@ class BuildOptionParser(object):
         'api-16-without-google-play-services': 'builds/releng_sub_%s_configs/%s_api_16_without_google_play_services.py',
         'rusttests': 'builds/releng_sub_%s_configs/%s_rusttests.py',
         'rusttests-debug': 'builds/releng_sub_%s_configs/%s_rusttests_debug.py',
         'x86': 'builds/releng_sub_%s_configs/%s_x86.py',
         'x86-artifact': 'builds/releng_sub_%s_configs/%s_x86_artifact.py',
         'api-16-partner-sample1': 'builds/releng_sub_%s_configs/%s_api_16_partner_sample1.py',
         'aarch64': 'builds/releng_sub_%s_configs/%s_aarch64.py',
         'android-test': 'builds/releng_sub_%s_configs/%s_test.py',
+        'android-test-ccov': 'builds/releng_sub_%s_configs/%s_test_ccov.py',
         'android-checkstyle': 'builds/releng_sub_%s_configs/%s_checkstyle.py',
         'android-lint': 'builds/releng_sub_%s_configs/%s_lint.py',
         'android-findbugs': 'builds/releng_sub_%s_configs/%s_findbugs.py',
         'android-geckoview-docs': 'builds/releng_sub_%s_configs/%s_geckoview_docs.py',
         'valgrind' : 'builds/releng_sub_%s_configs/%s_valgrind.py',
         'artifact': 'builds/releng_sub_%s_configs/%s_artifact.py',
         'debug-artifact': 'builds/releng_sub_%s_configs/%s_debug_artifact.py',
         'devedition': 'builds/releng_sub_%s_configs/%s_devedition.py',