Bug 1371445 - Add |mach android {lint,findbugs,checkstyle,test}| commands for running Android-specific test suites. r=gps
It be ideal to have |mach test {findbugs,test}| and |mach lint
{lint,checkstyle}|, but the |mach test| command is very difficult to
extend in a direction orthogonal to the existing direction. The
existing |mach test| is built around in-tree manifests, tagged and
divided into suites, intended to support |mach test
path/to/arbitrary/test|. The Android findbugs task is a global static
analysis that doesn't fit into the path/manifest model. The Android
test task is based on JUnit and not easy to build manifest support
for. The |mach lint| command is intended to be extended, but the
effort to extend it is non-trivial and not worth the effort (at this
time).
Therefore, I've taken the existing, little used |mach android| command
and added subcommands for use by local developers and automation. If
nothing else, this reduces the number of "special Gradle targets" --
the equivalent of "special Make targets" -- sprinkled throughout the
tree, which can only be a good thing!
MozReview-Commit-ID: 24b1vbgykpN
--- a/mobile/android/mach_commands.py
+++ b/mobile/android/mach_commands.py
@@ -18,16 +18,17 @@ from mozbuild.base import (
from mozbuild.shellutil import (
split as shell_split,
)
from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
+ SubCommand,
)
# NOTE python/mach/mach/commands/commandinfo.py references this function
# by name. If this function is renamed or removed, that file should
# be updated accordingly as well.
def REMOVED(cls):
"""Command no longer exists! Use the Gradle configuration rooted in the top source directory instead.
@@ -35,37 +36,284 @@ def REMOVED(cls):
See https://developer.mozilla.org/en-US/docs/Simple_Firefox_for_Android_build#Developing_Firefox_for_Android_in_Android_Studio_or_IDEA_IntelliJ.
"""
return False
@CommandProvider
class MachCommands(MachCommandBase):
@Command('android', category='devenv',
- description='Run the Android package manager tool.',
+ description='Run Android-specific commands.',
conditions=[conditions.is_android])
+ def android(self):
+ pass
+
+
+ @SubCommand('android', 'test',
+ """Run Android local unit tests.
+ See https://developer.mozilla.org/en-US/docs/Mozilla/Android-specific_test_suites#android-test""")
+ @CommandArgument('args', nargs=argparse.REMAINDER)
+ def android_test(self, args):
+ gradle_targets = [
+ 'app:testOfficialAustralisDebugUnitTest',
+ 'app:testOfficialPhotonDebugUnitTest',
+ ]
+ ret = self.gradle(gradle_targets + ["--continue"] + args, verbose=True)
+
+ # Findbug produces both HTML and XML reports. Visit the
+ # XML report(s) to report errors and link to the HTML
+ # report(s) for human consumption.
+ import itertools
+ import xml.etree.ElementTree as ET
+
+ from mozpack.files import (
+ FileFinder,
+ )
+
+ if 'TASK_ID' in os.environ and 'RUN_ID' in os.environ:
+ root_url = "https://queue.taskcluster.net/v1/task/{}/runs/{}/artifacts/public/android/unittest".format(os.environ['TASK_ID'], os.environ['RUN_ID'])
+ else:
+ root_url = os.path.join(self.topobjdir, 'gradle/build/mobile/android/app/reports/tests')
+
+ reports = ('officialAustralisDebug',
+ 'officialPhotonDebug',)
+ for report in reports:
+ finder = FileFinder(os.path.join(self.topobjdir, 'gradle/build/mobile/android/app/test-results/', report))
+ for p, _ in finder.find('TEST-*.xml'):
+ f = open(os.path.join(finder.base, p), 'rt')
+ tree = ET.parse(f)
+ root = tree.getroot()
+
+ print('SUITE-START | android-test | {} {}'.format(report, root.get('name')))
+
+ for testcase in root.findall('testcase'):
+ name = testcase.get('name')
+ print('TEST-START | {}'.format(name))
+
+ # Schema cribbed from
+ # http://llg.cubic.org/docs/junit/. There's no
+ # particular advantage to formatting the error, so
+ # for now let's just output the unexpected XML
+ # tag.
+ error_count = 0
+ for unexpected in itertools.chain(testcase.findall('error'),
+ testcase.findall('failure')):
+ for line in ET.tostring(unexpected).strip().splitlines():
+ print('TEST-UNEXPECTED-FAIL | {} | {}'.format(name, line))
+ error_count += 1
+ ret |= 1
+
+ # Skipped tests aren't unexpected at this time; we
+ # disable some tests that require live remote
+ # endpoints.
+ for skipped in testcase.findall('skipped'):
+ for line in ET.tostring(skipped).strip().splitlines():
+ print('TEST-INFO | {} | {}'.format(name, line))
+
+ if not error_count:
+ print('TEST-PASS | {}'.format(name))
+
+ print('SUITE-END | android-test | {} {}'.format(report, root.get('name')))
+
+ title = report
+ print("TinderboxPrint: report<br/><a href='{}/{}/index.html'>HTML {} report</a>, visit \"Inspect Task\" link for details".format(root_url, report, title))
+
+ return ret
+
+
+ @SubCommand('android', 'lint',
+ """Run Android lint.
+ See https://developer.mozilla.org/en-US/docs/Mozilla/Android-specific_test_suites#android-lint""")
+ @CommandArgument('args', nargs=argparse.REMAINDER)
+ def android_lint(self, args):
+ gradle_targets = [
+ 'app:lintOfficialAustralisDebug',
+ 'app:lintOfficialPhotonDebug',
+ ]
+ ret = self.gradle(gradle_targets + ["--continue"] + args, verbose=True)
+
+ # Android Lint produces both HTML and XML reports. Visit the
+ # XML report(s) to report errors and link to the HTML
+ # report(s) for human consumption.
+ import xml.etree.ElementTree as ET
+
+ if 'TASK_ID' in os.environ and 'RUN_ID' in os.environ:
+ root_url = "https://queue.taskcluster.net/v1/task/{}/runs/{}/artifacts/public/android/lint".format(os.environ['TASK_ID'], os.environ['RUN_ID'])
+ else:
+ root_url = os.path.join(self.topobjdir, 'gradle/build/mobile/android/app/outputs')
+
+ reports = ('officialAustralisDebug',
+ 'officialPhotonDebug',)
+ for report in reports:
+ f = open(os.path.join(self.topobjdir, 'gradle/build/mobile/android/app/outputs/lint-results-{}.xml'.format(report)), 'rt')
+ tree = ET.parse(f)
+ root = tree.getroot()
+
+ print('SUITE-START | android-lint | {}'.format(report))
+ for issue in root.findall("issue[@severity='Error']"):
+ # There's no particular advantage to formatting the
+ # error, so for now let's just output the <issue> XML
+ # tag.
+ for line in ET.tostring(issue).strip().splitlines():
+ print('TEST-UNEXPECTED-FAIL | {}'.format(line))
+ ret |= 1
+ print('SUITE-END | android-lint | {}'.format(report))
+
+ title = report
+ print("TinderboxPrint: report<br/><a href='{}/lint-results-{}.html'>HTML {} report</a>, visit \"Inspect Task\" link for details".format(root_url, report, title))
+ print("TinderboxPrint: report<br/><a href='{}/lint-results-{}.xml'>XML {} report</a>, visit \"Inspect Task\" link for details".format(root_url, report, title))
+
+ return ret
+
+
+ @SubCommand('android', 'checkstyle',
+ """Run Android checkstyle.
+ See https://developer.mozilla.org/en-US/docs/Mozilla/Android-specific_test_suites#android-checkstyle""")
@CommandArgument('args', nargs=argparse.REMAINDER)
- def android(self, args):
- # Avoid logging the command
- self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
+ def android_checkstyle(self, args):
+ gradle_targets = [
+ 'app:checkstyle',
+ ]
+ ret = self.gradle(gradle_targets + ["--continue"] + args, verbose=True)
+
+ # Checkstyle produces both HTML and XML reports. Visit the
+ # XML report(s) to report errors and link to the HTML
+ # report(s) for human consumption.
+ import xml.etree.ElementTree as ET
+
+ f = open(os.path.join(self.topobjdir, 'gradle/build/mobile/android/app/reports/checkstyle/checkstyle.xml'), 'rt')
+ tree = ET.parse(f)
+ root = tree.getroot()
+
+ print('SUITE-START | android-checkstyle')
+ for file in root.findall('file'):
+ name = file.get('name')
+
+ print('TEST-START | {}'.format(name))
+ error_count = 0
+ for error in file.findall('error'):
+ # There's no particular advantage to formatting the
+ # error, so for now let's just output the <error> XML
+ # tag.
+ print('TEST-UNEXPECTED-FAIL | {}'.format(name))
+ for line in ET.tostring(error).strip().splitlines():
+ print('TEST-UNEXPECTED-FAIL | {}'.format(line))
+ error_count += 1
+ ret |= 1
+
+ if not error_count:
+ print('TEST-PASS | {}'.format(name))
+ print('SUITE-END | android-checkstyle')
+
+ # Now the reports, linkified.
+ if 'TASK_ID' in os.environ and 'RUN_ID' in os.environ:
+ root_url = "https://queue.taskcluster.net/v1/task/{}/runs/{}/artifacts/public/android/checkstyle".format(os.environ['TASK_ID'], os.environ['RUN_ID'])
+ else:
+ root_url = os.path.join(self.topobjdir, 'gradle/build/mobile/android/app/reports/checkstyle')
+
+ print("TinderboxPrint: report<br/><a href='{}/checkstyle.html'>HTML checkstyle report</a>, visit \"Inspect Task\" link for details".format(root_url))
+ print("TinderboxPrint: report<br/><a href='{}/checkstyle.xml'>XML checkstyle report</a>, visit \"Inspect Task\" link for details".format(root_url))
+
+ return ret
+
+
+ @SubCommand('android', 'findbugs',
+ """Run Android findbugs.
+ See https://developer.mozilla.org/en-US/docs/Mozilla/Android-specific_test_suites#android-findbugs""")
+ @CommandArgument('args', nargs=argparse.REMAINDER)
+ def android_findbugs(self, dryrun=False, args=[]):
+ gradle_targets = [
+ 'app:findbugsXmlOfficialAustralisDebug',
+ 'app:findbugsHtmlOfficialAustralisDebug',
+ 'app:findbugsXmlOfficialPhotonDebug',
+ 'app:findbugsHtmlOfficialPhotonDebug',
+ ]
+ ret = self.gradle(gradle_targets + ["--continue"] + args, verbose=True)
+
+ # Findbug produces both HTML and XML reports. Visit the
+ # XML report(s) to report errors and link to the HTML
+ # report(s) for human consumption.
+ import xml.etree.ElementTree as ET
- return self.run_process(
- [os.path.join(self.substs['ANDROID_TOOLS'], 'android')] + args,
- pass_thru=True, # Allow user to run gradle interactively.
- ensure_exit_code=False, # Don't throw on non-zero exit code.
- cwd=mozpath.join(self.topsrcdir))
+ if 'TASK_ID' in os.environ and 'RUN_ID' in os.environ:
+ root_url = "https://queue.taskcluster.net/v1/task/{}/runs/{}/artifacts/public/artifacts/findbugs".format(os.environ['TASK_ID'], os.environ['RUN_ID'])
+ else:
+ root_url = os.path.join(self.topobjdir, 'gradle/build/mobile/android/app/outputs/findbugs')
+
+ reports = ('findbugs-officialAustralisDebug-output.xml',
+ 'findbugs-officialPhotonDebug-output.xml',)
+ for report in reports:
+ try:
+ f = open(os.path.join(self.topobjdir, 'gradle/build/mobile/android/app/outputs/findbugs', report), 'rt')
+ except IOError:
+ continue
+
+ tree = ET.parse(f)
+ root = tree.getroot()
+
+ print('SUITE-START | android-findbugs | {}'.format(report))
+ for error in root.findall('./BugInstance'):
+ # There's no particular advantage to formatting the
+ # error, so for now let's just output the <error> XML
+ # tag.
+ print('TEST-UNEXPECTED-FAIL | {}:{} | {}'.format(report, error.get('type'), error.find('Class').get('classname')))
+ for line in ET.tostring(error).strip().splitlines():
+ print('TEST-UNEXPECTED-FAIL | {}:{} | {}'.format(report, error.get('type'), line))
+ ret |= 1
+ print('SUITE-END | android-findbugs | {}'.format(report))
+
+ title = report.replace('findbugs-', '').replace('-output.xml', '')
+ print("TinderboxPrint: report<br/><a href='{}/{}'>HTML {} report</a>, visit \"Inspect Task\" link for details".format(root_url, report.replace('.xml', '.html'), title))
+ print("TinderboxPrint: report<br/><a href='{}/{}'>XML {} report</a>, visit \"Inspect Task\" link for details".format(root_url, report, title))
+
+ return ret
+
+
+ @SubCommand('android', 'gradle-dependencies',
+ """Collect Android Gradle dependencies.
+ See https://gecko.readthedocs.io/en/latest/build/buildsystem/toolchains.html#firefox-for-android-with-gradle""")
+ @CommandArgument('args', nargs=argparse.REMAINDER)
+ def android_gradle_dependencies(self, args):
+ # The union, plus a bit more, of all of the Gradle tasks
+ # invoked by the android-* automation jobs.
+ gradle_targets = [
+ 'app:checkstyle',
+ 'app:assembleOfficialAustralisRelease',
+ 'app:assembleOfficialAustralisDebug',
+ 'app:assembleOfficialAustralisDebugAndroidTest',
+ 'app:findbugsXmlOfficialAustralisDebug',
+ 'app:findbugsHtmlOfficialAustralisDebug',
+ 'app:lintOfficialAustralisDebug',
+ 'app:assembleOfficialPhotonRelease',
+ 'app:assembleOfficialPhotonDebug',
+ 'app:assembleOfficialPhotonDebugAndroidTest',
+ 'app:findbugsXmlOfficialPhotonDebug',
+ 'app:findbugsHtmlOfficialPhotonDebug',
+ 'app:lintOfficialPhotonDebug',
+ # Does not include Gecko binaries -- see mobile/android/gradle/with_gecko_binaries.gradle.
+ 'geckoview:assembleWithoutGeckoBinaries',
+ # So that we pick up the test dependencies for the builders.
+ 'geckoview_example:assembleWithoutGeckoBinaries',
+ 'geckoview_example:assembleWithoutGeckoBinariesAndroidTest',
+ ]
+ ret = self.gradle(gradle_targets + ["--continue"] + args, verbose=True)
+
+ return ret
+
@Command('gradle', category='devenv',
description='Run gradle.',
conditions=[conditions.is_android])
+ @CommandArgument('-v', '--verbose', action='store_true',
+ help='Verbose output for what commands the build is running.')
@CommandArgument('args', nargs=argparse.REMAINDER)
- def gradle(self, args):
- # Avoid logging the command
- self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
-
+ def gradle(self, args, verbose=False):
+ if not verbose:
+ # Avoid logging the command
+ self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
# In automation, JAVA_HOME is set via mozconfig, which needs
# to be specially handled in each mach command. This turns
# $JAVA_HOME/bin/java into $JAVA_HOME.
java_home = os.path.dirname(os.path.dirname(self.substs['JAVA']))
gradle_flags = shell_split(self.substs.get('GRADLE_FLAGS', ''))
--- a/testing/mozharness/configs/builds/releng_sub_android_configs/64_api_15_gradle_dependencies.py
+++ b/testing/mozharness/configs/builds/releng_sub_android_configs/64_api_15_gradle_dependencies.py
@@ -2,27 +2,14 @@ config = {
'base_name': 'Android armv7 API 15+ Gradle dependencies %(branch)s',
'stage_platform': 'android-api-15-gradle-dependencies',
'build_type': 'api-15-opt',
'src_mozconfig': 'mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly',
'multi_locale_config_platform': 'android',
# gradle-dependencies doesn't produce a package. So don't collect package metrics.
'disable_package_metrics': True,
'postflight_build_mach_commands': [
- ['gradle',
- 'app:assembleOfficialAustralisRelease',
- 'app:assembleOfficialAustralisDebug',
- 'app:assembleOfficialAustralisDebugAndroidTest',
- 'app:findbugsOfficialAustralisDebug',
- 'app:assembleOfficialPhotonRelease',
- 'app:assembleOfficialPhotonDebug',
- 'app:assembleOfficialPhotonDebugAndroidTest',
- 'app:findbugsOfficialPhotonDebug',
- # Does not include Gecko binaries -- see mobile/android/gradle/with_gecko_binaries.gradle.
- 'geckoview:assembleWithoutGeckoBinaries',
- # So that we pick up the test dependencies for the builders.
- 'geckoview_example:assembleWithoutGeckoBinaries',
- 'geckoview_example:assembleWithoutGeckoBinariesAndroidTest',
- 'checkstyle',
+ ['android',
+ 'gradle-dependencies',
],
],
'artifact_flag_build_variant_in_try': None, # There's no artifact equivalent.
}
--- a/testing/mozharness/configs/builds/releng_sub_android_configs/64_checkstyle.py
+++ b/testing/mozharness/configs/builds/releng_sub_android_configs/64_checkstyle.py
@@ -2,12 +2,14 @@ config = {
'base_name': 'Android checkstyle %(branch)s',
'stage_platform': 'android-checkstyle',
'build_type': 'api-15-opt',
'src_mozconfig': 'mobile/android/config/mozconfigs/android-api-15-frontend/nightly',
'multi_locale_config_platform': 'android',
# checkstyle doesn't produce a package. So don't collect package metrics.
'disable_package_metrics': True,
'postflight_build_mach_commands': [
- ['gradle', 'app:checkstyle'],
+ ['android',
+ 'checkstyle',
+ ],
],
'artifact_flag_build_variant_in_try': None, # There's no artifact equivalent.
}
--- a/testing/mozharness/configs/builds/releng_sub_android_configs/64_findbugs.py
+++ b/testing/mozharness/configs/builds/releng_sub_android_configs/64_findbugs.py
@@ -2,15 +2,14 @@ config = {
'base_name': 'Android findbugs %(branch)s',
'stage_platform': 'android-findbugs',
'build_type': 'api-15-opt',
'src_mozconfig': 'mobile/android/config/mozconfigs/android-api-15-frontend/nightly',
'multi_locale_config_platform': 'android',
# findbugs doesn't produce a package. So don't collect package metrics.
'disable_package_metrics': True,
'postflight_build_mach_commands': [
- ['gradle',
- 'app:findbugsOfficialAustralisDebug',
- 'app:findbugsOfficialPhotonDebug',
+ ['android',
+ 'findbugs',
],
],
'artifact_flag_build_variant_in_try': None, # There's no artifact equivalent.
}
--- a/testing/mozharness/configs/builds/releng_sub_android_configs/64_lint.py
+++ b/testing/mozharness/configs/builds/releng_sub_android_configs/64_lint.py
@@ -2,15 +2,14 @@ config = {
'base_name': 'Android lint %(branch)s',
'stage_platform': 'android-lint',
'build_type': 'api-15-opt',
'src_mozconfig': 'mobile/android/config/mozconfigs/android-api-15-frontend/nightly',
'multi_locale_config_platform': 'android',
# lint doesn't produce a package. So don't collect package metrics.
'disable_package_metrics': True,
'postflight_build_mach_commands': [
- ['gradle',
- 'app:lintOfficialAustralisDebug',
- 'app:lintOfficialPhotonDebug',
+ ['android',
+ 'lint',
],
],
'artifact_flag_build_variant_in_try': None, # There's no artifact equivalent.
}
--- a/testing/mozharness/configs/builds/releng_sub_android_configs/64_test.py
+++ b/testing/mozharness/configs/builds/releng_sub_android_configs/64_test.py
@@ -2,15 +2,14 @@ config = {
'base_name': 'Android armv7 unit tests %(branch)s',
'stage_platform': 'android-test',
'build_type': 'api-15-opt',
'src_mozconfig': 'mobile/android/config/mozconfigs/android-api-15-frontend/nightly',
'multi_locale_config_platform': 'android',
# unit tests don't produce a package. So don't collect package metrics.
'disable_package_metrics': True,
'postflight_build_mach_commands': [
- ['gradle',
- 'app:testOfficialAustralisDebugUnitTest',
- 'app:testOfficialPhotonDebugUnitTest',
+ ['android',
+ 'test',
],
],
'artifact_flag_build_variant_in_try': None, # There's no artifact equivalent.
}