--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1625,17 +1625,17 @@ class StaticAnalysisMonitor(object):
return (warning, True)
@CommandProvider
class StaticAnalysis(MachCommandBase):
"""Utilities for running C++ static analysis checks and format."""
# List of file extension to consider (should start with dot)
- _format_include_extensions = ('.cpp', '.c', '.h')
+ _format_include_extensions = ('.cpp', '.c', '.h', '.java')
# File contaning all paths to exclude from formatting
_format_ignore_file = '.clang-format-ignore'
@Command('static-analysis', category='testing',
description='Run C++ static analysis checks')
def static_analysis(self):
# If not arguments are provided, just print a help message.
mach = Mach(os.getcwd())
@@ -1707,17 +1707,151 @@ class StaticAnalysis(MachCommandBase):
footer = StaticAnalysisFooter(self.log_manager.terminal, monitor)
with StaticAnalysisOutputManager(self.log_manager, monitor, footer) as output:
rc = self.run_process(args=args, line_handler=output.on_line, cwd=cwd)
self.log(logging.WARNING, 'warning_summary',
{'count': len(monitor.warnings_db)},
'{count} warnings present.')
+ if rc != 0:
return rc
+ # if we are building firefox for android it might be nice to
+ # also analyze the java code base
+ if self.substs['MOZ_BUILD_APP'] == 'mobile/android':
+ rc = self.check_java(source, jobs, strip, verbose, skip_export=True)
+ return rc
+
+ @StaticAnalysisSubCommand('static-analysis', 'check-java',
+ 'Run infer on the java codebase.')
+ @CommandArgument('source', nargs='*', default=['mobile'],
+ help='Source files to be analyzed. '
+ 'Can be omitted, in which case the entire code base '
+ 'is analyzed. The source argument is ignored if '
+ 'there is anything fed through stdin, in which case '
+ 'the analysis is only performed on the files changed '
+ 'in the patch streamed through stdin. This is called '
+ 'the diff mode.')
+ @CommandArgument('--checks', '-c', default=[], metavar='checks', nargs='*',
+ help='Static analysis checks to enable.')
+ @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int,
+ help='Number of concurrent jobs to run.'
+ ' Default is the number of CPUs.')
+ @CommandArgument('--task', '-t', type=str,
+ default='compileLocalWithGeckoBinariesNoMinApiDebugSources',
+ help='Which gradle tasks to use to compile the java codebase.')
+ def check_java(self, source=['mobile'], jobs=2, strip=1, verbose=False, checks=[],
+ task='compileLocalWithGeckoBinariesNoMinApiDebugSources',
+ skip_export=False):
+ self._set_log_level(verbose)
+ self.log_manager.enable_all_structured_loggers()
+ if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
+ self.log(logging.WARNING, 'static-analysis', {},
+ 'Cannot check java source code unless you are building for android!')
+ return 1
+ # if source contains the whole mobile folder, then we just have to
+ # analyze everything
+ check_all = any(i.rstrip(os.sep).split(os.sep)[-1] == 'mobile' for i in source)
+ # gather all java sources from the source variable
+ java_sources = []
+ if not check_all:
+ java_sources = self._get_java_files(source)
+ if not java_sources:
+ return 0
+ if not skip_export:
+ rc = self._build_export(jobs=jobs, verbose=verbose)
+ if rc != 0:
+ return rc
+ rc = self._get_infer(verbose=verbose)
+ if rc != 0:
+ self.log(logging.WARNING, 'static-analysis', {},
+ 'This command is only available for linux64!')
+ return rc
+ # which checkers to use, and which folders to exclude
+ all_checkers, third_party_path = self._get_infer_config()
+ checkers, excludes = self._get_infer_args(
+ checks=checks or all_checkers,
+ third_party_path=third_party_path
+ )
+ gradlew = mozpath.join(self.topsrcdir, 'gradlew')
+ # infer capture command
+ capture_cmd = [self._infer_path, 'capture'] + excludes + \
+ ['--', gradlew, task]
+ tmp_file, args = self._get_infer_source_args(java_sources)
+ # infer analyze command
+ analysis_cmd = [self._infer_path, 'analyze', '--keep-going'] + \
+ checkers + args
+ # capture, then analyze the sources
+ for args in [[gradlew, 'clean'], capture_cmd, analysis_cmd]:
+ rc = self.run_process(args=args, cwd=self.topsrcdir,
+ pass_thru=True)
+ # if a command fails, break and close the tmp file before returning
+ if rc != 0:
+ break
+ if tmp_file:
+ tmp_file.close()
+ return rc
+
+ def _get_java_files(self, sources):
+ java_sources = []
+ for i in sources:
+ f = mozpath.join(self.topsrcdir, i)
+ if os.path.isdir(f):
+ for root, dirs, files in os.walk(f):
+ dirs.sort()
+ for file in sorted(files):
+ if file.endswith('.java'):
+ java_sources.append(mozpath.join(root, file))
+ elif f.endswith('.java'):
+ java_sources.append(f)
+ return java_sources
+
+ def _get_infer_source_args(self, sources):
+ '''Return the arguments to only analyze <sources>'''
+ if not sources:
+ return (None, [])
+ # create a temporary file in which we place all sources
+ # this is used by the analysis command to only analyze certain files
+ f = tempfile.NamedTemporaryFile()
+ for source in sources:
+ f.write(source+'\n')
+ f.flush()
+ return (f, ['--changed-files-index', f.name])
+
+ def _get_infer_config(self):
+ '''Load the infer config file.'''
+ import yaml
+ checkers = []
+ tp_path = ''
+ with open(mozpath.join(self.topsrcdir, 'tools',
+ 'infer', 'config.yaml')) as f:
+ try:
+ config = yaml.safe_load(f)
+ for item in config['infer_checkers']:
+ if item['publish']:
+ checkers.append(item['name'])
+ tp_path = mozpath.join(self.topsrcdir, config['third_party'])
+ except Exception as e:
+ print('Looks like config.yaml is not valid, so we are unable '
+ 'to determine default checkers, and which folder to '
+ 'exclude, using defaults provided by infer')
+ return checkers, tp_path
+
+ def _get_infer_args(self, checks, third_party_path):
+ '''Return the arguments which include the checkers <checks>, and
+ excludes all folder in <third_party_path>.'''
+ checkers = ['-a', 'checkers']
+ excludes = []
+ for checker in checks:
+ checkers.append('--' + checker)
+ with open(third_party_path) as f:
+ for line in f:
+ excludes.append('--skip-analysis-in-path')
+ excludes.append(line.strip('\n'))
+ return checkers, excludes
def _get_clang_tidy_command(self, checks, header_filter, sources, jobs, fix):
if checks == '-*':
checks = self._get_checks()
common_args = ['-clang-tidy-binary', self._clang_tidy_path,
'-clang-apply-replacements-binary', self._clang_apply_replacements,
@@ -1741,17 +1875,18 @@ class StaticAnalysis(MachCommandBase):
@StaticAnalysisSubCommand('static-analysis', 'autotest',
'Run the auto-test suite in order to determine that'
' the analysis did not regress.')
@CommandArgument('--dump-results', '-d', default=False, action='store_true',
help='Generate the baseline for the regression test. Based on'
' this baseline we will test future results.')
@CommandArgument('--intree-tool', '-i', default=False, action='store_true',
help='Use a pre-aquired in-tree clang-tidy package.')
- @CommandArgument('checker_names', nargs='*', default=[], help='Checkers that are going to be auto-tested.')
+ @CommandArgument('checker_names', nargs='*', default=[],
+ help='Checkers that are going to be auto-tested.')
def autotest(self, verbose=False, dump_results=False, intree_tool=False, checker_names=[]):
# If 'dump_results' is True than we just want to generate the issues files for each
# checker in particulat and thus 'force_download' becomes 'False' since we want to
# do this on a local trusted clang-tidy package.
self._set_log_level(verbose)
self._dump_results = dump_results
force_download = True
@@ -1788,29 +1923,31 @@ class StaticAnalysis(MachCommandBase):
"clang", "clang-format-diff.py")
# Ensure that clang-tidy is present
rc = not os.path.exists(self._clang_tidy_path)
else:
rc = self._get_clang_tools(force=force_download, verbose=verbose)
if rc != 0:
- self.log(logging.ERROR, 'ERROR: static-analysis', {}, 'clang-tidy unable to locate package.')
+ self.log(logging.ERROR, 'ERROR: static-analysis', {},
+ 'clang-tidy unable to locate package.')
return self.TOOLS_FAILED_DOWNLOAD
self._clang_tidy_base_path = mozpath.join(self.topsrcdir, "tools", "clang-tidy")
# For each checker run it
f = open(mozpath.join(self._clang_tidy_base_path, "config.yaml"))
import yaml
config = yaml.safe_load(f)
platform, _ = self.platform
if platform not in config['platforms']:
- self.log(logging.ERROR, 'static-analysis', {},"RUNNING: clang-tidy autotest for platform {} not supported.".format(platform))
+ self.log(logging.ERROR, 'static-analysis', {},
+ "RUNNING: clang-tidy autotest for platform {} not supported.".format(platform))
return TOOLS_UNSUPORTED_PLATFORM
import concurrent.futures
import multiprocessing
import shutil
max_workers = multiprocessing.cpu_count()
@@ -1870,50 +2007,65 @@ class StaticAnalysis(MachCommandBase):
json.dump(compile_commands, file_handler)
file_handler.flush()
return directory
@StaticAnalysisSubCommand('static-analysis', 'install',
'Install the static analysis helper tool')
@CommandArgument('source', nargs='?', type=str,
- help='Where to fetch a local archive containing the static-analysis and format helper tool.'
- 'It will be installed in ~/.mozbuild/clang-tools/.'
- 'Can be omitted, in which case the latest clang-tools '
- ' helper for the platform would be automatically '
- 'detected and installed.')
+ help='Where to fetch a local archive containing the static-analysis and '
+ 'format helper tool.'
+ 'It will be installed in ~/.mozbuild/clang-tools and ~/.mozbuild/infer.'
+ 'Can be omitted, in which case the latest clang-tools and infer '
+ 'helper for the platform would be automatically detected and installed.')
@CommandArgument('--skip-cache', action='store_true',
help='Skip all local caches to force re-fetching the helper tool.',
default=False)
def install(self, source=None, skip_cache=False, verbose=False):
self._set_log_level(verbose)
rc = self._get_clang_tools(force=True, skip_cache=skip_cache,
source=source, verbose=verbose)
+ if rc == 0:
+ # XXX ignore the return code because if it fails or not, infer is
+ # not mandatory, but clang-tidy is
+ self._get_infer(force=True, skip_cache=skip_cache, verbose=verbose)
return rc
@StaticAnalysisSubCommand('static-analysis', 'clear-cache',
'Delete local helpers and reset static analysis helper tool cache')
def clear_cache(self, verbose=False):
self._set_log_level(verbose)
rc = self._get_clang_tools(force=True, download_if_needed=True, skip_cache=True,
verbose=verbose)
+ if rc == 0:
+ self._get_infer(force=True, download_if_needed=True, skip_cache=True,
+ verbose=verbose)
if rc != 0:
return rc
-
- self._artifact_manager.artifact_clear_cache()
+ return self._artifact_manager.artifact_clear_cache()
@StaticAnalysisSubCommand('static-analysis', 'print-checks',
'Print a list of the static analysis checks performed by default')
def print_checks(self, verbose=False):
self._set_log_level(verbose)
rc = self._get_clang_tools(verbose=verbose)
+ if rc == 0:
+ rc = self._get_infer(verbose=verbose)
if rc != 0:
return rc
args = [self._clang_tidy_path, '-list-checks', '-checks=%s' % self._get_checks()]
- return self._run_command_in_objdir(args=args, pass_thru=True)
+ rc = self._run_command_in_objdir(args=args, pass_thru=True)
+ if rc != 0:
+ return rc
+ checkers, _ = self._get_infer_config()
+ print('Infer checks:')
+ for checker in checkers:
+ print(' '*4 + checker)
+ return 0
@Command('clang-format', category='misc', description='Run clang-format on current changes')
@CommandArgument('--show', '-s', action='store_true', default=False,
help='Show diff output on instead of applying changes')
@CommandArgument('--path', '-p', nargs='+', default=None,
help='Specify the path(s) to reformat')
def clang_format(self, show, path, verbose=False):
# Run clang-format or clang-format-diff on the local changes
@@ -1934,17 +2086,18 @@ class StaticAnalysis(MachCommandBase):
return self._run_clang_format_path(self._clang_format_path, show, path)
def _verify_checker(self, item):
check = item['name']
test_file_path = mozpath.join(self._clang_tidy_base_path, "test", check)
test_file_path_cpp = test_file_path + '.cpp'
test_file_path_json = test_file_path + '.json'
- self.log(logging.INFO, 'static-analysis', {},"RUNNING: clang-tidy checker {}.".format(check))
+ self.log(logging.INFO, 'static-analysis', {},
+ "RUNNING: clang-tidy checker {}.".format(check))
# Verify if this checker actually exists
if not check in self._clang_tidy_checks:
self.log(logging.ERROR, 'static-analysis', {}, "ERROR: clang-tidy checker {} doesn't exist in this clang-tidy version.".format(check))
return self.TOOLS_CHECKER_NOT_FOUND
# Verify if the test file exists for this checker
if not os.path.exists(test_file_path_cpp):
@@ -2226,16 +2379,57 @@ class StaticAnalysis(MachCommandBase):
continue # empty or comment
magics = ['exclude']
if pattern.startswith('^'):
magics += ['top']
pattern = pattern[1:]
args += [':({0}){1}'.format(','.join(magics), pattern)]
return args
+ def _get_infer(self, force=False, skip_cache=False,
+ download_if_needed=True, verbose=False):
+ rc, config, _ = self._get_config_environment()
+ if rc != 0:
+ return rc
+ infer_path = mozpath.join(self._mach_context.state_dir, 'infer')
+ self._infer_path = mozpath.join(infer_path, 'infer', 'bin',
+ 'infer' +
+ config.substs.get('BIN_SUFFIX', ''))
+ if os.path.exists(self._infer_path) and not force:
+ return 0
+ else:
+ if os.path.isdir(infer_path) and download_if_needed:
+ # The directory exists, perhaps it's corrupted? Delete it
+ # and start from scratch.
+ import shutil
+ shutil.rmtree(infer_path)
+ return self._get_infer(force=force, skip_cache=skip_cache,
+ verbose=verbose,
+ download_if_needed=download_if_needed)
+ os.mkdir(infer_path)
+ self._artifact_manager = PackageFrontend(self._mach_context)
+ if not download_if_needed:
+ return 0
+ job, _ = self.platform
+ if job != 'linux64':
+ return -1
+ else:
+ job += '-infer'
+ # We want to unpack data in the infer mozbuild folder
+ currentWorkingDir = os.getcwd()
+ os.chdir(infer_path)
+ rc = self._artifact_manager.artifact_toolchain(verbose=verbose,
+ skip_cache=skip_cache,
+ from_build=[job],
+ no_unpack=False,
+ retry=0)
+ # Change back the cwd
+ os.chdir(currentWorkingDir)
+ return rc
+
def _run_clang_format_diff(self, clang_format_diff, clang_format, show):
# Run clang-format on the diff
# Note that this will potentially miss a lot things
from subprocess import Popen, PIPE, check_output, CalledProcessError
diff_process = Popen(self._get_clang_format_diff_command(), stdout=PIPE)
args = [sys.executable, clang_format_diff, "-p1", "-binary=%s" % clang_format]
@@ -2527,9 +2721,8 @@ class Analyze(MachCommandBase):
if os.path.isfile(path):
g = Graph(path)
g.file_summaries(files)
g.close()
else:
res = 'Please make sure you have a local tup db *or* specify the location with --path.'
print ('Could not find a valid tup db in ' + path, res, sep='\n')
return 1
-