Bug 1381069 - Remove ccov gcda files after talos browser initialization (production only); r?jmaher draft
authorRob Wood <rwood@mozilla.com>
Wed, 20 Dec 2017 18:38:23 -0500
changeset 715922 cdb9f7d00447bde493c1385228fa6d2b30ac438e
parent 715308 cc9caa712156f8c278c34cf8844af0bf17cd8150
child 744914 423246da93a36bc6958826c9f6b6db583cf7e0ac
push id94290
push userrwood@mozilla.com
push dateThu, 04 Jan 2018 21:59:12 +0000
reviewersjmaher
bugs1381069
milestone59.0a1
Bug 1381069 - Remove ccov gcda files after talos browser initialization (production only); r?jmaher MozReview-Commit-ID: 5V5xghMqfzY
testing/mozharness/mozharness/mozilla/testing/talos.py
testing/talos/talos/cmdline.py
testing/talos/talos/ffsetup.py
testing/talos/talos/run_tests.py
testing/talos/talos/ttest.py
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -366,16 +366,18 @@ class Talos(TestingMixin, MercurialScrip
             options.extend(['--%s' % key, value])
         # configure profiling options
         options.extend(self.query_gecko_profile_options())
         # extra arguments
         if args is not None:
             options += args
         if 'talos_extra_options' in self.config:
             options += self.config['talos_extra_options']
+        if self.config.get('code_coverage', False):
+            options.extend(['--code-coverage'])
         return options
 
     def populate_webroot(self):
         """Populate the production test slaves' webroots"""
         self.talos_path = os.path.join(
             self.query_abs_dirs()['abs_work_dir'], 'tests', 'talos'
         )
 
--- a/testing/talos/talos/cmdline.py
+++ b/testing/talos/talos/cmdline.py
@@ -175,15 +175,21 @@ def create_parser(mach_interface=False):
     debug_options.add_argument('--debug', action='store_true',
                                help='Enable the debugger. Not specifying a --debugger option will'
                                     'result in the default debugger being used.')
     debug_options.add_argument('--debugger', default=None,
                                help='Name of debugger to use.')
     debug_options.add_argument('--debugger-args', default=None, metavar='params',
                                help='Command-line arguments to pass to the debugger itself; split'
                                     'as the Bourne shell would.')
+    add_arg('--code-coverage', action="store_true",
+            dest='code_coverage',
+            help='Remove any existing ccov gcda output files after browser'
+                 ' initialization but before starting the tests. NOTE:'
+                 ' Currently only supported in production.')
+
     add_logging_group(parser)
     return parser
 
 
 def parse_args(argv=None):
     parser = create_parser()
     return parser.parse_args(argv)
--- a/testing/talos/talos/ffsetup.py
+++ b/testing/talos/talos/ffsetup.py
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 """
 Set up a browser environment before running a test.
 """
 from __future__ import absolute_import, print_function
 
 import os
+import shutil
 import tempfile
 
 import mozfile
 import mozinfo
 import mozrunner
 from mozlog import get_proxy_logger
 from mozprocess import ProcessHandlerMixin
 from mozprofile.profile import Profile
@@ -180,31 +181,113 @@ class FFSetup(object):
                                               self.browser_config,
                                               self.test_config)
             self.gecko_profile.update_env(self.env)
 
     def clean(self):
         try:
             mozfile.remove(self._tmp_dir)
         except Exception as e:
-            print("Exception while removing profile directory: %s" % self._tmp_dir)
-            print(e)
+            LOG.info("Exception while removing profile directory: %s" % self._tmp_dir)
+            LOG.info(e)
 
         if self.gecko_profile:
             self.gecko_profile.clean()
 
+    def collect_or_clean_ccov(self, clean=False):
+        # NOTE: Currently only supported when running in production
+        if not self.browser_config.get('develop', False):
+            # first see if we an find any ccov files at the ccov output dirs
+            if clean:
+                LOG.info("Cleaning ccov files before starting the talos test")
+            else:
+                LOG.info("Collecting ccov files that were generated during the talos test")
+            gcov_prefix = os.getenv('GCOV_PREFIX', None)
+            js_ccov_dir = os.getenv('JS_CODE_COVERAGE_OUTPUT_DIR', None)
+            gcda_archive_folder_name = 'gcda-archive'
+            _gcda_files_found = []
+
+            for _ccov_env in [gcov_prefix, js_ccov_dir]:
+                if _ccov_env is not None:
+                    # ccov output dir env vars exist; now search for gcda files to remove
+                    _ccov_path = os.path.abspath(_ccov_env)
+                    if os.path.exists(_ccov_path):
+                        # now walk through and look for gcda files
+                        LOG.info("Recursive search for gcda files in: %s" % _ccov_path)
+                        for root, dirs, files in os.walk(_ccov_path):
+                            for next_file in files:
+                                if next_file.endswith(".gcda"):
+                                    # don't want to move or delete files in our 'gcda-archive'
+                                    if root.find(gcda_archive_folder_name) == -1:
+                                        _gcda_files_found.append(os.path.join(root, next_file))
+                    else:
+                        LOG.info("The ccov env var path doesn't exist: %s" % str(_ccov_path))
+
+            # now  clean or collect gcda files accordingly
+            if clean:
+                # remove ccov data
+                LOG.info("Found %d gcda files to clean. Deleting..." % (len(_gcda_files_found)))
+                for _gcda in _gcda_files_found:
+                    try:
+                        mozfile.remove(_gcda)
+                    except Exception as e:
+                        LOG.info("Exception while removing file: %s" % _gcda)
+                        LOG.info(e)
+                LOG.info("Finished cleaning ccov gcda files")
+            else:
+                # copy gcda files to archive folder to be collected later
+                gcda_archive_top = os.path.join(gcov_prefix,
+                                                gcda_archive_folder_name,
+                                                self.test_config['name'])
+                LOG.info("Found %d gcda files to collect. Moving to gcda archive %s"
+                         % (len(_gcda_files_found), str(gcda_archive_top)))
+                if not os.path.exists(gcda_archive_top):
+                    try:
+                        os.makedirs(gcda_archive_top)
+                    except OSError:
+                        LOG.critical("Unable to make gcda archive folder %s" % gcda_archive_top)
+                for _gcda in _gcda_files_found:
+                    # want to copy the existing directory strucutre but put it under archive-dir
+                    # need to remove preceeding '/' from _gcda file name so can join the path
+                    gcda_archive_file = os.path.join(gcov_prefix,
+                                                     gcda_archive_folder_name,
+                                                     self.test_config['name'],
+                                                     _gcda.strip(gcov_prefix + "//"))
+                    gcda_archive_dest = os.path.dirname(gcda_archive_file)
+
+                    # create archive folder, mirroring structure
+                    if not os.path.exists(gcda_archive_dest):
+                        try:
+                            os.makedirs(gcda_archive_dest)
+                        except OSError:
+                            LOG.critical("Unable to make archive folder %s" % gcda_archive_dest)
+                    # now copy the file there
+                    try:
+                        shutil.copy(_gcda, gcda_archive_dest)
+                    except Exception as e:
+                        LOG.info("Error copying %s to %s" % (str(_gcda), str(gcda_archive_dest)))
+                        LOG.info(e)
+                LOG.info("Finished collecting ccov gcda files. Copied to: %s" % gcda_archive_top)
+
     def __enter__(self):
         LOG.info('Initialising browser for %s test...'
                  % self.test_config['name'])
         self._init_env()
         self._init_profile()
         try:
             if not self.debug_mode and self.test_config['name'] != "damp":
                 self._run_profile()
         except:
             self.clean()
             raise
         self._init_gecko_profile()
         LOG.info('Browser initialized.')
+        # remove ccov files before actual tests start
+        if self.browser_config.get('code_coverage', False):
+            # if the Firefox build was instrumented for ccov, initializing the browser
+            # will have caused ccov to output some gcda files; in order to have valid
+            # ccov data for the talos test we want to remove these files before starting
+            # the actual talos test(s)
+            self.collect_or_clean_ccov(clean=True)
         return self
 
     def __exit__(self, type, value, tb):
         self.clean()
--- a/testing/talos/talos/run_tests.py
+++ b/testing/talos/talos/run_tests.py
@@ -113,16 +113,28 @@ def run_tests(config, browser_config):
     # instance
     if browser_config['develop']:
         browser_config['extra_args'] = '--no-remote'
 
     # Pass subtests filter argument via a preference
     if browser_config['subtests']:
         browser_config['preferences']['talos.subtests'] = browser_config['subtests']
 
+    # If --code-coverage files are expected, set flag in browser config so ffsetup knows
+    # that it needs to delete any ccov files resulting from browser initialization
+    # NOTE: This is only supported in production; local setup of ccov folders and
+    # data collection not supported yet, so if attempting to run with --code-coverage
+    # flag locally, that is not supported yet
+    if config.get('code_coverage', False):
+        if browser_config['develop']:
+            raise TalosError('Aborting: talos --code-coverage flag is only '
+                             'supported in production')
+        else:
+            browser_config['code_coverage'] = True
+
     # set defaults
     testdate = config.get('testdate', '')
 
     # get the process name from the path to the browser
     if not browser_config['process']:
         browser_config['process'] = \
             os.path.basename(browser_config['browser_path'])
 
--- a/testing/talos/talos/ttest.py
+++ b/testing/talos/talos/ttest.py
@@ -286,10 +286,15 @@ class TTest(object):
         # include global (cross-cycle) counters
         test_results.all_counter_results.extend(
             [{key: value} for key, value in global_counters.items()]
         )
         for c in test_results.all_counter_results:
             for key, value in c.items():
                 LOG.debug('COUNTER %r: %s' % (key, value))
 
+        # if running against a code-coverage instrumented build, move the
+        # produced gcda files to a folder where they will be collected later
+        if browser_config.get('code_coverage', False):
+            setup.collect_or_clean_ccov()
+
         # return results
         return test_results