Bug 1234500 - Allow mach talos-test to download tp5 pageset; r?jmaher draft
authorRob Wood <rwood@mozilla.com>
Mon, 07 Nov 2016 19:35:12 -0500
changeset 438597 feb8b38f96052a9e81bd7988cf8465c7c848e5eb
parent 434838 060f80b690b8aaa5d927e03578673a3eff3b4c64
child 536959 6213314cd3f78189918561e61f55765903ac99b3
push id35774
push userrwood@mozilla.com
push dateMon, 14 Nov 2016 21:01:11 +0000
reviewersjmaher
bugs1234500
milestone52.0a1
Bug 1234500 - Allow mach talos-test to download tp5 pageset; r?jmaher MozReview-Commit-ID: Jj5H3G2hLg0
.hgignore
testing/mozharness/mozharness/mozilla/testing/talos.py
testing/talos/mach_commands.py
testing/talos/talos.json
testing/talos/tp5n-pageset.manifest
--- a/.hgignore
+++ b/.hgignore
@@ -110,25 +110,27 @@ GPATH
 # Ignore tox generated dir
 .tox/
 
 # Ignore node_modules
 ^tools/lint/eslint/node_modules/
 
 # Ignore talos virtualenv and tp5n files.
 # The tp5n set is supposed to be decompressed at
-# testing/talos/talos/page_load_test/tp5n in order to run tests like tps
+# testing/talos/talos/tests/tp5n in order to run tests like tps
 # locally. Similarly, running talos requires a Python package virtual
 # environment. Both the virtual environment and tp5n files end up littering
 # the status command, so we ignore them.
 ^testing/talos/.Python
 ^testing/talos/bin/
 ^testing/talos/include/
 ^testing/talos/lib/
 ^testing/talos/talos/tests/tp5n.zip
+^testing/talos/talos/tests/tp5n.tar.gz
 ^testing/talos/talos/tests/tp5n
 ^testing/talos/talos/tests/devtools/damp.manifest.develop
+^talos-venv
 
 # Ignore files created when running a reftest.
 ^lextab.py$
 
 # tup database
 ^\.tup
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -20,16 +20,17 @@ from mozharness.base.errors import Pytho
 from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL
 from mozharness.base.log import INFO, WARNING
 from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 from mozharness.base.vcs.vcsbase import MercurialScript
 from mozharness.mozilla.testing.errors import TinderBoxPrintRe
 from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE
 from mozharness.mozilla.buildbot import TBPL_RETRY, TBPL_FAILURE, TBPL_WARNING
+from mozharness.mozilla.tooltool import TooltoolMixin
 
 external_tools_path = os.path.join(
     os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
     'external_tools',
 )
 
 TalosErrorList = PythonErrorList + [
     {'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
@@ -81,17 +82,17 @@ class TalosOutputParser(OutputParser):
         harness_retry_re = TinderBoxPrintRe['harness_error']['retry_regex']
         if harness_retry_re.search(line):
             self.critical(' %s' % line)
             self.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_RETRY)
             return  # skip base parse_single_line
         super(TalosOutputParser, self).parse_single_line(line)
 
 
-class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
+class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin):
     """
     install and run Talos tests:
     https://wiki.mozilla.org/Buildbot/Talos
     """
     config_options = [
         [["--use-talos-json"],
          {"action": "store_true",
           "dest": "use_talos_json",
@@ -159,19 +160,19 @@ class Talos(TestingMixin, MercurialScrip
         self.workdir = self.query_abs_dirs()['abs_work_dir']  # convenience
 
         self.run_local = self.config.get('run_local')
         self.installer_url = self.config.get("installer_url")
         self.talos_json_url = self.config.get("talos_json_url")
         self.talos_json = self.config.get("talos_json")
         self.talos_json_config = self.config.get("talos_json_config")
         self.tests = None
-        self.pagesets_url = None
         self.sps_profile = self.config.get('sps_profile')
         self.sps_profile_interval = self.config.get('sps_profile_interval')
+        self.pagesets_name = None
 
     # We accept some configuration options from the try commit message in the format mozharness: <options>
     # Example try commit message:
     #   mozharness: --spsProfile try: <stuff>
     def query_sps_profile_options(self):
         sps_results = []
         if self.buildbot_config:
             # this is inside automation
@@ -218,25 +219,52 @@ class Talos(TestingMixin, MercurialScrip
         if self.talos_json_config:
             return self.talos_json_config
         if not self.talos_json:
             self.talos_json = os.path.join(self.talos_path, 'talos.json')
         self.talos_json_config = parse_config_file(self.talos_json)
         self.info(pprint.pformat(self.talos_json_config))
         return self.talos_json_config
 
-    def query_pagesets_url(self):
+    def query_pagesets_name(self):
         """Certain suites require external pagesets to be downloaded and
         extracted.
         """
-        if self.pagesets_url:
-            return self.pagesets_url
-        if self.query_talos_json_config() and 'suite' in self.config:
-            self.pagesets_url = self.talos_json_config['suites'][self.config['suite']].get('pagesets_url')
-            return self.pagesets_url
+        if self.pagesets_name:
+            return self.pagesets_name
+        if self.query_talos_json_config() and self.suite is not None:
+            self.pagesets_name = self.talos_json_config['suites'][self.suite].get('pagesets_name')
+            return self.pagesets_name
+
+    def get_suite_from_test(self):
+        """ Retrieve the talos suite name from a given talos test name."""
+        # running locally, single test name provided instead of suite; go through tests and find suite name
+        suite_name = None
+        if self.query_talos_json_config():
+            if '-a' in self.config['talos_extra_options']:
+                test_name_index = self.config['talos_extra_options'].index('-a') + 1
+            if '--activeTests' in self.config['talos_extra_options']:
+                test_name_index = self.config['talos_extra_options'].index('--activeTests') + 1
+            if test_name_index < len(self.config['talos_extra_options']):
+                test_name = self.config['talos_extra_options'][test_name_index]
+                for talos_suite in self.talos_json_config['suites']:
+                    if test_name in self.talos_json_config['suites'][talos_suite].get('tests'):
+                        suite_name = talos_suite
+            if not suite_name:
+                # no suite found to contain the specified test, error out
+                self.fatal("Test name is missing or invalid")
+        else:
+            self.fatal("Talos json config not found, cannot verify suite")
+        return suite_name
+
+    def validate_suite(self):
+        """ Ensure suite name is a valid talos suite. """
+        if self.query_talos_json_config() and self.suite is not None:
+            if not self.suite in self.talos_json_config.get('suites'):
+                self.fatal("Suite '%s' is not valid (not found in talos json config)" % self.suite)
 
     def talos_options(self, args=None, **kw):
         """return options to talos"""
         # binary path
         binary_path = self.binary_path or self.config.get('binary_path')
         if not binary_path:
             self.fatal("Talos requires a path to the binary.  You can specify binary_path or add download-and-extract to your action list.")
 
@@ -269,31 +297,51 @@ class Talos(TestingMixin, MercurialScrip
         if args is not None:
             options += args
         if 'talos_extra_options' in self.config:
             options += self.config['talos_extra_options']
         return options
 
     def populate_webroot(self):
         """Populate the production test slaves' webroots"""
-        c = self.config
-
         self.talos_path = os.path.join(
             self.query_abs_dirs()['abs_work_dir'], 'tests', 'talos'
         )
-        if c.get('run_local'):
-            self.talos_path = os.path.dirname(self.talos_json)
-
-        src_talos_webdir = os.path.join(self.talos_path, 'talos')
 
-        if self.query_pagesets_url():
-            self.info('Downloading pageset...')
-            dirs = self.query_abs_dirs()
-            src_talos_pageset = os.path.join(src_talos_webdir, 'tests')
-            archive = self.download_file(self.pagesets_url, parent_dir=dirs['abs_work_dir'])
+        # need to determine if talos pageset is required to be downloaded
+        if self.config.get('run_local'):
+            # talos initiated locally, get and verify test/suite from cmd line
+            self.talos_path = os.path.dirname(self.talos_json)
+            if '-a' in self.config['talos_extra_options'] or '--activeTests' in self.config['talos_extra_options']:
+                # test name (-a or --activeTests) specified, find out what suite it is a part of
+                self.suite = self.get_suite_from_test()
+            elif '--suite' in self.config['talos_extra_options']:
+                # --suite specified, get suite from cmd line and ensure is valid
+                suite_name_index = self.config['talos_extra_options'].index('--suite') + 1
+                if suite_name_index < len(self.config['talos_extra_options']):
+                    self.suite = self.config['talos_extra_options'][suite_name_index]
+                    self.validate_suite()
+                else:
+                    self.fatal("Suite name not provided")
+        else:
+            # talos initiated in production via mozharness
+            self.suite = self.config['suite']
+
+        # now that have the suite name, check if pageset is required, if so download it
+        if self.query_pagesets_name():
+            self.info("Downloading pageset with tooltool...")
+            self.src_talos_webdir = os.path.join(self.talos_path, 'talos')
+            src_talos_pageset = os.path.join(self.src_talos_webdir, 'tests')
+            manifest_file = os.path.join(self.talos_path, 'tp5n-pageset.manifest')
+            self.tooltool_fetch(
+                manifest_file,
+                output_dir=src_talos_pageset,
+                cache=self.config.get('tooltool_cache')
+            )
+            archive = os.path.join(src_talos_pageset, self.pagesets_name)
             unzip = self.query_exe('unzip')
             unzip_cmd = [unzip, '-q', '-o', archive, '-d', src_talos_pageset]
             self.run_command(unzip_cmd, halt_on_failure=True)
 
     # Action methods. {{{1
     # clobber defined in BaseScript
     # read_buildbot_config defined in BuildbotMixin
 
--- a/testing/talos/mach_commands.py
+++ b/testing/talos/mach_commands.py
@@ -61,16 +61,17 @@ class TalosRunner(MozbuildObject):
                 'virtualenv': [self.python_interp, self.virtualenv_script]
             },
             'title': socket.gethostname(),
             'default_actions': [
                 'populate-webroot',
                 'create-virtualenv',
                 'run-tests',
             ],
+            'download_tooltool': True,
             'talos_extra_options': ['--develop'] + self.talos_args,
         }
 
     def make_args(self):
         self.args = {
             'config': {},
             'initial_config_file': self.config_file_path,
         }
--- a/testing/talos/talos.json
+++ b/testing/talos/talos.json
@@ -40,30 +40,30 @@
             "talos_options": ["--disable-e10s"]
         },
         "other-e10s_l64": {
             "tests": ["a11yr", "ts_paint", "tpaint", "sessionrestore", "sessionrestore_no_auto_restore", "tabpaint"]
         },
         "g1": {
             "tests": ["tp5o_scroll", "glterrain"],
             "talos_options": ["--disable-e10s"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+            "pagesets_name": "tp5n.zip"
         },
         "g1-e10s": {
             "tests": ["tp5o_scroll", "glterrain"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+            "pagesets_name": "tp5n.zip"
         },
         "g2": {
             "tests": ["damp", "tps"],
             "talos_options": ["--disable-e10s"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+            "pagesets_name": "tp5n.zip"
         },
         "g2-e10s": {
             "tests": ["damp", "tps"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+            "pagesets_name": "tp5n.zip"
         },
         "g3": {
             "tests": ["dromaeo_dom"],
             "talos_options": ["--disable-e10s"]
         },
         "g3-e10s": {
             "tests": ["dromaeo_dom"]
         },
@@ -78,35 +78,35 @@
             "tests": ["tsvgx", "tsvgr_opacity", "tart", "tscrollx", "cart"],
             "talos_options": ["--disable-e10s"]
         },
         "svgr-e10s": {
             "tests": ["tsvgx", "tsvgr_opacity", "tart", "tscrollx", "cart"]
         },
         "tp5o": {
             "tests": ["tp5o"],
-            "talos_options": ["--disable-e10s"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+            "pagesets_name": "tp5n.zip",
+            "talos_options": ["--disable-e10s"]
         },
         "tp5o-e10s": {
             "tests": ["tp5o"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+            "pagesets_name": "tp5n.zip"
         },
         "xperf": {
             "tests": ["tp5n"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
+            "pagesets_name": "tp5n.zip",
             "talos_options": [
                 "--disable-e10s",
                 "--xperf_path",
                 "\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
             ]
         },
         "xperf-e10s": {
             "tests": ["tp5n"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
+            "pagesets_name": "tp5n.zip",
             "talos_options": [
                 "--xperf_path",
                 "\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
             ]
         }
     },
     "mobile-suites": {
         "remote-tsvgx": {
new file mode 100644
--- /dev/null
+++ b/testing/talos/tp5n-pageset.manifest
@@ -0,0 +1,9 @@
+[
+    {
+        "filename": "tp5n.zip",
+        "size": 81753769,
+        "digest": "7e74bc532d220fa2484f84bd7c2659da7d2ae3aa0bc225ba63e3db70dc0c0697503427209098afa85e235397c4ec58cd488cab7b3435e8079583d3994fff8326",
+        "algorithm": "sha512",
+        "unpack": false
+    }
+]
\ No newline at end of file