Bug 1417051 - Mozharness has to run Marionette tests from tests folder. draft
authorHenrik Skupin <mail@hskupin.info>
Thu, 30 Nov 2017 17:55:03 +0100
changeset 707451 21a93dae1e9041189429a2ed2de5aaa855076166
parent 707450 26afade6d137205e94d19a66cb9428b79f6f5fe1
child 707452 8186f207e081b059c22d3b884a6b00c2b9028745
push id92108
push userbmo:hskupin@gmail.com
push dateTue, 05 Dec 2017 07:58:48 +0000
bugs1417051
milestone59.0a1
Bug 1417051 - Mozharness has to run Marionette tests from tests folder. Updates the mozharness script to run the Marionette command by using the test folder as current working directory. This will make sure that the relative path to the tests is reported. It's identical to the location in the tree. MozReview-Commit-ID: 6hOQnJSqfv0
testing/mozharness/configs/marionette/prod_config.py
testing/mozharness/configs/marionette/test_config.py
testing/mozharness/configs/marionette/windows_config.py
testing/mozharness/configs/marionette/windows_taskcluster_config.py
testing/mozharness/scripts/marionette.py
--- a/testing/mozharness/configs/marionette/prod_config.py
+++ b/testing/mozharness/configs/marionette/prod_config.py
@@ -26,31 +26,31 @@ config = {
         'clobber',
         'read-buildbot-config',
         'download-and-extract',
         'create-virtualenv',
         'install',
         'run-tests',
     ],
     "default_blob_upload_servers": [
-         "https://blobupload.elasticbeanstalk.com",
+        "https://blobupload.elasticbeanstalk.com",
     ],
-    "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
     "download_symbols": "ondemand",
     "download_minidump_stackwalk": True,
     "tooltool_cache": "/builds/worker/tooltool-cache",
     "suite_definitions": {
         "marionette_desktop": {
             "options": [
                 "-vv",
                 "--log-raw=%(raw_log_file)s",
                 "--log-errorsummary=%(error_summary_file)s",
                 "--log-html=%(html_report_file)s",
                 "--binary=%(binary)s",
                 "--address=%(address)s",
                 "--symbols-path=%(symbols_path)s"
             ],
             "run_filename": "",
-            "testsdir": ""
+            "testsdir": "marionette"
         }
     },
     "structured_output": True,
 }
--- a/testing/mozharness/configs/marionette/test_config.py
+++ b/testing/mozharness/configs/marionette/test_config.py
@@ -19,12 +19,12 @@ config = {
                 "--log-raw=%(raw_log_file)s",
                 "--log-errorsummary=%(error_summary_file)s",
                 "--log-html=%(html_report_file)s",
                 "--binary=%(binary)s",
                 "--address=%(address)s",
                 "--symbols-path=%(symbols_path)s"
             ],
             "run_filename": "",
-            "testsdir": ""
+            "testsdir": "marionette"
         },
     },
 }
--- a/testing/mozharness/configs/marionette/windows_config.py
+++ b/testing/mozharness/configs/marionette/windows_config.py
@@ -30,29 +30,29 @@ config = {
         'clobber',
         'read-buildbot-config',
         'download-and-extract',
         'create-virtualenv',
         'install',
         'run-tests',
     ],
     "default_blob_upload_servers": [
-         "https://blobupload.elasticbeanstalk.com",
+        "https://blobupload.elasticbeanstalk.com",
     ],
-    "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
     "download_minidump_stackwalk": True,
     "download_symbols": "ondemand",
     "suite_definitions": {
         "marionette_desktop": {
             "options": [
                 "-vv",
                 "--log-raw=%(raw_log_file)s",
                 "--log-errorsummary=%(error_summary_file)s",
                 "--log-html=%(html_report_file)s",
                 "--binary=%(binary)s",
                 "--address=%(address)s",
                 "--symbols-path=%(symbols_path)s"
             ],
             "run_filename": "",
-            "testsdir": ""
+            "testsdir": "marionette"
         },
     },
 }
--- a/testing/mozharness/configs/marionette/windows_taskcluster_config.py
+++ b/testing/mozharness/configs/marionette/windows_taskcluster_config.py
@@ -29,29 +29,29 @@ config = {
     "default_actions": [
         'clobber',
         'download-and-extract',
         'create-virtualenv',
         'install',
         'run-tests',
     ],
     "default_blob_upload_servers": [
-         "https://blobupload.elasticbeanstalk.com",
+        "https://blobupload.elasticbeanstalk.com",
     ],
-    "blob_uploader_auth_file" : 'C:/builds/oauth.txt',
+    "blob_uploader_auth_file": 'C:/builds/oauth.txt',
     "download_minidump_stackwalk": True,
     "download_symbols": "ondemand",
     "suite_definitions": {
         "marionette_desktop": {
             "options": [
                 "-vv",
                 "--log-raw=%(raw_log_file)s",
                 "--log-errorsummary=%(error_summary_file)s",
                 "--log-html=%(html_report_file)s",
                 "--binary=%(binary)s",
                 "--address=%(address)s",
                 "--symbols-path=%(symbols_path)s"
             ],
             "run_filename": "",
-            "testsdir": ""
+            "testsdir": "marionette"
         },
     },
 }
--- a/testing/mozharness/scripts/marionette.py
+++ b/testing/mozharness/scripts/marionette.py
@@ -142,27 +142,37 @@ class MarionetteTest(TestingMixin, Mercu
         # these are necessary since self.config is read only
         c = self.config
         self.installer_url = c.get('installer_url')
         self.installer_path = c.get('installer_path')
         self.binary_path = c.get('binary_path')
         self.test_url = c.get('test_url')
         self.test_packages_url = c.get('test_packages_url')
 
+        self.test_suite = self._get_test_suite(c.get('emulator'))
+        if self.test_suite not in self.config["suite_definitions"]:
+            self.fatal("{} is not defined in the config!".format(self.test_suite))
+
         if c.get('structured_output'):
             self.parser_class = StructuredOutputParser
         else:
             self.parser_class = TestSummaryOutputParserHelper
 
     def _pre_config_lock(self, rw_config):
         super(MarionetteTest, self)._pre_config_lock(rw_config)
         if not self.config.get('emulator') and not self.config.get('marionette_address'):
                 self.fatal("You need to specify a --marionette-address for non-emulator tests! "
                            "(Try --marionette-address localhost:2828 )")
 
+    def _query_tests_dir(self):
+        dirs = self.query_abs_dirs()
+        test_dir = self.config["suite_definitions"][self.test_suite]["testsdir"]
+
+        return os.path.join(dirs['abs_test_install_dir'], test_dir)
+
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
         abs_dirs = super(MarionetteTest, self).query_abs_dirs()
         dirs = {}
         dirs['abs_test_install_dir'] = os.path.join(
             abs_dirs['abs_work_dir'], 'tests')
         dirs['abs_marionette_dir'] = os.path.join(
@@ -204,17 +214,17 @@ class MarionetteTest(TestingMixin, Mercu
                       'mozcrash', 'mozinstall', 'mozdevice', 'mozprofile',
                       'mozprocess', 'mozrunner'):
                 self.register_virtualenv_module(
                     m, os.path.join(mozbase_dir, m))
 
             self.register_virtualenv_module(
                 'marionette', os.path.join('tests', 'marionette'))
 
-    def _get_options_group(self, is_emulator):
+    def _get_test_suite(self, is_emulator):
         """
         Determine which in tree options group to use and return the
         appropriate key.
         """
         platform = 'emulator' if is_emulator else 'desktop'
         # Currently running marionette on an emulator means webapi
         # tests. This method will need to change if this does.
         testsuite = 'webapi' if is_emulator else 'marionette'
@@ -290,23 +300,18 @@ class MarionetteTest(TestingMixin, Mercu
         if self.config['headless']:
             cmd.append('--headless')
 
         cmd.append('--gecko-log=-')
 
         if self.config.get("structured_output"):
             cmd.append("--log-raw=-")
 
-        options_group = self._get_options_group(self.config.get('emulator'))
-
-        if options_group not in self.config["suite_definitions"]:
-            self.fatal("%s is not defined in the config!" % options_group)
-
-        for s in self.config["suite_definitions"][options_group]["options"]:
-            cmd.append(s % config_fmt_args)
+        for arg in self.config["suite_definitions"][self.test_suite]["options"]:
+            cmd.append(arg % config_fmt_args)
 
         if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
             # Make sure that the logging directory exists
             self.fatal("Could not create blobber upload directory")
 
         cmd.append(manifest)
 
         try_options, try_tests = self.try_args("marionette")
@@ -325,21 +330,28 @@ class MarionetteTest(TestingMixin, Mercu
         if self.config['enable_webrender']:
             env['MOZ_WEBRENDER'] = '1'
             env['MOZ_ACCELERATED'] = '1'
 
         if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
             self.mkdir_p(env['MOZ_UPLOAD_DIR'])
         env = self.query_env(partial_env=env)
 
+        try:
+            cwd = self._query_tests_dir()
+        except Exception as e:
+            self.fatal("Don't know how to run --test-suite '{0}': {1}!".format(
+                self.test_suite, e))
+
         marionette_parser = self.parser_class(config=self.config,
                                               log_obj=self.log_obj,
                                               error_list=BaseErrorList + HarnessErrorList,
                                               strict=False)
         return_code = self.run_command(cmd,
+                                       cwd=cwd,
                                        output_timeout=1000,
                                        output_parser=marionette_parser,
                                        env=env)
         level = INFO
         tbpl_status, log_level = marionette_parser.evaluate_parser(
             return_code=return_code)
         marionette_parser.append_tinderboxprint_line("marionette")