Bug 1334925 - Add twitch test suite support to FF mozharness script. r?maja_zf draft
authorBryce Van Dyk <bvandyk@mozilla.com>
Fri, 27 Jan 2017 08:10:35 +1300
changeset 467958 d5e233f4837973c099575a8e80420ef3e343b48d
parent 467957 64f89244b01912647153e9d4b6fc4390dbd30b92
child 543814 1f315b7cce1968a02c898ae76d8000cfacdd890c
push id43317
push userbvandyk@mozilla.com
push dateMon, 30 Jan 2017 13:37:50 +0000
reviewersmaja_zf
bugs1334925
milestone54.0a1
Bug 1334925 - Add twitch test suite support to FF mozharness script. r?maja_zf Expand firefox_media_tests.py to include support for twitch.tv tests added to the external media tests. Other changes: - Improve -h docs to mention more clearly available test suites. - Remove duplate test suite config from platform specific configs. The different test suites can all be configured the same, independant of platform, so this has been lifted into the firefox_media_tests.py file to reduce duplication. - Use symbolic constans for test suite names to avoid magic strings and reduce risk of typos. MozReview-Commit-ID: H6Y4wgPUfSk
testing/mozharness/configs/mediatests/buildbot_posix_config.py
testing/mozharness/configs/mediatests/buildbot_windows_config.py
testing/mozharness/configs/mediatests/jenkins_config.py
testing/mozharness/configs/mediatests/taskcluster_posix_config.py
testing/mozharness/configs/mediatests/taskcluster_windows_config.py
testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
--- a/testing/mozharness/configs/mediatests/buildbot_posix_config.py
+++ b/testing/mozharness/configs/mediatests/buildbot_posix_config.py
@@ -31,20 +31,9 @@ config = {
         'run-media-tests',
     ],
     "default_blob_upload_servers": [
          "https://blobupload.elasticbeanstalk.com",
     ],
     "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
     "download_minidump_stackwalk": True,
     "download_symbols": "ondemand",
-
-    "suite_definitions": {
-        "media-tests": {
-            "options": [],
-        },
-        "media-youtube-tests": {
-            "options": [
-                "{test_manifest}"
-            ],
-        },
-    },
 }
--- a/testing/mozharness/configs/mediatests/buildbot_windows_config.py
+++ b/testing/mozharness/configs/mediatests/buildbot_windows_config.py
@@ -37,20 +37,9 @@ config = {
     ],
     "default_blob_upload_servers": [
          "https://blobupload.elasticbeanstalk.com",
     ],
     "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
     "in_tree_config": "config/mozharness/marionette.py",
     "download_minidump_stackwalk": True,
     "download_symbols": "ondemand",
-
-    "suite_definitions": {
-        "media-tests": {
-            "options": [],
-        },
-        "media-youtube-tests": {
-            "options": [
-                "{test_manifest}"
-            ],
-        },
-    },
 }
--- a/testing/mozharness/configs/mediatests/jenkins_config.py
+++ b/testing/mozharness/configs/mediatests/jenkins_config.py
@@ -17,30 +17,16 @@ config = {
     'find_links': ['http://pypi.pub.build.mozilla.org/pub'],
     'pip_index': False,
 
     # mozcrash support
     'download_minidump_stackwalk': True,
     'download_symbols': 'ondemand',
     'download_tooltool': True,
 
-    # Default test suite
-    'test_suite': 'media-tests',
-
-    'suite_definitions': {
-        'media-tests': {
-            'options': [],
-        },
-        'media-youtube-tests': {
-            'options': [
-                '{test_manifest}'
-            ],
-        },
-    },
-
     'default_actions': [
         'clobber',
         'download-and-extract',
         'create-virtualenv',
         'install',
         'run-media-tests',
     ],
 
--- a/testing/mozharness/configs/mediatests/taskcluster_posix_config.py
+++ b/testing/mozharness/configs/mediatests/taskcluster_posix_config.py
@@ -28,20 +28,9 @@ config = {
 
     "default_actions": [
         'clobber',
         'download-and-extract',
         'create-virtualenv',
         'install',
         'run-media-tests',
     ],
-
-    "suite_definitions": {
-        "media-tests": {
-            "options": [],
-        },
-        "media-youtube-tests": {
-            "options": [
-                "{test_manifest}"
-            ],
-        },
-    },
 }
--- a/testing/mozharness/configs/mediatests/taskcluster_windows_config.py
+++ b/testing/mozharness/configs/mediatests/taskcluster_windows_config.py
@@ -31,20 +31,9 @@ config = {
 
     "default_actions": [
         'clobber',
         'download-and-extract',
         'create-virtualenv',
         'install',
         'run-media-tests',
     ],
-
-    "suite_definitions": {
-        "media-tests": {
-            "options": [],
-        },
-        "media-youtube-tests": {
-            "options": [
-                "{test_manifest}"
-            ],
-        },
-    },
 }
--- a/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
+++ b/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
@@ -18,16 +18,20 @@ from mozharness.mozilla.testing.unittest
 from mozharness.mozilla.vcstools import VCSToolsScript
 
 BUSTED = 'busted'
 TESTFAILED = 'testfailed'
 UNKNOWN = 'unknown'
 EXCEPTION = 'exception'
 SUCCESS = 'success'
 
+MEDIA_SUITE = 'media-tests'
+YOUTUBE_SUITE = 'media-youtube-tests'
+TWITCH_SUITE = 'media-twitch-tests'
+
 media_test_config_options = [
     [["--media-urls"],
      {"action": "store",
       "dest": "media_urls",
       "help": "Path to ini file that lists media urls for tests.",
       }],
     [["--profile"],
      {"action": "store",
@@ -54,18 +58,21 @@ media_test_config_options = [
      {"dest": "e10s",
       "action": "store_true",
       "default": False,
       "help": "Enable e10s when running marionette tests."
       }],
     [["--suite"],
      {"action": "store",
       "dest": "test_suite",
-      "default": "media-tests",
-      "help": "suite name",
+      "default": MEDIA_SUITE,
+      "help": ("Name of test suite to run. Possible values: "
+               "{} (default), {}, {}".format(MEDIA_SUITE,
+                                             YOUTUBE_SUITE,
+                                             TWITCH_SUITE)),
       }],
     [['--browsermob-script'],
      {'help': 'path to the browsermob-proxy shell script or batch file',
      }],
     [['--browsermob-port'],
      {'help': 'port to run the browsermob proxy on',
      }],
     [["--allow-software-gl-layers"],
@@ -179,17 +186,17 @@ class FirefoxMediaTestsBase(TestingMixin
         super(FirefoxMediaTestsBase, self).download_and_extract(extract_dirs=extract_dirs)
 
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
         abs_dirs = super(FirefoxMediaTestsBase, self).query_abs_dirs()
         dirs = {
             'abs_test_install_dir' : os.path.join(abs_dirs['abs_work_dir'],
-                                                    'tests')
+                                                  'tests')
         }
         dirs['external-media-tests'] = os.path.join(dirs['abs_test_install_dir'],
                                                     'external-media-tests')
         abs_dirs.update(dirs)
         self.abs_dirs = abs_dirs
         return self.abs_dirs
 
     def _query_cmd(self):
@@ -204,43 +211,59 @@ class FirefoxMediaTestsBase(TestingMixin
         cmd = [
             self.query_python_path(),
             external_media_harness.runtests.__file__
         ]
 
         cmd += ['--binary', self.binary_path]
         if self.symbols_path:
             cmd += ['--symbols-path', self.symbols_path]
-        if self.media_urls:
-            cmd += ['--urls', self.media_urls]
         if self.profile:
             cmd += ['--profile', self.profile]
         if self.tests:
             cmd.append(self.tests)
         if not self.e10s:
             cmd.append('--disable-e10s')
         if self.browsermob_script:
             cmd += ['--browsermob-script', self.browsermob_script]
         if self.browsermob_port:
             cmd += ['--browsermob-port', self.browsermob_port]
 
         test_suite = self.config.get('test_suite')
-        if test_suite not in self.config["suite_definitions"]:
-            self.fatal("{} is not defined in the config!".format(test_suite))
+        if test_suite not in [MEDIA_SUITE,
+                              YOUTUBE_SUITE,
+                              TWITCH_SUITE]:
+            self.fatal("{} is not on of the possibele suites! "
+                       "See the --suites arg for more info"
+                       .format(test_suite))
 
-        test_manifest = None if test_suite != 'media-youtube-tests' else \
-            os.path.join(dirs['external-media-tests'],
-                         'external_media_tests',
-                         'playback', 'youtube', 'manifest.ini')
-        config_fmt_args = {
-            'test_manifest': test_manifest,
-        }
+        if test_suite == YOUTUBE_SUITE:
+            test_manifest = os.path.join(dirs['external-media-tests'],
+                                         'external_media_tests',
+                                         'playback', 'youtube',
+                                         'manifest.ini')
+        elif test_suite == TWITCH_SUITE:
+            test_manifest = os.path.join(dirs['external-media-tests'],
+                                         'external_media_tests',
+                                         'playback', 'twitch',
+                                         'manifest.ini')
+            if not self.media_urls:
+                # The default URLs are for youtube and won't play nice with
+                # the twitch tests, so set URLs to the default twitch ones
+                self.media_urls = os.path.join(dirs['external-media-tests'],
+                                               'external_media_tests', 'urls',
+                                               'twitch', 'default.ini')
+        else:
+            test_manifest = None
 
-        for s in self.config["suite_definitions"][test_suite]["options"]:
-            cmd.append(s.format(**config_fmt_args))
+        if test_manifest:
+            cmd.append(test_manifest)
+
+        if self.media_urls:
+            cmd += ['--urls', self.media_urls]
 
         return cmd
 
     def query_minidump_stackwalk(self):
         """We don't have an extracted test package available to get the manifest file.
 
         So we have to explicitely download the latest version of the manifest from the
         mozilla-central repository and feed it into the query_minidump_stackwalk() method.