Bug 1334925 - Add twitch test suite support to FF mozharness script. r?maja_zf
Expand firefox_media_tests.py to include support for twitch.tv tests added to
the external media tests.
Other changes:
- Improve -h docs to mention more clearly available test suites.
- Remove duplate test suite config from platform specific configs. The different
test suites can all be configured the same, independant of platform, so this
has been lifted into the firefox_media_tests.py file to reduce duplication.
- Use symbolic constans for test suite names to avoid magic strings and reduce
risk of typos.
MozReview-Commit-ID: H6Y4wgPUfSk
--- a/testing/mozharness/configs/mediatests/buildbot_posix_config.py
+++ b/testing/mozharness/configs/mediatests/buildbot_posix_config.py
@@ -31,20 +31,9 @@ config = {
'run-media-tests',
],
"default_blob_upload_servers": [
"https://blobupload.elasticbeanstalk.com",
],
"blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
"download_minidump_stackwalk": True,
"download_symbols": "ondemand",
-
- "suite_definitions": {
- "media-tests": {
- "options": [],
- },
- "media-youtube-tests": {
- "options": [
- "{test_manifest}"
- ],
- },
- },
}
--- a/testing/mozharness/configs/mediatests/buildbot_windows_config.py
+++ b/testing/mozharness/configs/mediatests/buildbot_windows_config.py
@@ -37,20 +37,9 @@ config = {
],
"default_blob_upload_servers": [
"https://blobupload.elasticbeanstalk.com",
],
"blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
"in_tree_config": "config/mozharness/marionette.py",
"download_minidump_stackwalk": True,
"download_symbols": "ondemand",
-
- "suite_definitions": {
- "media-tests": {
- "options": [],
- },
- "media-youtube-tests": {
- "options": [
- "{test_manifest}"
- ],
- },
- },
}
--- a/testing/mozharness/configs/mediatests/jenkins_config.py
+++ b/testing/mozharness/configs/mediatests/jenkins_config.py
@@ -17,30 +17,16 @@ config = {
'find_links': ['http://pypi.pub.build.mozilla.org/pub'],
'pip_index': False,
# mozcrash support
'download_minidump_stackwalk': True,
'download_symbols': 'ondemand',
'download_tooltool': True,
- # Default test suite
- 'test_suite': 'media-tests',
-
- 'suite_definitions': {
- 'media-tests': {
- 'options': [],
- },
- 'media-youtube-tests': {
- 'options': [
- '{test_manifest}'
- ],
- },
- },
-
'default_actions': [
'clobber',
'download-and-extract',
'create-virtualenv',
'install',
'run-media-tests',
],
--- a/testing/mozharness/configs/mediatests/taskcluster_posix_config.py
+++ b/testing/mozharness/configs/mediatests/taskcluster_posix_config.py
@@ -28,20 +28,9 @@ config = {
"default_actions": [
'clobber',
'download-and-extract',
'create-virtualenv',
'install',
'run-media-tests',
],
-
- "suite_definitions": {
- "media-tests": {
- "options": [],
- },
- "media-youtube-tests": {
- "options": [
- "{test_manifest}"
- ],
- },
- },
}
--- a/testing/mozharness/configs/mediatests/taskcluster_windows_config.py
+++ b/testing/mozharness/configs/mediatests/taskcluster_windows_config.py
@@ -31,20 +31,9 @@ config = {
"default_actions": [
'clobber',
'download-and-extract',
'create-virtualenv',
'install',
'run-media-tests',
],
-
- "suite_definitions": {
- "media-tests": {
- "options": [],
- },
- "media-youtube-tests": {
- "options": [
- "{test_manifest}"
- ],
- },
- },
}
--- a/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
+++ b/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
@@ -18,16 +18,20 @@ from mozharness.mozilla.testing.unittest
from mozharness.mozilla.vcstools import VCSToolsScript
BUSTED = 'busted'
TESTFAILED = 'testfailed'
UNKNOWN = 'unknown'
EXCEPTION = 'exception'
SUCCESS = 'success'
+MEDIA_SUITE = 'media-tests'
+YOUTUBE_SUITE = 'media-youtube-tests'
+TWITCH_SUITE = 'media-twitch-tests'
+
media_test_config_options = [
[["--media-urls"],
{"action": "store",
"dest": "media_urls",
"help": "Path to ini file that lists media urls for tests.",
}],
[["--profile"],
{"action": "store",
@@ -54,18 +58,21 @@ media_test_config_options = [
{"dest": "e10s",
"action": "store_true",
"default": False,
"help": "Enable e10s when running marionette tests."
}],
[["--suite"],
{"action": "store",
"dest": "test_suite",
- "default": "media-tests",
- "help": "suite name",
+ "default": MEDIA_SUITE,
+ "help": ("Name of test suite to run. Possible values: "
+ "{} (default), {}, {}".format(MEDIA_SUITE,
+ YOUTUBE_SUITE,
+ TWITCH_SUITE)),
}],
[['--browsermob-script'],
{'help': 'path to the browsermob-proxy shell script or batch file',
}],
[['--browsermob-port'],
{'help': 'port to run the browsermob proxy on',
}],
[["--allow-software-gl-layers"],
@@ -179,17 +186,17 @@ class FirefoxMediaTestsBase(TestingMixin
super(FirefoxMediaTestsBase, self).download_and_extract(extract_dirs=extract_dirs)
def query_abs_dirs(self):
if self.abs_dirs:
return self.abs_dirs
abs_dirs = super(FirefoxMediaTestsBase, self).query_abs_dirs()
dirs = {
'abs_test_install_dir' : os.path.join(abs_dirs['abs_work_dir'],
- 'tests')
+ 'tests')
}
dirs['external-media-tests'] = os.path.join(dirs['abs_test_install_dir'],
'external-media-tests')
abs_dirs.update(dirs)
self.abs_dirs = abs_dirs
return self.abs_dirs
def _query_cmd(self):
@@ -204,43 +211,59 @@ class FirefoxMediaTestsBase(TestingMixin
cmd = [
self.query_python_path(),
external_media_harness.runtests.__file__
]
cmd += ['--binary', self.binary_path]
if self.symbols_path:
cmd += ['--symbols-path', self.symbols_path]
- if self.media_urls:
- cmd += ['--urls', self.media_urls]
if self.profile:
cmd += ['--profile', self.profile]
if self.tests:
cmd.append(self.tests)
if not self.e10s:
cmd.append('--disable-e10s')
if self.browsermob_script:
cmd += ['--browsermob-script', self.browsermob_script]
if self.browsermob_port:
cmd += ['--browsermob-port', self.browsermob_port]
test_suite = self.config.get('test_suite')
- if test_suite not in self.config["suite_definitions"]:
- self.fatal("{} is not defined in the config!".format(test_suite))
+ if test_suite not in [MEDIA_SUITE,
+ YOUTUBE_SUITE,
+ TWITCH_SUITE]:
+ self.fatal("{} is not on of the possibele suites! "
+ "See the --suites arg for more info"
+ .format(test_suite))
- test_manifest = None if test_suite != 'media-youtube-tests' else \
- os.path.join(dirs['external-media-tests'],
- 'external_media_tests',
- 'playback', 'youtube', 'manifest.ini')
- config_fmt_args = {
- 'test_manifest': test_manifest,
- }
+ if test_suite == YOUTUBE_SUITE:
+ test_manifest = os.path.join(dirs['external-media-tests'],
+ 'external_media_tests',
+ 'playback', 'youtube',
+ 'manifest.ini')
+ elif test_suite == TWITCH_SUITE:
+ test_manifest = os.path.join(dirs['external-media-tests'],
+ 'external_media_tests',
+ 'playback', 'twitch',
+ 'manifest.ini')
+ if not self.media_urls:
+ # The default URLs are for youtube and won't play nice with
+ # the twitch tests, so set URLs to the default twitch ones
+ self.media_urls = os.path.join(dirs['external-media-tests'],
+ 'external_media_tests', 'urls',
+ 'twitch', 'default.ini')
+ else:
+ test_manifest = None
- for s in self.config["suite_definitions"][test_suite]["options"]:
- cmd.append(s.format(**config_fmt_args))
+ if test_manifest:
+ cmd.append(test_manifest)
+
+ if self.media_urls:
+ cmd += ['--urls', self.media_urls]
return cmd
def query_minidump_stackwalk(self):
"""We don't have an extracted test package available to get the manifest file.
So we have to explicitely download the latest version of the manifest from the
mozilla-central repository and feed it into the query_minidump_stackwalk() method.