Bug 1272113: Creating awsy task and mozharness script
* * *
[mq]: test_only
MozReview-Commit-ID: AawtpSUIj0P
--- a/taskcluster/ci/test/test-platforms.yml
+++ b/taskcluster/ci/test/test-platforms.yml
@@ -29,16 +29,17 @@ linux32/opt:
- external-media-tests-slow
linux32-nightly/opt:
build-platform: linux-nightly/opt
test-sets:
- linux32-tests
- linux32-opt-tests
- external-media-tests
- external-media-tests-slow
+ - awsy
linux64/debug:
build-platform: linux64/debug
test-sets:
- common-tests
- external-media-tests
- web-platform-tests
linux64/opt:
@@ -55,16 +56,17 @@ linux64-nightly/opt:
build-platform: linux64-nightly/opt
test-sets:
- common-tests
- external-media-tests
- web-platform-tests
- opt-only-tests
- desktop-screenshot-capture
- talos
+ - awsy
# TODO: use 'pgo' and 'asan' labels here, instead of -pgo/opt
linux64-pgo/opt:
build-platform: linux64-pgo/opt
test-sets:
- common-tests
- external-media-tests
- external-media-tests-slow
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -60,16 +60,19 @@ talos:
- talos-g1
- talos-g2
- talos-g3
- talos-g4
- talos-other
- talos-svgr
- talos-tp5o
+awsy:
+ - awsy
+
##
# Limited test sets for specific platforms
stylo-tests:
- cppunit
- crashtest
- reftest-stylo
- mochitest-style
--- a/taskcluster/ci/test/tests.yml
+++ b/taskcluster/ci/test/tests.yml
@@ -1231,16 +1231,34 @@ talos-tp5o:
config:
by-test-platform:
default:
- talos/linux_config.py
- remove_executables.py
extra-options:
- --suite=tp5o
+awsy:
+ description: "Are we slim yet"
+ suite: awsy
+ treeherder-symbol: tc-SY(sy)
+ run-on-projects: ['mozilla-beta', 'mozilla-aurora', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
+ docker-image: {"in-tree": "desktop1604-test"}
+ max-run-time: 7200
+ e10s: true
+ instance-size: xlarge
+ allow-software-gl-layers: false
+ mozharness:
+ script: awsy_script.py
+ config:
+ by-test-platform:
+ default:
+ - awsy/linux_config.py
+ - remove_executables.py
+
web-platform-tests:
description: "Web platform test run"
suite: web-platform-tests
treeherder-symbol: tc-W(wpt)
chunks:
by-test-platform:
macosx64/opt: 5
macosx64/debug: 10
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -358,16 +358,17 @@ GROUP_NAMES = {
'tc-Fxfn-r-e10s': 'Firefox functional tests (remote) executed by TaskCluster with e10s',
'tc-M': 'Mochitests executed by TaskCluster',
'tc-M-e10s': 'Mochitests executed by TaskCluster with e10s',
'tc-M-V': 'Mochitests on Valgrind executed by TaskCluster',
'tc-R': 'Reftests executed by TaskCluster',
'tc-R-e10s': 'Reftests executed by TaskCluster with e10s',
'tc-T': 'Talos performance tests executed by TaskCluster',
'tc-T-e10s': 'Talos performance tests executed by TaskCluster with e10s',
+ 'tc-SY-e10s': 'Are we slim yet tests by TaskCluster with e10s',
'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
'tc-W': 'Web platform tests executed by TaskCluster',
'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
'tc-X': 'Xpcshell tests executed by TaskCluster',
'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
'tc-L10n': 'Localised Repacks executed by Taskcluster',
'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -595,17 +595,17 @@ def set_tag(config, tests):
@transforms.add
def remove_linux_pgo_try_talos(config, tests):
"""linux64-pgo talos tests don't run on try."""
def predicate(test):
return not(
test['test-platform'] == 'linux64-pgo/opt'
- and test['suite'] == 'talos'
+ and (test['suite'] == 'talos' or test['suite'] == 'awsy')
and config.params['project'] == 'try'
)
for test in filter(predicate, tests):
yield test
@transforms.add
def make_job_description(config, tests):
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/awsy/linux_config.py
@@ -0,0 +1,50 @@
+import os
+import platform
+
+PYTHON = "/usr/bin/env python"
+VENV_PATH = '%s/build/venv' % os.getcwd()
+if platform.architecture()[0] == '64bit':
+ TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
+ MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
+else:
+ TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux32/releng.manifest"
+ MINIDUMP_STACKWALK_PATH = "linux32-minidump_stackwalk"
+ABS_WORK_DIR = os.path.join(os.getcwd(), "build")
+BINARY_PATH = os.path.join(ABS_WORK_DIR, "application", "firefox", "firefox-bin")
+INSTALLER_PATH = os.path.join(ABS_WORK_DIR, "installer.tar.bz2")
+
+config = {
+ "log_name": "awsy",
+ "binary_path": BINARY_PATH,
+ "download_tooltool": True,
+ "installer_path": INSTALLER_PATH,
+ "virtualenv_path": VENV_PATH,
+ "find_links": [
+ "http://pypi.pvt.build.mozilla.org/pub",
+ "http://pypi.pub.build.mozilla.org/pub",
+ ],
+ "cmd_timeout": 6500,
+ "pip_index": False,
+ "exes": {
+ 'python': PYTHON,
+ 'virtualenv': [PYTHON, '/tools/misc-python/virtualenv.py'],
+ 'tooltool.py': "/tools/tooltool.py",
+ },
+ "title": os.uname()[1].lower().split('.')[0],
+ "default_actions": [
+ "clobber",
+ "download-and-extract",
+ "populate-webroot",
+ "create-virtualenv",
+ "install",
+ "run-tests",
+ ],
+ "default_blob_upload_servers": [
+ "https://blobupload.elasticbeanstalk.com",
+ ],
+ "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+ "download_minidump_stackwalk": True,
+ "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
+ "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
+ "tooltool_cache": os.path.join(os.getcwd(), "tooltool_cache"),
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/scripts/awsy_script.py
@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+"""
+run awsy tests in a virtualenv
+"""
+
+import os
+import sys
+import copy
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.script import PreScriptAction
+from mozharness.base.log import INFO, ERROR, WARNING, CRITICAL
+from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
+from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
+from mozharness.mozilla.tooltool import TooltoolMixin
+from mozharness.mozilla.structuredlog import StructuredOutputParser
+
+
+class AWSY(TestingMixin, MercurialScript, BlobUploadMixin,TooltoolMixin):
+ config_options = [
+ [["--e10s"],
+ {"action": "store_true",
+ "dest": "e10s",
+ "default": False,
+ "help": "Run tests with multiple processes. (Desktop builds only)",
+ }]
+ ] + testing_config_options + copy.deepcopy(blobupload_config_options)
+
+ def __init__(self, **kwargs):
+
+ kwargs.setdefault('config_options', self.config_options)
+ kwargs.setdefault('all_actions', ['clobber',
+ 'download-and-extract',
+ 'populate-webroot',
+ 'create-virtualenv',
+ 'install',
+ 'run-tests',
+ ])
+ kwargs.setdefault('default_actions', ['clobber',
+ 'download-and-extract',
+ 'populate-webroot',
+ 'create-virtualenv',
+ 'install',
+ 'run-tests',
+ ])
+ kwargs.setdefault('config', {})
+ super(AWSY, self).__init__(**kwargs)
+ self.installer_url = self.config.get("installer_url")
+ self.tests = None
+
+ self.workdir = self.query_abs_dirs()['abs_work_dir']
+ self.testdir = os.path.join(self.workdir, 'tests')
+ self.awsy_path = os.path.join(
+ self.testdir, 'awsy'
+ )
+ self.awsy_libdir = os.path.join(self.awsy_path, 'awsy')
+ self.awsy_testdir = os.path.join(self.awsy_path, 'page_load_test')
+ self.binary_path = self.config.get('binary_path')
+
+ def query_abs_dirs(self):
+ if self.abs_dirs:
+ return self.abs_dirs
+ abs_dirs = super(AWSY, self).query_abs_dirs()
+
+ dirs = {}
+ dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
+ abs_dirs.update(dirs)
+ self.abs_dirs = abs_dirs
+ return self.abs_dirs
+
+ def download_and_extract(self, extract_dirs=None, suite_categories=None):
+ ret = super(AWSY, self).download_and_extract(
+ suite_categories=['common', 'awsy']
+ )
+ return ret
+
+ @PreScriptAction('create-virtualenv')
+ def _pre_create_virtualenv(self, action):
+ requirements_files = [os.path.join(self.testdir,
+ 'config',
+ 'marionette_requirements.txt')]
+
+ for requirements_file in requirements_files:
+ self.register_virtualenv_module(requirements=[requirements_file],
+ two_pass=True)
+
+ self.register_virtualenv_module('awsy', self.awsy_path)
+
+
+ def populate_webroot(self):
+ """Populate the production test slaves' webroots"""
+ self.info("Downloading pageset with tooltool...")
+ manifest_file = os.path.join(self.awsy_path, 'tp5n-pageset.manifest')
+ if not os.path.isdir(self.awsy_testdir):
+ self.mkdir_p(self.awsy_testdir)
+ self.tooltool_fetch(
+ manifest_file,
+ output_dir=self.awsy_testdir,
+ cache=self.config.get('tooltool_cache')
+ )
+ archive = os.path.join(self.awsy_testdir, 'tp5n.zip')
+ unzip = self.query_exe('unzip')
+ unzip_cmd = [unzip, '-q', '-o', archive, '-d', self.awsy_testdir]
+ self.run_command(unzip_cmd, halt_on_failure=True)
+ self.run_command("ls %s" % self.awsy_testdir)
+
+
+ def run_tests(self, args=None, **kw):
+ '''
+ AWSY test should be implemented here
+ '''
+ dirs = self.abs_dirs
+ env = {}
+ error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
+ 'marionette_errorsummary.log')
+
+ cmd = ['marionette']
+ cmd.append("--preferences=%s" % os.path.join(self.awsy_path, "conf", "prefs.json"))
+ cmd.append("--testvars=%s" % os.path.join(self.awsy_path, "conf", "testvars.json"))
+ cmd.append("--log-raw=-")
+ cmd.append("--log-errorsummary=%s" % error_summary_file)
+ cmd.append("--binary=%s" % self.binary_path)
+ cmd.append("--profile=%s" % (os.path.join(dirs['abs_work_dir'], 'profile')))
+ if not self.config['e10s']:
+ cmd.append('--disable-e10s')
+ cmd.append('--gecko-log=%s' % os.path.join(dirs["abs_blob_upload_dir"],
+ 'gecko.log'))
+
+ test_file = os.path.join(self.awsy_libdir, 'test_memory_usage.py')
+ cmd.append(test_file)
+
+ env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir']
+ if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
+ self.mkdir_p(env['MOZ_UPLOAD_DIR'])
+ env = self.query_env(partial_env=env)
+ parser = StructuredOutputParser(config=self.config,
+ log_obj=self.log_obj)
+ return_code = self.run_command(command=cmd,
+ cwd=self.awsy_path,
+ output_timeout=self.config.get("cmd_timeout"),
+ env=env,
+ output_parser=parser)
+
+ level = INFO
+ tbpl_status, log_level = parser.evaluate_parser(
+ return_code=return_code)
+
+ self.log("AWSY exited with return code %s: %s" % (return_code, tbpl_status),
+ level=level)
+ self.buildbot_status(tbpl_status)
+
+
+if __name__ == '__main__':
+ awsy_test = AWSY()
+ awsy_test.run_and_exit()