Bug 1468865 - Port raptor firefox tp6 to win; r=jmaher draft
authorRob Wood <rwood@mozilla.com>
Tue, 26 Jun 2018 17:56:20 -0400
changeset 812142 bffe3cfdd3384ebbff41c01347dde03f656935a7
parent 812139 bad873c0dbbe1c34582ba3ceefd58c7be7eecd8e
push id114455
push userrwood@mozilla.com
push dateThu, 28 Jun 2018 15:21:08 +0000
reviewersjmaher
bugs1468865
milestone63.0a1
Bug 1468865 - Port raptor firefox tp6 to win; r=jmaher MozReview-Commit-ID: 78VGzPkhkww
taskcluster/ci/test/raptor.yml
taskcluster/ci/test/test-platforms.yml
testing/mozharness/mozharness/mozilla/testing/raptor.py
testing/raptor/raptor/manifest.py
testing/raptor/raptor/playback/mitmproxy.py
testing/raptor/raptor/playback/mitmproxy_requirements.txt
testing/raptor/raptor/playback/python3.manifest
testing/raptor/raptor/playback/python3_x64.manifest
testing/raptor/raptor/raptor.py
testing/raptor/raptor/tests/raptor-tp6.ini
testing/raptor/raptor/utils.py
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -24,73 +24,57 @@ job-defaults:
                     - raptor/linux64_config_taskcluster.py
                 default:
                     - raptor/linux_config.py
 
 raptor-firefox-tp6:
     description: "Raptor Firefox tp6"
     try-name: raptor-firefox-tp6
     treeherder-symbol: Rap(tp6)
-    run-on-projects:
-        by-test-platform:
-            macosx.*: ['try', 'mozilla-central']
-            linux64.*: ['try', 'mozilla-central']
-            default: ['try']
+    run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1200
     mozharness:
         extra-options:
             - --test=raptor-tp6
 
 raptor-firefox-speedometer:
     description: "Raptor Firefox speedometer"
     try-name: raptor-firefox-speedometer
     treeherder-symbol: Rap(sp)
-    run-on-projects:
-        by-test-platform:
-            macosx.*: ['try', 'mozilla-central']
-            default: ['try', 'mozilla-central']
+    run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1500
     mozharness:
         extra-options:
             - --test=raptor-speedometer
 
 raptor-firefox-stylebench:
     description: "Raptor Firefox StyleBench"
     try-name: raptor-firefox-stylebench
     treeherder-symbol: Rap(sb)
-    run-on-projects:
-        by-test-platform:
-            macosx.*: ['try', 'mozilla-central']
-            default: ['try', 'mozilla-central']
+    run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1800
     mozharness:
         extra-options:
             - --test=raptor-stylebench
 
 raptor-firefox-motionmark-htmlsuite:
     description: "Raptor Firefox MotionMark HtmlSuite"
     try-name: raptor-firefox-motionmark-htmlsuite
     treeherder-symbol: Rap(mm-h)
-    run-on-projects:
-        by-test-platform:
-            macosx.*: ['try', 'mozilla-central']
-            default: ['try', 'mozilla-central']
+    run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1800
     mozharness:
         extra-options:
             - --test=raptor-motionmark-htmlsuite
 
 raptor-firefox-motionmark-animometer:
     description: "Raptor Firefox MotionMark Animometer"
     try-name: raptor-firefox-motionmark-animometer
     treeherder-symbol: Rap(mm-a)
-    run-on-projects:
-        by-test-platform:
-            macosx.*: ['try', 'mozilla-central']
-            default: ['try', 'mozilla-central']
+    run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1800
     mozharness:
         extra-options:
             - --test=raptor-motionmark-animometer
 
 raptor-chrome-tp6:
     description: "Raptor Chrome tp6"
     try-name: raptor-chrome-tp6
@@ -104,59 +88,47 @@ raptor-chrome-tp6:
         extra-options:
             - --test=raptor-tp6
             - --app=chrome
 
 raptor-chrome-speedometer:
     description: "Raptor Chrome speedometer"
     try-name: raptor-chrome-speedometer
     treeherder-symbol: Rap-C(sp)
-    run-on-projects:
-        by-test-platform:
-            macosx.*: ['try', 'mozilla-central']
-            default: ['try']
+    run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1500
     mozharness:
         extra-options:
             - --test=raptor-speedometer
             - --app=chrome
 
 raptor-chrome-stylebench:
     description: "Raptor Chrome StyleBench"
     try-name: raptor-chrome-stylebench
     treeherder-symbol: Rap-C(sb)
-    run-on-projects:
-        by-test-platform:
-            macosx.*: ['try', 'mozilla-central']
-            default: ['try']
+    run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1800
     mozharness:
         extra-options:
             - --test=raptor-stylebench
             - --app=chrome
 
 raptor-chrome-motionmark-htmlsuite:
     description: "Raptor Chrome MotionMark HtmlSuite"
     try-name: raptor-chrome-motionmark-htmlsuite
     treeherder-symbol: Rap-C(mm-h)
-    run-on-projects:
-        by-test-platform:
-            macosx.*: ['try', 'mozilla-central']
-            default: ['try']
+    run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1800
     mozharness:
         extra-options:
             - --test=raptor-motionmark-htmlsuite
             - --app=chrome
 
 raptor-chrome-motionmark-animometer:
     description: "Raptor Chrome MotionMark Animometer"
     try-name: raptor-chrome-motionmark-animometer
     treeherder-symbol: Rap-C(mm-a)
-    run-on-projects:
-        by-test-platform:
-            macosx.*: ['try', 'mozilla-central']
-            default: ['try']
+    run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1800
     mozharness:
         extra-options:
             - --test=raptor-motionmark-animometer
             - --app=chrome
--- a/taskcluster/ci/test/test-platforms.yml
+++ b/taskcluster/ci/test/test-platforms.yml
@@ -130,25 +130,27 @@ windows7-32/debug:
 windows7-32/opt:
     build-platform: win32/opt
     test-sets:
         - awsy
         - desktop-screenshot-capture
         - windows-reftest-gpu
         - windows-talos
         - windows-tests
+        - raptor
 
 windows7-32-pgo/opt:
     build-platform: win32-pgo/opt
     test-sets:
         - awsy
         - desktop-screenshot-capture
         - windows-reftest-gpu
         - windows-talos
         - windows-tests
+        - raptor
 
 windows7-32-nightly/opt:
     build-platform: win32-nightly/opt
     test-sets:
         - awsy
         - desktop-screenshot-capture
         - windows-reftest-gpu
         - windows-tests
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -13,17 +13,16 @@ import subprocess
 import time
 
 from shutil import copyfile
 
 import mozharness
 
 from mozharness.base.errors import PythonErrorList
 from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, INFO
-from mozharness.base.python import Python3Virtualenv
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 from mozharness.base.vcs.vcsbase import MercurialScript
 from mozharness.mozilla.testing.codecoverage import (
     CodeCoverageMixin,
     code_coverage_config_options
 )
 
 scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
@@ -39,17 +38,17 @@ RaptorErrorList = PythonErrorList + [
     {'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
     {'substr': r"""No such file or directory: 'browser_output.txt'""",
      'level': CRITICAL,
      'explanation': "Most likely the browser failed to launch, or the test was otherwise "
      "unsuccessful in even starting."},
 ]
 
 
-class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin):
+class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin):
     """
     install and run raptor tests
     """
     config_options = [
         [["--test"],
          {"action": "store",
           "dest": "test",
           "help": "Raptor test to run"
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -15,17 +15,17 @@ raptor_ini = os.path.join(here, 'raptor.
 tests_dir = os.path.join(here, 'tests')
 LOG = get_proxy_logger(component="raptor-manifest")
 
 required_settings = ['apps', 'type', 'page_cycles', 'test_url', 'measure',
                      'unit', 'lower_is_better', 'alert_threshold']
 
 playback_settings = ['playback_binary_manifest', 'playback_binary_zip_mac',
                      'playback_pageset_manifest', 'playback_pageset_zip_mac',
-                     'playback_recordings']
+                     'playback_recordings', 'python3_win_manifest']
 
 
 def filter_app(tests, values):
     for test in tests:
         if values["app"] in test['apps']:
             yield test
 
 
--- a/testing/raptor/raptor/playback/mitmproxy.py
+++ b/testing/raptor/raptor/playback/mitmproxy.py
@@ -17,18 +17,29 @@ from mozlog import get_proxy_logger
 from mozprocess import ProcessHandler
 
 from .base import Playback
 
 here = os.path.dirname(os.path.realpath(__file__))
 LOG = get_proxy_logger(component='raptor-mitmproxy')
 
 # needed so unit tests can find their imports
-mozharness_dir = os.path.join(here, '../../../mozharness')
+if os.environ.get('SCRIPTSPATH', None) is not None:
+    # in production it is env SCRIPTS_PATH
+    mozharness_dir = os.environ['SCRIPTSPATH']
+else:
+    # locally it's in source tree
+    mozharness_dir = os.path.join(here, '../../../mozharness')
 sys.path.insert(0, mozharness_dir)
+
+# required for using a python3 virtualenv on win for mitmproxy
+from mozharness.base.python import Python3Virtualenv
+from mozharness.mozilla.testing.testbase import TestingMixin
+from mozharness.base.vcs.vcsbase import MercurialScript
+
 raptor_dir = os.path.join(here, '..')
 sys.path.insert(0, raptor_dir)
 
 from utils import transform_platform
 
 external_tools_path = os.environ.get('EXTERNALTOOLSPATH', None)
 
 if external_tools_path is not None:
@@ -64,21 +75,22 @@ certdb2.addCertFromBase64(cert, "C,C,C",
 pref("network.proxy.type", 1);
 pref("network.proxy.http", "127.0.0.1");
 pref("network.proxy.http_port", 8080);
 pref("network.proxy.ssl", "127.0.0.1");
 pref("network.proxy.ssl_port", 8080);
 '''
 
 
-class Mitmproxy(Playback):
+class Mitmproxy(Playback, Python3Virtualenv, TestingMixin, MercurialScript):
 
     def __init__(self, config):
         self.config = config
         self.mitmproxy_proc = None
+        self.mitmdump_path = None
         self.recordings = config.get('playback_recordings', None)
         self.browser_path = config.get('binary', None)
 
         # raptor_dir is where we will download all mitmproxy required files
         # when running locally it comes from obj_path via mozharness/mach
         if self.config.get("obj_path", None) is not None:
             self.raptor_dir = self.config.get("obj_path")
         else:
@@ -89,16 +101,21 @@ class Mitmproxy(Playback):
 
         # add raptor to raptor_dir
         self.raptor_dir = os.path.join(self.raptor_dir, "testing", "raptor")
         self.recordings_path = self.raptor_dir
         LOG.info("raptor_dir used for mitmproxy downloads and exe files: %s" % self.raptor_dir)
 
         # go ahead and download and setup mitmproxy
         self.download()
+
+        # on windows we must use a python3 virtualen for mitmproxy
+        if 'win' in self.config['platform']:
+            self.setup_py3_virtualenv()
+
         # mitmproxy must be started before setup, so that the CA cert is available
         self.start()
         self.setup()
 
     def _tooltool_fetch(self, manifest):
         def outputHandler(line):
             LOG.info(line)
         command = [sys.executable, TOOLTOOL_PATH, 'fetch', '-o', '-m', manifest]
@@ -115,41 +132,86 @@ class Mitmproxy(Playback):
             if proc.poll() is None:
                 proc.kill(signal.SIGTERM)
 
     def download(self):
         # download mitmproxy binary and pageset using tooltool
         # note: tooltool automatically unpacks the files as well
         if not os.path.exists(self.raptor_dir):
             os.makedirs(self.raptor_dir)
-        LOG.info("downloading mitmproxy binary")
-        _manifest = os.path.join(here, self.config['playback_binary_manifest'])
-        transformed_manifest = transform_platform(_manifest, self.config['platform'])
-        self._tooltool_fetch(transformed_manifest)
+
+        if 'win' in self.config['platform']:
+            # on windows we need a python3 environment and use our own package from tooltool
+            self.py3_path = self.fetch_python3()
+            LOG.info("python3 path is: %s" % self.py3_path)
+        else:
+            # on osx and linux we use pre-built binaries
+            LOG.info("downloading mitmproxy binary")
+            _manifest = os.path.join(here, self.config['playback_binary_manifest'])
+            transformed_manifest = transform_platform(_manifest, self.config['platform'])
+            self._tooltool_fetch(transformed_manifest)
+
+        # we use one pageset for all platforms (pageset was recorded on win10)
         LOG.info("downloading mitmproxy pageset")
         _manifest = os.path.join(here, self.config['playback_pageset_manifest'])
         transformed_manifest = transform_platform(_manifest, self.config['platform'])
         self._tooltool_fetch(transformed_manifest)
         return
 
+    def fetch_python3(self):
+        """Mitmproxy on windows needs Python 3.x"""
+        python3_path = os.path.join(self.raptor_dir, 'python3.6', 'python')
+        if not os.path.exists(os.path.dirname(python3_path)):
+            _manifest = os.path.join(here, self.config['python3_win_manifest'])
+            transformed_manifest = transform_platform(_manifest, self.config['platform'],
+                                                      self.config['processor'])
+            LOG.info("downloading py3 package for mitmproxy windows: %s" % transformed_manifest)
+            self._tooltool_fetch(transformed_manifest)
+        cmd = [python3_path, '--version']
+        # just want python3 ver printed in production log
+        subprocess.Popen(cmd, env=os.environ.copy())
+        return python3_path
+
+    def setup_py3_virtualenv(self):
+        """Mitmproxy on windows needs Python 3.x; set up a separate py 3.x env here"""
+        LOG.info("Setting up python 3.x virtualenv, required for mitmproxy on windows")
+        # these next two are required for py3_venv_configuration
+        self.abs_dirs = {'base_work_dir': mozharness_dir}
+        self.log_obj = None
+        # now create the py3 venv
+        venv_path = os.path.join(self.raptor_dir, 'py3venv')
+        self.py3_venv_configuration(python_path=self.py3_path, venv_path=venv_path)
+        self.py3_create_venv()
+        self.py3_install_modules(["cffi==1.10.0"])
+        requirements = [os.path.join(here, "mitmproxy_requirements.txt")]
+        self.py3_install_requirement_files(requirements)
+        # add py3 executables path to system path
+        sys.path.insert(1, self.py3_path_to_executables())
+        # install mitmproxy itself
+        self.py3_install_modules(modules=['mitmproxy'])
+        self.mitmdump_path = os.path.join(self.py3_path_to_executables(), 'mitmdump')
+
     def setup(self):
         # install the generated CA certificate into Firefox
         # mitmproxy cert setup needs path to mozharness install; mozharness has set this
         # value in the SCRIPTSPATH env var for us in mozharness/mozilla/testing/talos.py
         scripts_path = os.environ.get('SCRIPTSPATH')
         LOG.info('scripts_path: %s' % str(scripts_path))
         self.install_mitmproxy_cert(self.mitmproxy_proc,
                                     self.browser_path,
                                     str(scripts_path))
         return
 
     def start(self):
-        mitmdump_path = os.path.join(self.raptor_dir, 'mitmdump')
+        # if on windows, the mitmdump_path was already set when creating py3 env
+        if self.mitmdump_path is None:
+            self.mitmdump_path = os.path.join(self.raptor_dir, 'mitmdump')
+
         recordings_list = self.recordings.split()
-        self.mitmproxy_proc = self.start_mitmproxy_playback(mitmdump_path,
+        self.mitmproxy_proc = self.start_mitmproxy_playback(self.mitmdump_path,
                                                             self.recordings_path,
                                                             recordings_list,
                                                             self.browser_path)
         return
 
     def stop(self):
         self.stop_mitmproxy_playback()
         return
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy_requirements.txt
@@ -0,0 +1,35 @@
+argh==0.26.2
+asn1crypto==0.22.0
+blinker==1.4
+pycparser==2.17
+cffi==1.10.0
+brotlipy==0.6.0
+certifi==2017.4.17
+click==6.7
+construct==2.8.12
+cryptography==1.8.2
+cssutils==1.0.2
+EditorConfig==0.12.1
+h2==2.6.2
+hpack==3.0.0
+html2text==2016.9.19
+hyperframe==4.0.2
+idna==2.5
+jsbeautifier==1.6.12
+kaitaistruct==0.6
+mitmproxy==2.0.2
+packaging==16.8
+passlib==1.7.1
+pathtools==0.1.2
+pyasn1==0.2.3
+pyOpenSSL==16.2.0
+pyparsing==2.2.0
+pyperclip==1.5.27
+PyYAML==3.12
+requests==2.13.0
+ruamel.yaml==0.13.14
+six==1.10.0
+sortedcontainers==1.5.7
+tornado==4.4.3
+urwid==1.3.1
+watchdog==0.8.3
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/python3.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 15380470,
+    "visibility": "public",
+    "digest": "cd78b88d95b69bef99d7192b71dd34118700f44db0a0069a13bfd4943b131e8d7fdac83859f8ac15d873d4b329eef69d8d75d0a6746d06fdcfc5d06da0c9784c",
+    "algorithm": "sha512",
+    "unpack": true,
+    "filename": "python3.6.zip"
+  }
+]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/python3_x64.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 16026760,
+    "visibility": "public",
+    "digest": "379428e3955671213a245ccd9ccf6f9d17d368db68c02da8baed7be629f2691127cd3e3f86807b25e2098d9840083fdc07946ab1bed0c14db4a5b628a47ed9ef",
+    "algorithm": "sha512",
+    "unpack": true,
+    "filename": "python3.6.amd64.zip"
+  }
+]
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -39,16 +39,17 @@ from results import RaptorResultsHandler
 class Raptor(object):
     """Container class for Raptor"""
 
     def __init__(self, app, binary, run_local=False, obj_path=None):
         self.config = {}
         self.config['app'] = app
         self.config['binary'] = binary
         self.config['platform'] = mozinfo.os
+        self.config['processor'] = mozinfo.processor
         self.config['run_local'] = run_local
         self.config['obj_path'] = obj_path
         self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
         self.log = get_default_logger(component='raptor-main')
         self.control_server = None
         self.playback = None
         self.benchmark = None
 
@@ -95,16 +96,17 @@ class Raptor(object):
         self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
         self.config['playback_binary_manifest'] = test.get('playback_binary_manifest', None)
         _key = 'playback_binary_zip_%s' % self.config['platform']
         self.config['playback_binary_zip'] = test.get(_key, None)
         self.config['playback_pageset_manifest'] = test.get('playback_pageset_manifest', None)
         _key = 'playback_pageset_zip_%s' % self.config['platform']
         self.config['playback_pageset_zip'] = test.get(_key, None)
         self.config['playback_recordings'] = test.get('playback_recordings', None)
+        self.config['python3_win_manifest'] = test.get('python3_win_manifest', None)
 
     def run_test(self, test, timeout=None):
         self.log.info("starting raptor test: %s" % test['name'])
         self.log.info("test settings: %s" % str(test))
         self.log.info("raptor config: %s" % str(self.config))
 
         # benchmark-type tests require the benchmark test to be served out
         if test.get('type') == "benchmark":
@@ -135,16 +137,17 @@ class Raptor(object):
         self.runner.start()
 
         proc = self.runner.process_handler
         self.output_handler.proc = proc
         self.control_server.browser_proc = proc
 
         try:
             self.runner.wait(timeout)
+
         finally:
             try:
                 self.runner.check_for_crashes()
             except NotImplementedError:  # not implemented for Chrome
                 pass
 
         if self.playback is not None:
             self.playback.stop()
--- a/testing/raptor/raptor/tests/raptor-tp6.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6.ini
@@ -4,16 +4,17 @@
 
 # raptor tp6
 
 [DEFAULT]
 type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
 playback_binary_zip_mac = mitmproxy-2.0.2-{platform}.tar.gz
+python3_win_manifest = python3{x64}.manifest
 playback_pageset_manifest = mitmproxy-playback-set.manifest
 playback_pageset_zip_mac = mitmproxy-recording-set-win10.zip
 page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 
 [raptor-firefox-tp6-amazon]
--- a/testing/raptor/raptor/utils.py
+++ b/testing/raptor/raptor/utils.py
@@ -5,20 +5,32 @@
 
 from __future__ import absolute_import
 
 from mozlog import get_proxy_logger
 
 LOG = get_proxy_logger(component="raptor-utils")
 
 
-def transform_platform(str_to_transform, config_platform):
+def transform_platform(str_to_transform, config_platform, config_processor=None):
     # transform platform name i.e. 'mitmproxy-rel-bin-{platform}.manifest'
     # transforms to 'mitmproxy-rel-bin-osx.manifest'
-    if '{platform}' not in str_to_transform:
+    # also will transform '{x64}' if needed for 64 bit / win 10
+    if '{platform}' not in str_to_transform and '{x64}' not in str_to_transform:
         return str_to_transform
+
     if 'win' in config_platform:
         platform_id = 'win'
     elif config_platform == 'mac':
         platform_id = 'osx'
     else:
         platform_id = 'linux64'
-    return str_to_transform.replace('{platform}', platform_id)
+
+    if '{platform}' in str_to_transform:
+        str_to_transform = str_to_transform.replace('{platform}', platform_id)
+
+    if '{x64}' in str_to_transform and config_processor is not None:
+        if 'x86_64' in config_processor:
+            str_to_transform = str_to_transform.replace('{x64}', '_x64')
+        else:
+            str_to_transform = str_to_transform.replace('{x64}', '')
+
+    return str_to_transform