Bug 1431161 - run windows 32 and 64 bit builds on windows10-64 hardware for talos performance tests; r?jmaher draft
authorRob Wood <rwood@mozilla.com>
Tue, 06 Feb 2018 13:18:09 -0500
changeset 761881 2603c13f577e07a8ad1d155f9bdf762648ac6a2c
parent 761864 9683f24ff8ec5ba768c4a0a124d8439c228b2c8b
child 761882 6761c00d068dfd95ac88d16293d34c22438876bb
push id101026
push userrwood@mozilla.com
push dateThu, 01 Mar 2018 16:13:22 +0000
reviewersjmaher
bugs1431161
milestone60.0a1
Bug 1431161 - run windows 32 and 64 bit builds on windows10-64 hardware for talos performance tests; r?jmaher MozReview-Commit-ID: EVIHh42fpHU
taskcluster/taskgraph/transforms/tests.py
testing/mozharness/configs/talos/windows_config.py
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -47,37 +47,37 @@ LINUX_WORKER_TYPES = {
     'default': 'aws-provisioner-v1/gecko-t-linux-large',
 }
 
 # windows worker types keyed by test-platform and virtualization
 WINDOWS_WORKER_TYPES = {
     'windows7-32': {
       'virtual': 'aws-provisioner-v1/gecko-t-win7-32',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
-      'hardware': 'releng-hardware/gecko-t-win7-32-hw',
+      'hardware': 'releng-hardware/gecko-t-win10-64-hw',
     },
     'windows7-32-pgo': {
       'virtual': 'aws-provisioner-v1/gecko-t-win7-32',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
-      'hardware': 'releng-hardware/gecko-t-win7-32-hw',
+      'hardware': 'releng-hardware/gecko-t-win10-64-hw',
     },
     'windows7-32-nightly': {
       'virtual': 'aws-provisioner-v1/gecko-t-win7-32',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
-      'hardware': 'releng-hardware/gecko-t-win7-32-hw',
+      'hardware': 'releng-hardware/gecko-t-win10-64-hw',
     },
     'windows7-32-devedition': {
       'virtual': 'aws-provisioner-v1/gecko-t-win7-32',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
-      'hardware': 'releng-hardware/gecko-t-win7-32-hw',
+      'hardware': 'releng-hardware/gecko-t-win10-64-hw',
     },
     'windows7-32-stylo-disabled': {
       'virtual': 'aws-provisioner-v1/gecko-t-win7-32',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
-      'hardware': 'releng-hardware/gecko-t-win7-32-hw',
+      'hardware': 'releng-hardware/gecko-t-win10-64-hw',
     },
     'windows10-64': {
       'virtual': 'aws-provisioner-v1/gecko-t-win10-64',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win10-64-gpu',
       'hardware': 'releng-hardware/gecko-t-win10-64-hw',
     },
     'windows10-64-ccov': {
       'virtual': 'aws-provisioner-v1/gecko-t-win10-64',
@@ -921,59 +921,45 @@ def single_stylo_traversal_tests(config,
 
 @transforms.add
 def set_worker_type(config, tests):
     """Set the worker type based on the test platform."""
     for test in tests:
         # during the taskcluster migration, this is a bit tortured, but it
         # will get simpler eventually!
         test_platform = test['test-platform']
-        try_options = config.params['try_options'] if config.params['try_options'] else {}
         if test.get('worker-type'):
             # This test already has its worker type defined, so just use that (yields below)
             pass
         elif test_platform.startswith('macosx'):
             test['worker-type'] = MACOSX_WORKER_TYPES['macosx64']
         elif test_platform.startswith('win'):
-            win_worker_type_platform = WINDOWS_WORKER_TYPES[
-                test_platform.split('/')[0]
-            ]
-            if test.get('suite', '') == 'talos' and 'ccov' not in test['build-platform']:
-                if try_options.get('taskcluster_worker'):
-                    test['worker-type'] = win_worker_type_platform['hardware']
-                elif test['virtualization'] == 'virtual':
-                    test['worker-type'] = win_worker_type_platform[test['virtualization']]
-                else:
-                    test['worker-type'] = 'buildbot-bridge/buildbot-bridge'
+            # figure out what platform the job needs to run on
+            if test['virtualization'] == 'hardware':
+                # some jobs like talos and reftest run on real h/w - those are all win10
+                win_worker_type_platform = WINDOWS_WORKER_TYPES['windows10-64']
             else:
-                test['worker-type'] = win_worker_type_platform[test['virtualization']]
+                # the other jobs run on a vm which may or may not be a win10 vm
+                win_worker_type_platform = WINDOWS_WORKER_TYPES[
+                    test_platform.split('/')[0]
+                ]
+            # now we have the right platform set the worker type accordingly
+            test['worker-type'] = win_worker_type_platform[test['virtualization']]
         elif test_platform.startswith('linux') or test_platform.startswith('android'):
             if test.get('suite', '') == 'talos' and test['build-platform'] != 'linux64-ccov/opt':
                 test['worker-type'] = 'releng-hardware/gecko-t-linux-talos'
             else:
                 test['worker-type'] = LINUX_WORKER_TYPES[test['instance-size']]
         else:
             raise Exception("unknown test_platform {}".format(test_platform))
 
         yield test
 
 
 @transforms.add
-def skip_win10_hardware(config, tests):
-    """Windows 10 hardware isn't ready yet, don't even bother scheduling
-    unless we're on try"""
-    for test in tests:
-        if 'releng-hardware/gecko-t-win10-64-hw' not in test['worker-type']:
-            yield test
-        if config.params == 'try':
-            yield test
-        # Silently drop the test on the floor if its win10 hardware and we're not try
-
-
-@transforms.add
 def make_job_description(config, tests):
     """Convert *test* descriptions to *job* descriptions (input to
     taskgraph.transforms.job)"""
 
     for test in tests:
         label = '{}-{}-{}'.format(config.kind, test['test-platform'], test['test-name'])
         if test['chunks'] > 1:
             label += '-{}'.format(test['this-chunk'])
--- a/testing/mozharness/configs/talos/windows_config.py
+++ b/testing/mozharness/configs/talos/windows_config.py
@@ -1,12 +1,13 @@
 import os
 import socket
+import sys
 
-PYTHON = 'c:/mozilla-build/python27/python.exe'
+PYTHON = sys.executable
 PYTHON_DLL = 'c:/mozilla-build/python27/python27.dll'
 VENV_PATH = os.path.join(os.getcwd(), 'build/venv')
 
 config = {
     "log_name": "talos",
     "buildbot_json_path": "buildprops.json",
     "installer_path": "installer.exe",
     "virtualenv_path": VENV_PATH,
@@ -17,17 +18,18 @@ config = {
     ],
     "virtualenv_modules": ['pywin32', 'talos', 'mozinstall'],
     "exes": {
         'python': PYTHON,
         'easy_install': ['%s/scripts/python' % VENV_PATH,
                          '%s/scripts/easy_install-2.7-script.py' % VENV_PATH],
         'mozinstall': ['%s/scripts/python' % VENV_PATH,
                        '%s/scripts/mozinstall-script.py' % VENV_PATH],
-        'hg': 'c:/mozilla-build/hg/hg',
+        'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg'),
+        'tooltool.py': [PYTHON, os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')],
     },
     "title": socket.gethostname().split('.')[0],
     "default_actions": [
         "clobber",
         "read-buildbot-config",
         "download-and-extract",
         "populate-webroot",
         "create-virtualenv",