Bug 1350413 part 2: Fix generic-worker configuration. r=dustin draft
authorWander Lairson Costa <wcosta@mozilla.com>
Tue, 02 May 2017 16:12:52 -0300
changeset 571520 aa931c26ab887569724c0847c4a9c25a2c04c584
parent 571519 d5ee1605bf5ec70223e7676bd2d2fd760841d59a
child 626784 7218a0d5637bb04f6547e74e23a9fb5c14ea5734
push id56818
push userwcosta@mozilla.com
push dateTue, 02 May 2017 19:13:07 +0000
reviewersdustin
bugs1350413
milestone55.0a1
Bug 1350413 part 2: Fix generic-worker configuration. r=dustin * Add macosx64 talos tests configuration. * Fix logs artifacts. * jittest doesn't have blob_upload_dir. * e10s config on talos goes on suite name. MozReview-Commit-ID: E9rSv2zQqvL
taskcluster/ci/test/tests.yml
taskcluster/taskgraph/transforms/job/mozharness_test.py
taskcluster/taskgraph/transforms/tests.py
--- a/taskcluster/ci/test/tests.yml
+++ b/taskcluster/ci/test/tests.yml
@@ -1146,189 +1146,234 @@ talos-chrome:
             linux64-stylo/.*: ['mozilla-central', 'try']
             default: ['mozilla-beta', 'mozilla-aurora', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
     max-run-time: 3600
     mozharness:
         script: talos_script.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
+                macosx.*:
+                    - remove_executables.py
+                    - talos/mac_config.py
                 default:
                     - talos/linux_config.py
                     - remove_executables.py
         extra-options:
             - --suite=chromez
+            - --add-option
+            - --webServer,localhost
 
 talos-dromaeojs:
     description: "Talos dromaeojs"
     suite: talos
     talos-try-name: dromaeojs
     treeherder-symbol: tc-T(d)
     run-on-projects:
         by-test-platform:
             linux64-stylo/.*: ['mozilla-central', 'try']
             default: ['mozilla-beta', 'mozilla-aurora', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
     max-run-time: 3600
     mozharness:
         script: talos_script.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
+                macosx.*:
+                    - remove_executables.py
+                    - talos/mac_config.py
                 default:
                     - talos/linux_config.py
                     - remove_executables.py
         extra-options:
             - --suite=dromaeojs
+            - --add-option
+            - --webServer,localhost
 
 talos-g1:
     description: "Talos g1"
     suite: talos
     talos-try-name: g1
     treeherder-symbol: tc-T(g1)
     run-on-projects:
         by-test-platform:
             linux64-stylo/.*: ['mozilla-central', 'try']
             default: ['mozilla-beta', 'mozilla-aurora', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try'] 
     max-run-time: 3600
     mozharness:
         script: talos_script.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
+                macosx.*:
+                    - remove_executables.py
+                    - talos/mac_config.py
                 default:
                     - talos/linux_config.py
                     - remove_executables.py
         extra-options:
             - --suite=g1
+            - --add-option
+            - --webServer,localhost
 
 talos-g2:
     description: "Talos g2"
     suite: talos
     talos-try-name: g2
     treeherder-symbol: tc-T(g2)
     max-run-time: 3600
     run-on-projects:
         by-test-platform:
             linux64-stylo/.*: ['mozilla-central', 'try']
             default: ['mozilla-beta', 'mozilla-aurora', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try'] 
     mozharness:
         script: talos_script.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
+                macosx.*:
+                    - remove_executables.py
+                    - talos/mac_config.py
                 default:
                     - talos/linux_config.py
                     - remove_executables.py
         extra-options:
             - --suite=g2
+            - --add-option
+            - --webServer,localhost
 
 talos-g3:
     description: "Talos g3"
     suite: talos
     talos-try-name: g3
     treeherder-symbol: tc-T(g3)
     run-on-projects:
         by-test-platform:
             linux64-stylo/.*: ['mozilla-central', 'try']
             default: ['mozilla-beta', 'mozilla-aurora', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try'] 
     max-run-time: 3600
     mozharness:
         script: talos_script.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
+                macosx.*:
+                    - remove_executables.py
+                    - talos/mac_config.py
                 default:
                     - talos/linux_config.py
                     - remove_executables.py
         extra-options:
             - --suite=g3
+            - --add-option
+            - --webServer,localhost
 
 talos-g4:
     description: "Talos g4"
     suite: talos
     talos-try-name: g4
     treeherder-symbol: tc-T(g4)
     run-on-projects:
         by-test-platform:
             linux64-stylo/.*: ['mozilla-central', 'try']
             default: ['mozilla-beta', 'mozilla-aurora', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try'] 
     max-run-time: 3600
     mozharness:
         script: talos_script.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
+                macosx.*:
+                    - remove_executables.py
+                    - talos/mac_config.py
                 default:
                     - talos/linux_config.py
                     - remove_executables.py
         extra-options:
             - --suite=g4
+            - --add-option
+            - --webServer,localhost
 
 talos-other:
     description: "Talos other"
     suite: talos
     talos-try-name: other
     treeherder-symbol: tc-T(o)
     run-on-projects:
         by-test-platform:
             linux64-stylo/.*: ['mozilla-central', 'try']
             default: ['mozilla-beta', 'mozilla-aurora', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try'] 
     max-run-time: 3600
     mozharness:
         script: talos_script.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
+                macosx.*:
+                    - remove_executables.py
+                    - talos/mac_config.py
                 default:
                     - talos/linux_config.py
                     - remove_executables.py
         extra-options:
             - --suite=other
+            - --add-option
+            - --webServer,localhost
 
 talos-svgr:
     description: "Talos svgr"
     suite: talos
     talos-try-name: svgr
     treeherder-symbol: tc-T(s)
     run-on-projects:
         by-test-platform:
             linux64-stylo/.*: ['mozilla-central', 'try']
             default: ['mozilla-beta', 'mozilla-aurora', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
     max-run-time: 3600
     mozharness:
         script: talos_script.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
+                macosx.*:
+                    - remove_executables.py
+                    - talos/mac_config.py
                 default:
                     - talos/linux_config.py
                     - remove_executables.py
         extra-options:
             - --suite=svgr
+            - --add-option
+            - --webServer,localhost
 
 talos-tp5o:
     description: "Talos tp5o"
     suite: talos
     talos-try-name: tp5o
     treeherder-symbol: tc-T(tp)
     run-on-projects:
         by-test-platform:
             linux64-stylo/.*: ['mozilla-central', 'try']
             default: ['mozilla-beta', 'mozilla-aurora', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
     max-run-time: 3600
     mozharness:
         script: talos_script.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
+                macosx.*:
+                    - remove_executables.py
+                    - talos/mac_config.py
                 default:
                     - talos/linux_config.py
                     - remove_executables.py
         extra-options:
             - --suite=tp5o
+            - --add-option
+            - --webServer,localhost
 
 talos-perf-reftest:
     description: "Talos perf-reftest"
     suite: talos
     talos-try-name: perf-reftest
     treeherder-symbol: tc-T(p)
     run-on-projects:
         by-test-platform:
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@@ -171,49 +171,19 @@ def mozharness_test_on_docker(config, jo
 @run_job_using('generic-worker', 'mozharness-test', schema=mozharness_test_run_schema)
 def mozharness_test_on_generic_worker(config, job, taskdesc):
     test = taskdesc['run']['test']
     mozharness = test['mozharness']
     worker = taskdesc['worker']
 
     artifacts = [
         {
-            'name': 'public/logs/localconfig.json',
-            'path': 'logs/localconfig.json',
-            'type': 'file'
-        },
-        {
-            'name': 'public/logs/log_critical.log',
-            'path': 'logs/log_critical.log',
-            'type': 'file'
-        },
-        {
-            'name': 'public/logs/log_error.log',
-            'path': 'logs/log_error.log',
-            'type': 'file'
-        },
-        {
-            'name': 'public/logs/log_fatal.log',
-            'path': 'logs/log_fatal.log',
-            'type': 'file'
-        },
-        {
-            'name': 'public/logs/log_info.log',
-            'path': 'logs/log_info.log',
-            'type': 'file'
-        },
-        {
-            'name': 'public/logs/log_raw.log',
-            'path': 'logs/log_raw.log',
-            'type': 'file'
-        },
-        {
-            'name': 'public/logs/log_warning.log',
-            'path': 'logs/log_warning.log',
-            'type': 'file'
+            'name': 'public/logs',
+            'path': 'logs',
+            'type': 'directory'
         },
     ]
 
     # jittest doesn't have blob_upload_dir
     if test['test-name'] != 'jittest':
         artifacts.append({
             'name': 'public/test_info',
             'path': 'build/blobber_upload_dir',
@@ -290,16 +260,19 @@ def mozharness_test_on_generic_worker(co
         mh_command.append('--no-read-buildbot-config')
     mh_command.extend(['--installer-url', installer_url])
     mh_command.extend(['--test-packages-url', test_packages_url])
     if mozharness.get('download-symbols'):
         if isinstance(mozharness['download-symbols'], basestring):
             mh_command.extend(['--download-symbols', mozharness['download-symbols']])
         else:
             mh_command.extend(['--download-symbols', 'true'])
+    if mozharness.get('include-blob-upload-branch'):
+        mh_command.append('--blob-upload-branch=' + config.params['project'])
+    mh_command.extend(mozharness.get('extra-options', []))
 
     # TODO: remove the need for run['chunked']
     if mozharness.get('chunked') or test['chunks'] > 1:
         # Implement mozharness['chunking-args'], modifying command in place
         if mozharness['chunking-args'] == 'this-chunk':
             mh_command.append('--total-chunk={}'.format(test['chunks']))
             mh_command.append('--this-chunk={}'.format(test['this-chunk']))
         elif mozharness['chunking-args'] == 'test-suite-suffix':
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -535,17 +535,22 @@ def split_e10s(config, tests):
         if e10s:
             test['test-name'] += '-e10s'
             test['e10s'] = True
             test['attributes']['e10s'] = True
             group, symbol = split_symbol(test['treeherder-symbol'])
             if group != '?':
                 group += '-e10s'
             test['treeherder-symbol'] = join_symbol(group, symbol)
-            test['mozharness']['extra-options'].append('--e10s')
+            if test['suite'] == 'talos':
+                for i, option in enumerate(test['mozharness']['extra-options']):
+                    if option.startswith('--suite='):
+                        test['mozharness']['extra-options'][i] += '-e10s'
+            else:
+                test['mozharness']['extra-options'].append('--e10s')
         yield test
 
 
 @transforms.add
 def split_chunks(config, tests):
     """Based on the 'chunks' key, split tests up into chunks by duplicating
     them and assigning 'this-chunk' appropriately and updating the treeherder
     symbol."""