Bug 1350413 part 1: move macosx64/opt build to buildbot-bridge. r=dustin draft
authorWander Lairson Costa <wcosta@mozilla.com>
Tue, 02 May 2017 16:12:51 -0300
changeset 571519 d5ee1605bf5ec70223e7676bd2d2fd760841d59a
parent 571414 bfc7b187005cabbc828ed9f5b61daf139c3cfd90
child 571520 aa931c26ab887569724c0847c4a9c25a2c04c584
push id56818
push userwcosta@mozilla.com
push dateTue, 02 May 2017 19:13:07 +0000
reviewersdustin
bugs1350413, 1338651
milestone55.0a1
Bug 1350413 part 1: move macosx64/opt build to buildbot-bridge. r=dustin Because of bug 1338651, we need to stick to BBB macosx64 builds for now. MozReview-Commit-ID: AwQc5r6ikUN
taskcluster/ci/build/macosx.yml
taskcluster/taskgraph/morph.py
taskcluster/taskgraph/transforms/job/mozharness.py
taskcluster/taskgraph/transforms/job/mozharness_test.py
taskcluster/taskgraph/transforms/tests.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
--- a/taskcluster/ci/build/macosx.yml
+++ b/taskcluster/ci/build/macosx.yml
@@ -18,33 +18,32 @@ macosx64/debug:
             - builds/releng_base_mac_64_cross_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: cross-debug
         tooltool-downloads: internal
 
 macosx64/opt:
-    description: "MacOS X x64 Cross-compile"
+    description: "MacOS X x64"
     index:
         product: firefox
         job-name: macosx64-opt
     treeherder:
         platform: osx-10-7/opt
         symbol: tc(B)
         tier: 2
-    worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+    worker-type: buildbot-bridge/buildbot-bridge
     worker:
-        implementation: docker-worker
-        max-run-time: 36000
+        implementation: buildbot-bridge
     run:
         using: mozharness
         actions: [get-secrets build generate-build-stats update]
         config:
-            - builds/releng_base_mac_64_cross_builds.py
+            - builds/releng_base_mac_64_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         tooltool-downloads: internal
 
 macosx64-qr/debug:
     description: "MacOS X x64 QuantumRender"
     index:
--- a/taskcluster/taskgraph/morph.py
+++ b/taskcluster/taskgraph/morph.py
@@ -152,12 +152,97 @@ def add_index_tasks(taskgraph, label_to_
     if added:
         taskgraph, label_to_taskid = amend_taskgraph(
             taskgraph, label_to_taskid, added)
         logger.info('Added {} index tasks'.format(len(added)))
 
     return taskgraph, label_to_taskid
 
 
+def make_s3_uploader_task(parent_task):
+    if parent_task.task['payload']['sourcestamp']['branch'] == 'try':
+        worker_type = 'buildbot-try'
+    else:
+        worker_type = 'buildbot'
+
+    task_def = {
+        # The null-provisioner and buildbot worker type don't actually exist.
+        # So this task doesn't actually run - we just need to create the task so
+        # we have something to attach artifacts to.
+        "provisionerId": "null-provisioner",
+        "workerType": worker_type,
+        "created": {'relative-datestamp': '0 seconds'},
+        "deadline": parent_task.task['deadline'],
+        "routes": [],
+        "payload": {},
+        "metadata": {
+            "name": "Buildbot/mozharness S3 uploader",
+            "description": "Upload outputs of buildbot/mozharness builds to S3",
+            "owner": "mshal@mozilla.com",
+            "source": "http://hg.mozilla.org/build/mozharness/",
+        }
+    }
+    label = 's3-uploader-{}'.format(parent_task.label)
+    dependencies = {}
+    task = Task(kind='misc', label=label, attributes={}, task=task_def,
+                dependencies=dependencies)
+    task.task_id = parent_task.task['payload']['properties']['upload_to_task_id']
+    return task
+
+
+def update_test_tasks(taskid, build_taskid, taskgraph):
+    """Tests task must download artifacts from uploader task.
+
+    Notice they don't need to depend on uploader task because it finishes
+    before the build task.
+    """
+    # Notice we handle buildbot-bridge, native, and generic-worker payloads
+    # We can do better here in terms of graph searching
+    # We could do post order search and stop as soon as we
+    # reach the build task. Not worring about it because this is
+    # (supposed to be) a temporary solution.
+    for task in taskgraph.tasks.itervalues():
+        if build_taskid in task.task.get('dependencies', []):
+            payload = task.task['payload']
+            if 'command' in payload:
+                try:
+                    payload['command'] = [
+                        cmd.replace(build_taskid, taskid) for cmd in payload['command']
+                    ]
+                except AttributeError:
+                    # generic-worker command attribute is an list of lists
+                    payload['command'] = [
+                        [cmd.replace(build_taskid, taskid) for cmd in x]
+                        for x in payload['command']
+                    ]
+            if 'mounts' in payload:
+                for mount in payload['mounts']:
+                    if mount.get('content', {}).get('taskId', '') == build_taskid:
+                        mount['content']['taskId'] = taskid
+            if 'env' in payload:
+                payload['env'] = {
+                    k: v.replace(build_taskid, taskid) for k, v in payload['env'].iteritems()
+                }
+            if 'properties' in payload:
+                payload['properties']['parent_task_id'] = taskid
+
+
+def add_s3_uploader_task(taskgraph, label_to_taskid):
+    """The S3 uploader task is used by mozharness to upload buildbot artifacts."""
+    for task in taskgraph.tasks.itervalues():
+        if 'upload_to_task_id' in task.task.get('payload', {}).get('properties', {}):
+            added = make_s3_uploader_task(task)
+            taskgraph, label_to_taskid = amend_taskgraph(
+                taskgraph, label_to_taskid, [added])
+            update_test_tasks(added.task_id, task.task_id, taskgraph)
+            logger.info('Added s3-uploader task')
+    return taskgraph, label_to_taskid
+
+
 def morph(taskgraph, label_to_taskid):
     """Apply all morphs"""
-    taskgraph, label_to_taskid = add_index_tasks(taskgraph, label_to_taskid)
+    morphs = [
+        add_index_tasks,
+        add_s3_uploader_task,
+    ]
+    for m in morphs:
+        taskgraph, label_to_taskid = m(taskgraph, label_to_taskid)
     return taskgraph, label_to_taskid
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -4,16 +4,17 @@
 """
 
 Support for running jobs via mozharness.  Ideally, most stuff gets run this
 way, and certainly anything using mozharness should use this approach.
 
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
+import slugid
 
 from textwrap import dedent
 
 from taskgraph.util.schema import Schema
 from voluptuous import Required, Optional, Any
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
@@ -239,8 +240,32 @@ def mozharness_on_generic_worker(config,
             r'icacls z:\build /grant *S-1-1-0:D /L',
             r'cd /d z:\build',
         ])
 
     worker['command'].extend([
         ' '.join(hg_command),
         ' '.join(mh_command)
     ])
+
+
+@run_job_using('buildbot-bridge', 'mozharness', schema=mozharness_run_schema)
+def mozharness_on_buildbot_bridge(config, job, taskdesc):
+    run = job['run']
+    worker = taskdesc['worker']
+    branch = config.params['project']
+    product = run.get('index', {}).get('product', 'firefox')
+
+    worker.pop('env', None)
+
+    worker.update({
+        'buildername': 'OS X 10.7 {} build'.format(branch),
+        'sourcestamp': {
+            'branch': branch,
+            'repository': config.params['head_repository'],
+            'revision': config.params['head_rev'],
+        },
+        'properties': {
+            'product': product,
+            'who': config.params['owner'],
+            'upload_to_task_id': slugid.nice(),
+        }
+    })
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@@ -205,27 +205,35 @@ def mozharness_test_on_generic_worker(co
             'path': 'logs/log_raw.log',
             'type': 'file'
         },
         {
             'name': 'public/logs/log_warning.log',
             'path': 'logs/log_warning.log',
             'type': 'file'
         },
-        {
+    ]
+
+    # jittest doesn't have blob_upload_dir
+    if test['test-name'] != 'jittest':
+        artifacts.append({
             'name': 'public/test_info',
             'path': 'build/blobber_upload_dir',
             'type': 'directory'
-        }
-    ]
+        })
 
     build_platform = taskdesc['attributes']['build_platform']
+    build_type = taskdesc['attributes']['build_type']
 
-    target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform) \
-        if build_platform.startswith('win') else 'target'
+    if build_platform.startswith('win'):
+        target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform)
+    elif build_platform == 'macosx64' and build_type == 'opt':
+        target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac')
+    else:
+        target = 'target'
 
     installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
 
     test_packages_url = get_artifact_url(
         '<build>', 'public/build/{}.test_packages.json'.format(target))
 
     taskdesc['scopes'].extend(
         ['generic-worker:os-group:{}'.format(group) for group in test['os-groups']])
@@ -325,19 +333,24 @@ def mozharness_test_on_generic_worker(co
 
 
 @run_job_using('native-engine', 'mozharness-test', schema=mozharness_test_run_schema)
 def mozharness_test_on_native_engine(config, job, taskdesc):
     test = taskdesc['run']['test']
     mozharness = test['mozharness']
     worker = taskdesc['worker']
 
+    build_platform = taskdesc['attributes']['build_platform']
+    build_type = taskdesc['attributes']['build_type']
+    target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac') \
+        if build_platform == 'macosx64' and build_type == 'opt' else 'target'
+
     installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
     test_packages_url = get_artifact_url('<build>',
-                                         'public/build/target.test_packages.json')
+                                         'public/build/{}.test_packages.json'.format(target))
     mozharness_url = get_artifact_url('<build>',
                                       'public/build/mozharness.zip')
 
     worker['artifacts'] = [{
         'name': prefix.rstrip('/'),
         'path': path.rstrip('/'),
         'type': 'directory',
     } for (prefix, path) in ARTIFACTS]
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -341,17 +341,23 @@ def set_defaults(config, tests):
         yield test
 
 
 @transforms.add
 def set_target(config, tests):
     for test in tests:
         build_platform = test['build-platform']
         if build_platform.startswith('macosx'):
-            target = 'target.dmg'
+            if build_platform.split('/')[1] == 'opt':
+                target = 'firefox-{}.en-US.{}.dmg'.format(
+                    get_firefox_version(),
+                    'mac',
+                )
+            else:
+                target = 'target.dmg'
         elif build_platform.startswith('android'):
             if 'geckoview' in test['test-name']:
                 target = 'geckoview_example.apk'
             else:
                 target = 'target.apk'
         elif build_platform.startswith('win'):
             target = 'firefox-{}.en-US.{}.zip'.format(
                 get_firefox_version(),
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -1414,28 +1414,32 @@ or run without that action (ie: --no-{ac
             'locale': locale,
         }
         fmt.update(self.buildid_to_dict(self.query_buildid()))
         routes = []
         for template in templates:
             routes.append(template.format(**fmt))
         self.info("Using routes: %s" % routes)
 
+        taskid = self.buildbot_config['properties'].get('upload_to_task_id')
         tc = Taskcluster(
             branch=self.branch,
             rank=pushinfo.pushdate, # Use pushdate as the rank
             client_id=self.client_id,
             access_token=self.access_token,
             log_obj=self.log_obj,
             # `upload_to_task_id` is used by mozci to have access to where the artifacts
             # will be uploaded
-            task_id=self.buildbot_config['properties'].get('upload_to_task_id'),
+            task_id=taskid,
         )
 
-        task = tc.create_task(routes)
+        if taskid:
+            task = tc.get_task(taskid)
+        else:
+            task = tc.create_task(routes)
         tc.claim_task(task)
 
         # Only those files uploaded with valid extensions are processed.
         # This ensures that we get the correct packageUrl from the list.
         valid_extensions = (
             '.apk',
             '.dmg',
             '.mar',