Bug 1350413 part 1: move macosx64/opt build to buildbot-bridge. r=dustin
Because of
bug 1338651, we need to stick to BBB macosx64 builds for now.
MozReview-Commit-ID: AwQc5r6ikUN
--- a/taskcluster/ci/build/macosx.yml
+++ b/taskcluster/ci/build/macosx.yml
@@ -18,33 +18,32 @@ macosx64/debug:
- builds/releng_base_mac_64_cross_builds.py
- balrog/production.py
script: "mozharness/scripts/fx_desktop_build.py"
secrets: true
custom-build-variant-cfg: cross-debug
tooltool-downloads: internal
macosx64/opt:
- description: "MacOS X x64 Cross-compile"
+ description: "MacOS X x64"
index:
product: firefox
job-name: macosx64-opt
treeherder:
platform: osx-10-7/opt
symbol: tc(B)
tier: 2
- worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+ worker-type: buildbot-bridge/buildbot-bridge
worker:
- implementation: docker-worker
- max-run-time: 36000
+ implementation: buildbot-bridge
run:
using: mozharness
actions: [get-secrets build generate-build-stats update]
config:
- - builds/releng_base_mac_64_cross_builds.py
+ - builds/releng_base_mac_64_builds.py
- balrog/production.py
script: "mozharness/scripts/fx_desktop_build.py"
secrets: true
tooltool-downloads: internal
macosx64-qr/debug:
description: "MacOS X x64 QuantumRender"
index:
--- a/taskcluster/taskgraph/morph.py
+++ b/taskcluster/taskgraph/morph.py
@@ -152,12 +152,97 @@ def add_index_tasks(taskgraph, label_to_
if added:
taskgraph, label_to_taskid = amend_taskgraph(
taskgraph, label_to_taskid, added)
logger.info('Added {} index tasks'.format(len(added)))
return taskgraph, label_to_taskid
+def make_s3_uploader_task(parent_task):
+ if parent_task.task['payload']['sourcestamp']['branch'] == 'try':
+ worker_type = 'buildbot-try'
+ else:
+ worker_type = 'buildbot'
+
+ task_def = {
+ # The null-provisioner and buildbot worker type don't actually exist.
+ # So this task doesn't actually run - we just need to create the task so
+ # we have something to attach artifacts to.
+ "provisionerId": "null-provisioner",
+ "workerType": worker_type,
+ "created": {'relative-datestamp': '0 seconds'},
+ "deadline": parent_task.task['deadline'],
+ "routes": [],
+ "payload": {},
+ "metadata": {
+ "name": "Buildbot/mozharness S3 uploader",
+ "description": "Upload outputs of buildbot/mozharness builds to S3",
+ "owner": "mshal@mozilla.com",
+ "source": "http://hg.mozilla.org/build/mozharness/",
+ }
+ }
+ label = 's3-uploader-{}'.format(parent_task.label)
+ dependencies = {}
+ task = Task(kind='misc', label=label, attributes={}, task=task_def,
+ dependencies=dependencies)
+ task.task_id = parent_task.task['payload']['properties']['upload_to_task_id']
+ return task
+
+
+def update_test_tasks(taskid, build_taskid, taskgraph):
+ """Tests task must download artifacts from uploader task.
+
+ Notice they don't need to depend on uploader task because it finishes
+ before the build task.
+ """
+ # Notice we handle buildbot-bridge, native, and generic-worker payloads
+ # We can do better here in terms of graph searching
+ # We could do post order search and stop as soon as we
+ # reach the build task. Not worring about it because this is
+ # (supposed to be) a temporary solution.
+ for task in taskgraph.tasks.itervalues():
+ if build_taskid in task.task.get('dependencies', []):
+ payload = task.task['payload']
+ if 'command' in payload:
+ try:
+ payload['command'] = [
+ cmd.replace(build_taskid, taskid) for cmd in payload['command']
+ ]
+ except AttributeError:
+ # generic-worker command attribute is an list of lists
+ payload['command'] = [
+ [cmd.replace(build_taskid, taskid) for cmd in x]
+ for x in payload['command']
+ ]
+ if 'mounts' in payload:
+ for mount in payload['mounts']:
+ if mount.get('content', {}).get('taskId', '') == build_taskid:
+ mount['content']['taskId'] = taskid
+ if 'env' in payload:
+ payload['env'] = {
+ k: v.replace(build_taskid, taskid) for k, v in payload['env'].iteritems()
+ }
+ if 'properties' in payload:
+ payload['properties']['parent_task_id'] = taskid
+
+
+def add_s3_uploader_task(taskgraph, label_to_taskid):
+ """The S3 uploader task is used by mozharness to upload buildbot artifacts."""
+ for task in taskgraph.tasks.itervalues():
+ if 'upload_to_task_id' in task.task.get('payload', {}).get('properties', {}):
+ added = make_s3_uploader_task(task)
+ taskgraph, label_to_taskid = amend_taskgraph(
+ taskgraph, label_to_taskid, [added])
+ update_test_tasks(added.task_id, task.task_id, taskgraph)
+ logger.info('Added s3-uploader task')
+ return taskgraph, label_to_taskid
+
+
def morph(taskgraph, label_to_taskid):
"""Apply all morphs"""
- taskgraph, label_to_taskid = add_index_tasks(taskgraph, label_to_taskid)
+ morphs = [
+ add_index_tasks,
+ add_s3_uploader_task,
+ ]
+ for m in morphs:
+ taskgraph, label_to_taskid = m(taskgraph, label_to_taskid)
return taskgraph, label_to_taskid
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -4,16 +4,17 @@
"""
Support for running jobs via mozharness. Ideally, most stuff gets run this
way, and certainly anything using mozharness should use this approach.
"""
from __future__ import absolute_import, print_function, unicode_literals
+import slugid
from textwrap import dedent
from taskgraph.util.schema import Schema
from voluptuous import Required, Optional, Any
from taskgraph.transforms.job import run_job_using
from taskgraph.transforms.job.common import (
@@ -239,8 +240,32 @@ def mozharness_on_generic_worker(config,
r'icacls z:\build /grant *S-1-1-0:D /L',
r'cd /d z:\build',
])
worker['command'].extend([
' '.join(hg_command),
' '.join(mh_command)
])
+
+
+@run_job_using('buildbot-bridge', 'mozharness', schema=mozharness_run_schema)
+def mozharness_on_buildbot_bridge(config, job, taskdesc):
+ run = job['run']
+ worker = taskdesc['worker']
+ branch = config.params['project']
+ product = run.get('index', {}).get('product', 'firefox')
+
+ worker.pop('env', None)
+
+ worker.update({
+ 'buildername': 'OS X 10.7 {} build'.format(branch),
+ 'sourcestamp': {
+ 'branch': branch,
+ 'repository': config.params['head_repository'],
+ 'revision': config.params['head_rev'],
+ },
+ 'properties': {
+ 'product': product,
+ 'who': config.params['owner'],
+ 'upload_to_task_id': slugid.nice(),
+ }
+ })
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@@ -205,27 +205,35 @@ def mozharness_test_on_generic_worker(co
'path': 'logs/log_raw.log',
'type': 'file'
},
{
'name': 'public/logs/log_warning.log',
'path': 'logs/log_warning.log',
'type': 'file'
},
- {
+ ]
+
+ # jittest doesn't have blob_upload_dir
+ if test['test-name'] != 'jittest':
+ artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
- }
- ]
+ })
build_platform = taskdesc['attributes']['build_platform']
+ build_type = taskdesc['attributes']['build_type']
- target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform) \
- if build_platform.startswith('win') else 'target'
+ if build_platform.startswith('win'):
+ target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform)
+ elif build_platform == 'macosx64' and build_type == 'opt':
+ target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac')
+ else:
+ target = 'target'
installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
test_packages_url = get_artifact_url(
'<build>', 'public/build/{}.test_packages.json'.format(target))
taskdesc['scopes'].extend(
['generic-worker:os-group:{}'.format(group) for group in test['os-groups']])
@@ -325,19 +333,24 @@ def mozharness_test_on_generic_worker(co
@run_job_using('native-engine', 'mozharness-test', schema=mozharness_test_run_schema)
def mozharness_test_on_native_engine(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
+ build_platform = taskdesc['attributes']['build_platform']
+ build_type = taskdesc['attributes']['build_type']
+ target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac') \
+ if build_platform == 'macosx64' and build_type == 'opt' else 'target'
+
installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
test_packages_url = get_artifact_url('<build>',
- 'public/build/target.test_packages.json')
+ 'public/build/{}.test_packages.json'.format(target))
mozharness_url = get_artifact_url('<build>',
'public/build/mozharness.zip')
worker['artifacts'] = [{
'name': prefix.rstrip('/'),
'path': path.rstrip('/'),
'type': 'directory',
} for (prefix, path) in ARTIFACTS]
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -341,17 +341,23 @@ def set_defaults(config, tests):
yield test
@transforms.add
def set_target(config, tests):
for test in tests:
build_platform = test['build-platform']
if build_platform.startswith('macosx'):
- target = 'target.dmg'
+ if build_platform.split('/')[1] == 'opt':
+ target = 'firefox-{}.en-US.{}.dmg'.format(
+ get_firefox_version(),
+ 'mac',
+ )
+ else:
+ target = 'target.dmg'
elif build_platform.startswith('android'):
if 'geckoview' in test['test-name']:
target = 'geckoview_example.apk'
else:
target = 'target.apk'
elif build_platform.startswith('win'):
target = 'firefox-{}.en-US.{}.zip'.format(
get_firefox_version(),
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -1414,28 +1414,32 @@ or run without that action (ie: --no-{ac
'locale': locale,
}
fmt.update(self.buildid_to_dict(self.query_buildid()))
routes = []
for template in templates:
routes.append(template.format(**fmt))
self.info("Using routes: %s" % routes)
+ taskid = self.buildbot_config['properties'].get('upload_to_task_id')
tc = Taskcluster(
branch=self.branch,
rank=pushinfo.pushdate, # Use pushdate as the rank
client_id=self.client_id,
access_token=self.access_token,
log_obj=self.log_obj,
# `upload_to_task_id` is used by mozci to have access to where the artifacts
# will be uploaded
- task_id=self.buildbot_config['properties'].get('upload_to_task_id'),
+ task_id=taskid,
)
- task = tc.create_task(routes)
+ if taskid:
+ task = tc.get_task(taskid)
+ else:
+ task = tc.create_task(routes)
tc.claim_task(task)
# Only those files uploaded with valid extensions are processed.
# This ensures that we get the correct packageUrl from the list.
valid_extensions = (
'.apk',
'.dmg',
'.mar',