Bug 1277595 - Create initial nightly graph, r=dustin
MozReview-Commit-ID: 2rZYddnGdvh
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/nightly-fennec/android-api-15-nightly-build.yml
@@ -0,0 +1,66 @@
+$inherits:
+ from: 'mobile_base.yml'
+ variables:
+ build_name: 'android'
+ build_type: 'opt'
+task:
+ metadata:
+ name: '[TC] Android armv7 API 15+'
+ description: 'Android armv7 API 15+'
+
+ workerType: android-api-15
+
+ routes:
+ - 'index.buildbot.branches.{{project}}.android-api-15'
+ - 'index.buildbot.revisions.{{head_rev}}.{{project}}.android-api-15'
+
+ scopes:
+ - 'docker-worker:cache:level-{{level}}-{{project}}-build-android-api-15-workspace'
+ - 'docker-worker:cache:tooltool-cache'
+ - 'docker-worker:relengapi-proxy:tooltool.download.internal'
+ - 'docker-worker:relengapi-proxy:tooltool.download.public'
+
+ payload:
+ cache:
+ level-{{level}}-{{project}}-build-android-api-15-workspace: '/home/worker/workspace'
+ tooltool-cache: '/home/worker/tooltool-cache'
+
+ features:
+ relengAPIProxy: true
+
+ env:
+ # inputs to mozharness
+ MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
+ # TODO: make these additional configuration files go away
+ MOZHARNESS_CONFIG: >
+ builds/releng_base_android_64_builds.py
+ disable_signing.py
+ platform_supports_post_upload_to_latest.py
+ taskcluster_nightly.py
+ MOZHARNESS_ACTIONS: "get-secrets build multi-l10n update"
+ MH_CUSTOM_BUILD_VARIANT_CFG: api-15
+ MH_BRANCH: {{project}}
+ MH_BUILD_POOL: taskcluster
+ TOOLTOOL_CACHE: '/home/worker/tooltool-cache'
+
+ command: ["/bin/bash", "bin/build.sh"]
+
+ extra:
+ treeherderEnv:
+ - production
+ - staging
+ treeherder:
+ machine:
+ # see https://github.com/mozilla/treeherder/blob/master/ui/js/values.js
+ platform: android-4-0-armv7-api15
+ groupSymbol: tc
+ groupName: Submitted by taskcluster
+ symbol: B
+ tier: 2
+ # Rather then enforcing particular conventions we require that all build
+ # tasks provide the "build" extra field to specify where the build and tests
+ # files are located.
+ locations:
+ build: 'public/build/target.apk'
+ mozharness: 'public/build/mozharness.zip'
+ test_packages: 'public/build/target.test_packages.json'
copy from taskcluster/ci/legacy/tasks/build.yml
copy to taskcluster/ci/nightly-fennec/build.yml
copy from taskcluster/ci/legacy/tasks/docker_build.yml
copy to taskcluster/ci/nightly-fennec/docker_build.yml
--- a/taskcluster/ci/legacy/tasks/docker_build.yml
+++ b/taskcluster/ci/nightly-fennec/docker_build.yml
@@ -1,10 +1,10 @@
$inherits:
- from: 'tasks/build.yml'
+ from: 'build.yml'
task:
workerType: b2gbuild
routes:
- 'index.gecko.v1.{{project}}.revision.linux.{{head_rev}}.{{build_name}}.{{build_type}}'
- 'index.gecko.v1.{{project}}.latest.linux.{{build_name}}.{{build_type}}'
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/nightly-fennec/kind.yml
@@ -0,0 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: 'taskgraph.task.nightly_fennec:NightlyFennecTask'
+nightly_fennec_path: '.'
+
copy from taskcluster/ci/legacy/tasks/builds/mobile_base.yml
copy to taskcluster/ci/nightly-fennec/mobile_base.yml
--- a/taskcluster/ci/legacy/tasks/builds/mobile_base.yml
+++ b/taskcluster/ci/nightly-fennec/mobile_base.yml
@@ -1,10 +1,10 @@
$inherits:
- from: 'tasks/docker_build.yml'
+ from: 'docker_build.yml'
variables:
build_product: 'mobile'
docker-image: desktop-build
task:
payload:
image:
type: 'task-image'
path: 'public/image.tar'
copy from taskcluster/ci/legacy/routes.json
copy to taskcluster/ci/nightly-fennec/routes.json
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/signing/kind.yml
@@ -0,0 +1,8 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: 'taskgraph.task.signing:SigningTask'
+signing_path: '.'
+kind-dependencies:
+ - nightly-fennec
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/signing/signing.yml
@@ -0,0 +1,22 @@
+task:
+ provisionerId: "aws-provisioner-v1"
+ workerType: "desktop-test"
+ schedulerId: task-graph-scheduler
+
+ created:
+ relative-datestamp: "0 seconds"
+ deadline:
+ relative-datestamp: "24 hours"
+ payload:
+ image: "ubuntu:13.10"
+ command:
+ - "/bin/bash"
+ - "-c"
+ - "echo \"hello World\""
+ maxRunTime: 600
+ metadata:
+ name: "Signing Fennec Nightly Task"
+ description: "Markdown description of **what** this task does"
+ owner: "amiyaguchi@mozilla.com"
+ source: "https://tools.taskcluster.net/task-creator/"
+
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -79,8 +79,18 @@ def target_tasks_ash_tasks(full_task_gra
'linux64-asan',
'linux64-pgo',
])):
return False
if not attrmatch(attrs, e10s=True):
return False
return True
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
+
+
+@_target_task('nightly_fennec')
+def target_tasks_nightly(full_task_graph, parameters):
+ """Select the set of tasks required for a nightly build of fennec. The
+ nightly build process involves a pipeline of builds, signing,
+ and, eventually, uploading the tasks to balrog."""
+ return [t.label for t in full_task_graph.tasks.itervalues()
+ if t.attributes.get('kind') in ['nightly-fennec', 'signing']]
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/task/nightly_fennec.py
@@ -0,0 +1,113 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import json
+import os
+import time
+
+from . import base
+from .legacy import query_vcs_info, decorate_task_json_routes, gaia_info, \
+ mklabel
+from taskgraph.util.templates import Templates
+from taskgraph.util.docker import docker_image
+
+
+logger = logging.getLogger(__name__)
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
+
+class NightlyFennecTask(base.Task):
+
+ def __init__(self, *args, **kwargs):
+ self.task_dict = kwargs.pop('task_dict')
+ super(NightlyFennecTask, self).__init__(*args, **kwargs)
+
+ @classmethod
+ def load_tasks(cls, kind, path, config, params, loaded_tasks):
+ root = os.path.abspath(os.path.join(path, config[
+ 'nightly_fennec_path']))
+
+ project = params['project']
+
+ # Set up the parameters, including the time of the build
+ push_epoch = int(time.time())
+ vcs_info = query_vcs_info(params['head_repository'], params['head_rev'])
+ changed_files = set()
+ if vcs_info:
+ push_epoch = vcs_info.pushdate
+
+ logger.debug(
+ '{} commits influencing task scheduling:'.format(len(vcs_info.changesets)))
+ for c in vcs_info.changesets:
+ logger.debug("{cset} {desc}".format(
+ cset=c['node'][0:12],
+ desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
+ changed_files |= set(c['files'])
+
+ pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(push_epoch))
+
+ # Template parameters used when expanding the graph
+ parameters = dict(gaia_info().items() + {
+ 'index': 'index',
+ 'project': project,
+ 'pushlog_id': params.get('pushlog_id', 0),
+ 'base_repository': params['base_repository'] or
+ params['head_repository'],
+ 'docker_image': docker_image,
+ 'head_repository': params['head_repository'],
+ 'head_ref': params['head_ref'] or params['head_rev'],
+ 'head_rev': params['head_rev'],
+ 'pushdate': pushdate,
+ 'pushtime': pushdate[8:],
+ 'year': pushdate[0:4],
+ 'month': pushdate[4:6],
+ 'day': pushdate[6:8],
+ 'rank': push_epoch,
+ 'owner': params['owner'],
+ 'level': params['level'],
+ 'build_slugid': mklabel(),
+ 'source': '{repo}file/{rev}/taskcluster/ci/nightly-fennec'.format(
+ repo=params['head_repository'], rev=params['head_rev']),
+ 'build_name': 'android',
+ 'build_type': 'opt',
+ 'build_product': 'mobile'
+ }.items())
+
+ routes_file = os.path.join(root, 'routes.json')
+ with open(routes_file) as f:
+ contents = json.load(f)
+ json_routes = contents['routes']
+
+ tasks = []
+ templates = Templates(root)
+
+ task = templates.load('android-api-15-nightly-build.yml', parameters)
+ decorate_task_json_routes(task['task'], json_routes, parameters)
+
+ attributes = {'kind': 'nightly-fennec'}
+ tasks.append(cls(kind, 'build-nightly-fennec',
+ task=task['task'], attributes=attributes,
+ task_dict=task))
+
+ # Convert to a dictionary of tasks. The process above has invented a
+ # taskId for each task, and we use those as the *labels* for the tasks;
+ # taskgraph will later assign them new taskIds.
+ return tasks
+
+ def get_dependencies(self, taskgraph):
+ deps = [(label, label) for label in self.task_dict.get('requires', [])]
+
+ # add a dependency on an image task, if needed
+ if 'docker-image' in self.task_dict:
+ deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict),
+ 'docker-image'))
+
+ return deps
+
+ def optimize(self):
+ return False, None
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/task/signing.py
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import json
+import os
+import time
+
+from . import base
+from taskgraph.util.templates import Templates
+
+
+logger = logging.getLogger(__name__)
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
+
+class SigningTask(base.Task):
+
+ def __init__(self, *args, **kwargs):
+ super(SigningTask, self).__init__(*args, **kwargs)
+
+ @classmethod
+ def load_tasks(cls, kind, path, config, params, loaded_tasks):
+ root = os.path.abspath(os.path.join(path, config['signing_path']))
+
+ # get each nightly-fennec and add its name to this task
+ fennec_tasks = [t for t in loaded_tasks if t.attributes.get('kind')
+ == 'nightly-fennec']
+
+ tasks = []
+ for fennec_task in fennec_tasks:
+ templates = Templates(root)
+ task = templates.load('signing.yml', {})
+ attributes = {'kind': 'signing'}
+ tasks.append(cls(kind, 'signing-nightly-fennec', task=task['task'],
+ attributes=attributes))
+
+ return tasks
+
+ def get_dependencies(self, taskgraph):
+ return [('build-nightly-fennec', 'build-nightly-fennec')]
+
+ def optimize(self):
+ return False, None