Bug 1329282 - Task kind for building QEMU images
* Defines a loader for creating a task for each folder
* Defines a transform that creates a task which builds
and indexes a QEMU image.
* Defines a new task kind: qemu-image
MozReview-Commit-ID: 7iiCRjktP65
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/qemu-image/kind.yml
@@ -0,0 +1,13 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+loader: taskgraph.loader.folders:loader
+
+transforms:
+ - taskgraph.transforms.qemu_image:transforms
+ - taskgraph.transforms.task:transforms
+
+# taskgraph.loader.folders:loader will create a task for each subfolder in:
+folder: taskcluster/qemu/
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -149,16 +149,22 @@ Docker images are built from subdirector
``docker build``. There is currently no capability for one Docker image to
depend on another in-tree docker image, without uploading the latter to a
Docker repository
The task definition used to create the image-building tasks is given in
``image.yml`` in the kind directory, and is interpreted as a :doc:`YAML
Template <yaml-templates>`.
+qemu-image
+----------
+
+The ``qemu-image`` task kind builds QEMU images based on in-tree image definitions.
+See ``taskcluster/qemu/`` for images and details.
+
android-stuff
-------------
balrog
------
Balrog is the Mozilla Update Server. Jobs of this kind are submitting information
which assists in telling Firefox that an update is available for the related job.
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/loader/folders.py
@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import logging
+
+from .. import GECKO
+
+logger = logging.getLogger(__name__)
+
+
+def loader(kind, path, config, params, loaded_tasks):
+ """
+ Given keys "folder" and "exclude" this loader will create a job for each
+ subfolder in folder (relative to GECKO root), jobs will have the form:
+ ``{name, path}``, where name is the name of the subfolder and path is
+ the absolute path to the folder.
+ """
+ # Validate input
+ folder = config.get('folder')
+ if not isinstance(folder, basestring):
+ raise Exception('taskgraph.loader.folders:loader expects "folder" ' +
+ 'in kind.yml to be a string')
+ exclude = config.get('exclude', [])
+ if not isinstance(exclude, list) or not all((isinstance(s, basestring) for s in exclude)):
+ raise Exception('taskgraph.loader.folders:loader expects "exclude" ' +
+ 'in kind.yml to be a list of strings')
+
+ root_folder = os.path.join(GECKO, folder)
+ for entry in os.listdir(root_folder):
+ if not os.path.isdir(os.path.join(root_folder, entry)):
+ continue # Skip non-folders
+ if entry in exclude:
+ continue # Skip excluded folders
+ # Create job for subfolder
+ yield {
+ 'name': entry,
+ 'folder': os.path.join(root_folder, entry),
+ }
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/qemu_image.py
@@ -0,0 +1,107 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform {name, path} entries to task description template for QEMU images.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+from taskgraph.qemu.imagehash import compute_image_hash
+from taskgraph.qemu.imageschema import (
+ load_and_validate_image,
+ load_reference,
+ has_image,
+ has_reference,
+)
+from taskgraph.qemu.imagecache import (
+ PRIVATE_ARTIFACT_NAME,
+ PUBLIC_ARTIFACT_NAME,
+ index_namespace,
+ auxiliary_index_namespaces,
+)
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def create_qemu_image_task(config, tasks):
+ """
+ create tasks for building qemu images
+ """
+ # Find reference to qemu-image-builder image
+ builder_ref = load_reference('qemu-image-builder')
+
+ # For each folder on the form {name, path}
+ for folder in tasks:
+ image_name = folder['name']
+
+ # Skip folders that don't have image.yml or has a reference.yml
+ if not has_image(image_name) or has_reference(image_name):
+ continue
+
+ image = load_and_validate_image(image_name)
+ image_hash = compute_image_hash(image_name)
+
+ # Construct index namespaces
+ namespaces = [
+ index_namespace(image_name, image_hash, config.params['level']),
+ ] + auxiliary_index_namespaces(
+ image_name,
+ config.params['level'],
+ config.params['moz_build_date'],
+ )
+
+ # Allow it to be replaced with a level that is higher than or equal to
+ # current level, always prefer higher levels to improve caching.
+ optimization = {'index-search': [
+ index_namespace(image_name, image_hash, level)
+ for level in reversed(range(int(config.params['level']), 4))
+ ]}
+
+ yield {
+ 'label': 'build-qemu-image-{}'.format(image_name),
+ 'description': 'Build QEMU image: `{}` from in-tree recipe\n\n---\n{}'.format(
+ image_name, image['description'],
+ ),
+ 'attributes': {'image_name': image_name},
+ 'expires-after': '1 year',
+ 'routes': ['index.{}'.format(ns) for ns in namespaces],
+ 'optimization': optimization,
+ 'scopes': ['secrets:get:project/taskcluster/gecko/hgfingerprint'],
+ 'treeherder': {
+ 'symbol': image['symbol'],
+ 'platform': 'taskcluster-images/opt',
+ 'kind': 'other',
+ 'tier': 1,
+ },
+ 'run-on-projects': [],
+ 'worker-type': 'manual-packet/tc-worker-qemu-v1',
+ # can't use {in-tree: ..} here, otherwise we might try to build
+ # this image..
+ 'worker': {
+ 'implementation': 'qemu-engine',
+ 'image': {
+ 'url': builder_ref['url'],
+ 'sha256': builder_ref['sha256'],
+ },
+ 'command': [
+ 'run-task', '--vcs-checkout=/home/worker/checkouts/gecko', '--',
+ '/home/worker/checkouts/gecko/mach', 'qemu', 'build',
+ image_name, '--output', '/home/worker/image.tar.zst',
+ ],
+ 'env': {
+ 'GECKO_BASE_REPOSITORY': config.params['base_repository'],
+ 'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
+ 'GECKO_HEAD_REV': config.params['head_rev'],
+ },
+ 'artifacts': [{
+ 'type': 'file',
+ 'name': PRIVATE_ARTIFACT_NAME if image['private'] else PUBLIC_ARTIFACT_NAME,
+ 'path': '/home/worker/image.tar.zst',
+ }],
+ 'max-run-time': '3 hours',
+ },
+ }
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -23,16 +23,18 @@ from taskgraph.util.attributes import TR
from taskgraph.util.hash import hash_path
from taskgraph.util.treeherder import split_symbol
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import validate_schema, Schema
from taskgraph.util.scriptworker import get_release_config
from voluptuous import Any, Required, Optional, Extra
from taskgraph import GECKO
from ..util import docker as dockerutil
+import taskgraph.qemu.imageschema
+import taskgraph.qemu.imagecache
from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
RUN_TASK = os.path.join(GECKO, 'taskcluster', 'docker', 'recipes', 'run-task')
@memoize
@@ -397,16 +399,47 @@ task_description_schema = Schema({
# task image path from which to read artifact
Required('path'): basestring,
# name of the produced artifact (root of the names for
# type=directory)
Required('name'): basestring,
}],
}, {
+ Required('implementation'): 'qemu-engine',
+
+ # QEMU image reference
+ Required('image'): Any(
+ basestring, # URL to image
+ {'in-tree': basestring}, # in-tree image by name
+ {'url': basestring, 'sha256': basestring}, # url and sha256
+ ),
+
+ # the command to run
+ Required('command'): [basestring],
+
+ # environment variables
+ Required('env'): {basestring: taskref_or_string},
+
+ # Maximum runtime
+ Required('max-run-time'): basestring,
+
+ # artifacts to extract from the task image after completion
+ Optional('artifacts'): [{
+ # type of artifact -- simple file, or recursive directory
+ Required('type'): Any('file', 'directory'),
+
+ # task image path from which to read artifact
+ Required('path'): basestring,
+
+ # name of the produced artifact (root of the names for
+ # type=directory)
+ Required('name'): basestring,
+ }],
+ }, {
Required('implementation'): 'scriptworker-signing',
# the maximum time to spend signing, in seconds
Required('max-run-time', default=600): int,
# list of artifact URLs for the artifacts that should be signed
Required('upstream-artifacts'): [{
# taskId of the task with the artifact
@@ -970,16 +1003,53 @@ def build_macosx_engine_payload(config,
}
if worker.get('reboot'):
task_def['payload'] = worker['reboot']
if task.get('needs-sccache'):
raise Exception('needs-sccache not supported in native-engine')
+@payload_builder('qemu-engine')
+def build_taskcluster_worker_qemu_engine_payload(config, task, task_def):
+ worker = task['worker']
+ artifacts = map(lambda artifact: {
+ 'name': artifact['name'],
+ 'path': artifact['path'],
+ 'type': artifact['type'],
+ 'expires': task_def['expires'],
+ }, worker.get('artifacts', []))
+ image = worker['image']
+ if isinstance(image, dict) and 'in-tree' in image:
+ image_name = image['in-tree']
+ ref = taskgraph.qemu.imageschema.load_reference(image_name)
+ if ref:
+ image = {
+ 'url': ref['url'],
+ 'sha256': ref['sha256'],
+ }
+ else:
+ img = taskgraph.qemu.imageschema.load_and_validate_image(image_name)
+ task.setdefault('dependencies', {})['qemu-image'] = 'build-qemu-image-' + image_name
+ artifact = taskgraph.qemu.imagecache.PUBLIC_ARTIFACT_NAME
+ if img['private']:
+ artifact = taskgraph.qemu.imagecache.PRIVATE_ARTIFACT_NAME
+ image = {
+ 'taskId': {'task-reference': '<qemu-image>'},
+ 'artifact': artifact,
+ }
+ task_def['payload'] = {
+ 'image': image,
+ 'command': worker['command'],
+ 'env': worker['env'],
+ 'artifacts': artifacts,
+ 'maxRunTime': worker['max-run-time'],
+ }
+
+
@payload_builder('buildbot-bridge')
def build_buildbot_bridge_payload(config, task, task_def):
task['extra'].pop('treeherder', None)
task['extra'].pop('treeherderEnv', None)
worker = task['worker']
task_def['payload'] = {
'buildername': worker['buildername'],
'sourcestamp': worker['sourcestamp'],