--- a/taskcluster/ci/android-stuff/kind.yml
+++ b/taskcluster/ci/android-stuff/kind.yml
@@ -1,17 +1,17 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# The name of this kind should suggest it's not meant to be permanent. This is
# a temporary place to generate these tasks in Bug 1286075 until they are
# rewritten in a better way.
-implementation: taskgraph.task.transform:TransformTask
+loader: taskgraph.task.transform:load_tasks
transforms:
- taskgraph.transforms.android_stuff:transforms
- taskgraph.transforms.task:transforms
jobs:
android-api-15-gradle-dependencies:
description: "Android armv7 API 15+ gradle dependencies"
--- a/taskcluster/ci/artifact-build/kind.yml
+++ b/taskcluster/ci/artifact-build/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.transform:TransformTask
+loader: taskgraph.task.transform:load_tasks
transforms:
- taskgraph.transforms.build_attrs:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
jobs:
linux64-artifact/opt:
--- a/taskcluster/ci/balrog/kind.yml
+++ b/taskcluster/ci/balrog/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.balrog:BalrogTask
+loader: taskgraph.task.balrog:load_tasks
transforms:
- taskgraph.transforms.balrog:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- beetmover
- beetmover-l10n
--- a/taskcluster/ci/beetmover-checksums/kind.yml
+++ b/taskcluster/ci/beetmover-checksums/kind.yml
@@ -1,12 +1,12 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.beetmover_checksums:BeetmoverChecksumsTask
+loader: taskgraph.task.beetmover_checksums:load_tasks
transforms:
- taskgraph.transforms.beetmover_checksums:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- checksums-signing
--- a/taskcluster/ci/beetmover-l10n/kind.yml
+++ b/taskcluster/ci/beetmover-l10n/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.beetmover:BeetmoverTask
+loader: taskgraph.task.beetmover:load_tasks
transforms:
- taskgraph.transforms.beetmover_l10n:transforms
- taskgraph.transforms.beetmover:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- nightly-l10n-signing
--- a/taskcluster/ci/beetmover/kind.yml
+++ b/taskcluster/ci/beetmover/kind.yml
@@ -1,12 +1,12 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.beetmover:BeetmoverTask
+loader: taskgraph.task.beetmover:load_tasks
transforms:
- taskgraph.transforms.beetmover:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- build-signing
--- a/taskcluster/ci/build-signing/kind.yml
+++ b/taskcluster/ci/build-signing/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.signing:SigningTask
+loader: taskgraph.task.signing:load_tasks
transforms:
- taskgraph.transforms.build_signing:transforms
- taskgraph.transforms.signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- build
--- a/taskcluster/ci/build/kind.yml
+++ b/taskcluster/ci/build/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.transform:TransformTask
+loader: taskgraph.task.transform:load_tasks
transforms:
- taskgraph.transforms.build:transforms
- taskgraph.transforms.build_attrs:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
jobs-from:
--- a/taskcluster/ci/checksums-signing/kind.yml
+++ b/taskcluster/ci/checksums-signing/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.checksums_signing:ChecksumsSigningTask
+loader: taskgraph.task.checksums_signing:load_tasks
transforms:
- taskgraph.transforms.checksums_signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- beetmover
- beetmover-l10n
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: 'taskgraph.task.docker_image:DockerImageTask'
+loader: taskgraph.task.docker_image:load_tasks
images_path: '../../../taskcluster/docker'
# make a task for each docker-image we might want. For the moment, since we
# write artifacts for each, these are whitelisted, but ideally that will change
# (to use subdirectory clones of the proper directory), at which point we can
# generate tasks for every docker image in the directory, secure in the
# knowledge that unnecessary images will be omitted from the target task graph
images:
--- a/taskcluster/ci/hazard/kind.yml
+++ b/taskcluster/ci/hazard/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.transform:TransformTask
+loader: taskgraph.task.transform:load_tasks
transforms:
- taskgraph.transforms.build_attrs:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
job-defaults:
treeherder:
--- a/taskcluster/ci/l10n/kind.yml
+++ b/taskcluster/ci/l10n/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.repacks:RepackTask
+loader: taskgraph.task.repacks:load_tasks
transforms:
- taskgraph.transforms.l10n:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
--- a/taskcluster/ci/nightly-l10n-signing/kind.yml
+++ b/taskcluster/ci/nightly-l10n-signing/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.signing:SigningTask
+loader: taskgraph.task.signing:load_tasks
transforms:
- taskgraph.transforms.nightly_l10n_signing:transforms
- taskgraph.transforms.signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- nightly-l10n
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.repacks:RepackTask
+loader: taskgraph.task.repacks:load_tasks
transforms:
- taskgraph.transforms.l10n:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- build
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.transform:TransformTask
+loader: taskgraph.task.transform:load_tasks
transforms:
- taskgraph.transforms.source_test:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
jobs-from:
- python-tests.yml
--- a/taskcluster/ci/spidermonkey/kind.yml
+++ b/taskcluster/ci/spidermonkey/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.transform:TransformTask
+loader: taskgraph.task.transform:load_tasks
transforms:
- taskgraph.transforms.build_attrs:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
job-defaults:
treeherder:
--- a/taskcluster/ci/static-analysis/kind.yml
+++ b/taskcluster/ci/static-analysis/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.transform:TransformTask
+loader: taskgraph.task.transform:load_tasks
transforms:
- taskgraph.transforms.build_attrs:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
job-defaults:
index:
--- a/taskcluster/ci/test/kind.yml
+++ b/taskcluster/ci/test/kind.yml
@@ -1,9 +1,9 @@
-implementation: taskgraph.task.test:TestTask
+loader: taskgraph.task.test:load_tasks
kind-dependencies:
- build
transforms:
- taskgraph.transforms.tests:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
--- a/taskcluster/ci/toolchain/kind.yml
+++ b/taskcluster/ci/toolchain/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.transform:TransformTask
+loader: taskgraph.task.transform:load_tasks
transforms:
- taskgraph.transforms.build_attrs:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
jobs-from:
- linux.yml
--- a/taskcluster/ci/upload-symbols/kind.yml
+++ b/taskcluster/ci/upload-symbols/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.post_build:PostBuildTask
+loader: taskgraph.task.post_build:load_tasks
transforms:
- taskgraph.transforms.upload_symbols:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- build
--- a/taskcluster/ci/valgrind/kind.yml
+++ b/taskcluster/ci/valgrind/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-implementation: taskgraph.task.transform:TransformTask
+loader: taskgraph.task.transform:load_tasks
transforms:
- taskgraph.transforms.build_attrs:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
jobs:
linux64-valgrind/opt:
--- a/taskcluster/docs/loading.rst
+++ b/taskcluster/docs/loading.rst
@@ -1,17 +1,34 @@
Loading Tasks
=============
The full task graph generation involves creating tasks for each kind. Kinds
-are ordered to satisfy ``kind-dependencies``, and then the ``implementation``
-specified in ``kind.yml`` is used to load the tasks for that kind.
+are ordered to satisfy ``kind-dependencies``, and then the ``loader`` specified
+in ``kind.yml`` is used to load the tasks for that kind. It should point to
+a Python function like::
+
+ def load_tasks(cls, kind, path, config, parameters, loaded_tasks):
+ pass
+
+The ``kind`` is the name of the kind; the configuration for that kind
+named this class.
-Specifically, the class's ``load_tasks`` class method is called, and returns a
-list of new ``Task`` instances.
+The ``path`` is the path to the configuration directory for the kind. This
+can be used to load extra data, templates, etc.
+
+The ``parameters`` give details on which to base the task generation. See
+:ref:`parameters` for details.
+
+At the time this method is called, all kinds on which this kind depends
+(that is, specified in the ``kind-dependencies`` key in ``config``)
+have already loaded their tasks, and those tasks are available in
+the list ``loaded_tasks``.
+
+The return value is a list of Task instances.
TransformTask
-------------
Most kinds generate their tasks by starting with a set of items describing the
jobs that should be performed and transforming them into task definitions.
This is the familiar ``transforms`` key in ``kind.yml`` and is further
documented in :doc:`transforms`.
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -24,36 +24,34 @@ logger = logging.getLogger(__name__)
class Kind(object):
def __init__(self, name, path, config):
self.name = name
self.path = path
self.config = config
- def _get_impl_class(self):
- # load the class defined by implementation
+ def _get_loader(self):
try:
- impl = self.config['implementation']
+ loader = self.config['loader']
except KeyError:
- raise KeyError("{!r} does not define implementation".format(self.path))
- return find_object(impl)
+ raise KeyError("{!r} does not define `loader`".format(self.path))
+ return find_object(loader)
def load_tasks(self, parameters, loaded_tasks):
- impl_class = self._get_impl_class()
+ loader = self._get_loader()
config = copy.deepcopy(self.config)
if 'parse-commit' in self.config:
parse_commit = find_object(config['parse-commit'])
config['args'] = parse_commit(parameters['message'])
else:
config['args'] = None
- return impl_class.load_tasks(self.name, self.path, config,
- parameters, loaded_tasks)
+ return loader(self.name, self.path, config, parameters, loaded_tasks)
class TaskGraphGenerator(object):
"""
The central controller for taskgraph. This handles all phases of graph
generation. The task is generated from all of the kinds defined in
subdirectories of the generator's root directory.
--- a/taskcluster/taskgraph/task/balrog.py
+++ b/taskcluster/taskgraph/task/balrog.py
@@ -2,27 +2,31 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from . import transform
-class BalrogTask(transform.TransformTask):
+def get_inputs(kind, path, config, params, loaded_tasks):
+ """
+ Load tasks implementing balrog submission jobs. These depend on beetmover
+ jobs and submit the update to balrog as available after the files are moved
+ into place
"""
- A task implementing a balrog submission job. These depend on beetmover jobs
- and submits the update to balrog as available after the files are moved into place
- """
+ if config.get('kind-dependencies', []) != ["beetmover", "beetmover-l10n"]:
+ raise Exception("Balrog kinds must depend on beetmover kinds")
+ for task in loaded_tasks:
+ if not task.attributes.get('nightly'):
+ continue
+ if task.kind not in config.get('kind-dependencies', []):
+ continue
+ beetmover_task = {}
+ beetmover_task['dependent-task'] = task
- @classmethod
- def get_inputs(cls, kind, path, config, params, loaded_tasks):
- if config.get('kind-dependencies', []) != ["beetmover", "beetmover-l10n"]:
- raise Exception("Balrog kinds must depend on beetmover kinds")
- for task in loaded_tasks:
- if not task.attributes.get('nightly'):
- continue
- if task.kind not in config.get('kind-dependencies', []):
- continue
- beetmover_task = {}
- beetmover_task['dependent-task'] = task
+ yield beetmover_task
+
- yield beetmover_task
+def load_tasks(kind, path, config, params, loaded_tasks):
+ return transform.transform_inputs(
+ get_inputs(kind, path, config, params, loaded_tasks),
+ kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/base.py
+++ b/taskcluster/taskgraph/task/base.py
@@ -47,39 +47,16 @@ class Task(object):
def __eq__(self, other):
return self.kind == other.kind and \
self.label == other.label and \
self.attributes == other.attributes and \
self.task == other.task and \
self.task_id == other.task_id and \
self.index_paths == other.index_paths
- @classmethod
- @abc.abstractmethod
- def load_tasks(cls, kind, path, config, parameters, loaded_tasks):
- """
- Load the tasks for a given kind.
-
- The `kind` is the name of the kind; the configuration for that kind
- named this class.
-
- The `path` is the path to the configuration directory for the kind. This
- can be used to load extra data, templates, etc.
-
- The `parameters` give details on which to base the task generation.
- See `taskcluster/docs/parameters.rst` for details.
-
- At the time this method is called, all kinds on which this kind depends
- (that is, specified in the `kind-dependencies` key in `self.config`
- have already loaded their tasks, and those tasks are available in
- the list `loaded_tasks`.
-
- The return value is a list of Task instances.
- """
-
@abc.abstractmethod
def get_dependencies(self, taskgraph):
"""
Get the set of task labels this task depends on, by querying the full
task set, given as `taskgraph`.
Returns a list of (task_label, dependency_name) pairs describing the
dependencies.
--- a/taskcluster/taskgraph/task/beetmover.py
+++ b/taskcluster/taskgraph/task/beetmover.py
@@ -2,27 +2,31 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from . import transform
-class BeetmoverTask(transform.TransformTask):
+def get_inputs(kind, path, config, params, loaded_tasks):
+ """
+ Generate inputs implementing beetmover jobs. These depend on nightly build
+ and signing jobs and transfer the artifacts to S3 after build and signing
+ are completed.
"""
- A task implementing a beetmover job. These depend on nightly build and signing
- jobs and transfer the artifacts to S3 after build and signing are completed.
- """
+ if config.get('kind-dependencies', []) != ["build-signing"] and \
+ config.get('kind-dependencies', []) != ["nightly-l10n-signing"]:
+ raise Exception("Beetmover kinds must depend on builds or signing builds")
+ for task in loaded_tasks:
+ if not task.attributes.get('nightly'):
+ continue
+ if task.kind not in config.get('kind-dependencies'):
+ continue
+ beetmover_task = {'dependent-task': task}
- @classmethod
- def get_inputs(cls, kind, path, config, params, loaded_tasks):
- if config.get('kind-dependencies', []) != ["build-signing"] and \
- config.get('kind-dependencies', []) != ["nightly-l10n-signing"]:
- raise Exception("Beetmover kinds must depend on builds or signing builds")
- for task in loaded_tasks:
- if not task.attributes.get('nightly'):
- continue
- if task.kind not in config.get('kind-dependencies'):
- continue
- beetmover_task = {'dependent-task': task}
+ yield beetmover_task
+
- yield beetmover_task
+def load_tasks(kind, path, config, params, loaded_tasks):
+ return transform.transform_inputs(
+ get_inputs(kind, path, config, params, loaded_tasks),
+ kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/beetmover_checksums.py
+++ b/taskcluster/taskgraph/task/beetmover_checksums.py
@@ -2,27 +2,30 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from . import transform
-class BeetmoverChecksumsTask(transform.TransformTask):
+def get_inputs(kind, path, config, params, loaded_tasks):
"""
A task implementing a beetmover job specific for checksums.These depend on
the checksums signing jobs and transfer the checksums files to S3 after
it's being generated and signed.
"""
+ if config.get('kind-dependencies', []) != ["checksums-signing"]:
+ raise Exception("Beetmover checksums tasks depend on checksums signing tasks")
+ for task in loaded_tasks:
+ if not task.attributes.get('nightly'):
+ continue
+ if task.kind not in config.get('kind-dependencies'):
+ continue
+ beetmover_checksums_task = {'dependent-task': task}
- @classmethod
- def get_inputs(cls, kind, path, config, params, loaded_tasks):
- if config.get('kind-dependencies', []) != ["checksums-signing"]:
- raise Exception("Beetmover checksums tasks depend on checksums signing tasks")
- for task in loaded_tasks:
- if not task.attributes.get('nightly'):
- continue
- if task.kind not in config.get('kind-dependencies'):
- continue
- beetmover_checksums_task = {'dependent-task': task}
+ yield beetmover_checksums_task
+
- yield beetmover_checksums_task
+def load_tasks(kind, path, config, params, loaded_tasks):
+ return transform.transform_inputs(
+ get_inputs(kind, path, config, params, loaded_tasks),
+ kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/checksums_signing.py
+++ b/taskcluster/taskgraph/task/checksums_signing.py
@@ -2,26 +2,30 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from . import transform
-class ChecksumsSigningTask(transform.TransformTask):
+def get_inputs(kind, path, config, params, loaded_tasks):
"""
- A task implementing a checksums signing job. These depend on beetmover jobs
+ Generate tasks implementing checksums signing jobs. These depend on beetmover jobs
and sign the checksums after its being generated by beetmover
"""
- @classmethod
- def get_inputs(cls, kind, path, config, params, loaded_tasks):
- if (config.get('kind-dependencies', []) != ["beetmover", "beetmover-l10n"]):
- raise Exception("Checksums signing tasks must depend on beetmover tasks")
- for task in loaded_tasks:
- if not task.attributes.get('nightly'):
- continue
- if task.kind not in config.get('kind-dependencies'):
- continue
- checksums_signing_task = {'dependent-task': task}
+ if (config.get('kind-dependencies', []) != ["beetmover", "beetmover-l10n"]):
+ raise Exception("Checksums signing tasks must depend on beetmover tasks")
+ for task in loaded_tasks:
+ if not task.attributes.get('nightly'):
+ continue
+ if task.kind not in config.get('kind-dependencies'):
+ continue
+ checksums_signing_task = {'dependent-task': task}
- yield checksums_signing_task
+ yield checksums_signing_task
+
+
+def load_tasks(kind, path, config, params, loaded_tasks):
+ return transform.transform_inputs(
+ get_inputs(kind, path, config, params, loaded_tasks),
+ kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/docker_image.py
+++ b/taskcluster/taskgraph/task/docker_image.py
@@ -16,73 +16,72 @@ from taskgraph.util.docker import (
INDEX_PREFIX,
)
from taskgraph.util.taskcluster import get_artifact_url
from taskgraph.util.templates import Templates
logger = logging.getLogger(__name__)
-class DockerImageTask(base.Task):
+def load_tasks(kind, path, config, params, loaded_tasks):
+ parameters = {
+ 'pushlog_id': params.get('pushlog_id', 0),
+ 'pushdate': params['moz_build_date'],
+ 'pushtime': params['moz_build_date'][8:],
+ 'year': params['moz_build_date'][0:4],
+ 'month': params['moz_build_date'][4:6],
+ 'day': params['moz_build_date'][6:8],
+ 'project': params['project'],
+ 'docker_image': docker_image,
+ 'base_repository': params['base_repository'] or params['head_repository'],
+ 'head_repository': params['head_repository'],
+ 'head_ref': params['head_ref'] or params['head_rev'],
+ 'head_rev': params['head_rev'],
+ 'owner': params['owner'],
+ 'level': params['level'],
+ 'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml'
+ .format(repo=params['head_repository'], rev=params['head_rev']),
+ 'index_image_prefix': INDEX_PREFIX,
+ 'artifact_path': 'public/image.tar.zst',
+ }
- @classmethod
- def load_tasks(cls, kind, path, config, params, loaded_tasks):
- parameters = {
- 'pushlog_id': params.get('pushlog_id', 0),
- 'pushdate': params['moz_build_date'],
- 'pushtime': params['moz_build_date'][8:],
- 'year': params['moz_build_date'][0:4],
- 'month': params['moz_build_date'][4:6],
- 'day': params['moz_build_date'][6:8],
- 'project': params['project'],
- 'docker_image': docker_image,
- 'base_repository': params['base_repository'] or params['head_repository'],
- 'head_repository': params['head_repository'],
- 'head_ref': params['head_ref'] or params['head_rev'],
- 'head_rev': params['head_rev'],
- 'owner': params['owner'],
- 'level': params['level'],
- 'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml'
- .format(repo=params['head_repository'], rev=params['head_rev']),
- 'index_image_prefix': INDEX_PREFIX,
- 'artifact_path': 'public/image.tar.zst',
- }
+ tasks = []
+ templates = Templates(path)
+ for image_name, image_symbol in config['images'].iteritems():
+ context_path = os.path.join('taskcluster', 'docker', image_name)
+ context_hash = generate_context_hash(GECKO, context_path, image_name)
- tasks = []
- templates = Templates(path)
- for image_name, image_symbol in config['images'].iteritems():
- context_path = os.path.join('taskcluster', 'docker', image_name)
- context_hash = generate_context_hash(GECKO, context_path, image_name)
+ image_parameters = dict(parameters)
+ image_parameters['image_name'] = image_name
+ image_parameters['context_hash'] = context_hash
- image_parameters = dict(parameters)
- image_parameters['image_name'] = image_name
- image_parameters['context_hash'] = context_hash
+ image_task = templates.load('image.yml', image_parameters)
+ attributes = {'image_name': image_name}
- image_task = templates.load('image.yml', image_parameters)
- attributes = {'image_name': image_name}
+ # unique symbol for different docker image
+ if 'extra' in image_task['task']:
+ image_task['task']['extra']['treeherder']['symbol'] = image_symbol
- # unique symbol for different docker image
- if 'extra' in image_task['task']:
- image_task['task']['extra']['treeherder']['symbol'] = image_symbol
+ # As an optimization, if the context hash exists for a high level, that image
+ # task ID will be used. The reasoning behind this is that eventually everything ends
+ # up on level 3 at some point if most tasks use this as a common image
+ # for a given context hash, a worker within Taskcluster does not need to contain
+ # the same image per branch.
+ index_paths = ['{}.level-{}.{}.hash.{}'.format(
+ INDEX_PREFIX, level, image_name, context_hash)
+ for level in reversed(range(int(params['level']), 4))]
- # As an optimization, if the context hash exists for a high level, that image
- # task ID will be used. The reasoning behind this is that eventually everything ends
- # up on level 3 at some point if most tasks use this as a common image
- # for a given context hash, a worker within Taskcluster does not need to contain
- # the same image per branch.
- index_paths = ['{}.level-{}.{}.hash.{}'.format(
- INDEX_PREFIX, level, image_name, context_hash)
- for level in reversed(range(int(params['level']), 4))]
+ tasks.append(DockerImageTask(kind, 'build-docker-image-' + image_name,
+ task=image_task['task'], attributes=attributes,
+ index_paths=index_paths))
- tasks.append(cls(kind, 'build-docker-image-' + image_name,
- task=image_task['task'], attributes=attributes,
- index_paths=index_paths))
+ return tasks
- return tasks
+class DockerImageTask(base.Task):
def get_dependencies(self, taskgraph):
return []
def optimize(self, params):
optimized, taskId = super(DockerImageTask, self).optimize(params)
if optimized and taskId:
try:
# Only return the task ID if the artifact exists for the indexed
@@ -102,16 +101,16 @@ class DockerImageTask(base.Task):
@classmethod
def from_json(cls, task_dict):
# Generating index_paths for optimization
imgMeta = task_dict['task']['extra']['imageMeta']
image_name = imgMeta['imageName']
context_hash = imgMeta['contextHash']
index_paths = ['{}.level-{}.{}.hash.{}'.format(
- INDEX_PREFIX, level, image_name, context_hash)
- for level in reversed(range(int(imgMeta['level']), 4))]
+ INDEX_PREFIX, level, image_name, context_hash)
+ for level in reversed(range(int(imgMeta['level']), 4))]
docker_image_task = cls(kind='docker-image',
label=task_dict['label'],
attributes=task_dict['attributes'],
task=task_dict['task'],
index_paths=index_paths)
return docker_image_task
--- a/taskcluster/taskgraph/task/post_build.py
+++ b/taskcluster/taskgraph/task/post_build.py
@@ -8,47 +8,50 @@ import copy
import logging
from . import transform
from ..util.yaml import load_yaml
logger = logging.getLogger(__name__)
-class PostBuildTask(transform.TransformTask):
+def get_inputs(kind, path, config, params, loaded_tasks):
"""
- A task implementing a post-build job. These depend on jobs and perform
- various followup tasks after a build has completed.
+ Generate tasks implementing post-build jobs. These depend on builds and perform
+ various followup tasks after a that build has completed.
The `only-for-build-platforms` kind configuration, if specified, will limit
the build platforms for which a post-build task will be created.
The `job-template' kind configuration points to a yaml file which will
be used to create the input to the transforms. It will have added to it
keys `build-label`, the label for the build task, and `build-platform`, its
platform.
"""
-
- @classmethod
- def get_inputs(cls, kind, path, config, params, loaded_tasks):
- if config.get('kind-dependencies', []) != ["build"]:
- raise Exception("PostBuildTask kinds must depend on builds")
+ if config.get('kind-dependencies', []) != ["build"]:
+ raise Exception("PostBuildTask kinds must depend on builds")
- only_platforms = config.get('only-for-build-platforms')
- prototype = load_yaml(path, config.get('job-template'))
+ only_platforms = config.get('only-for-build-platforms')
+ prototype = load_yaml(path, config.get('job-template'))
- for task in loaded_tasks:
- if task.kind != 'build':
- continue
+ for task in loaded_tasks:
+ if task.kind != 'build':
+ continue
- build_platform = task.attributes.get('build_platform')
- build_type = task.attributes.get('build_type')
- if not build_platform or not build_type:
- continue
- platform = "{}/{}".format(build_platform, build_type)
- if only_platforms and platform not in only_platforms:
- continue
+ build_platform = task.attributes.get('build_platform')
+ build_type = task.attributes.get('build_type')
+ if not build_platform or not build_type:
+ continue
+ platform = "{}/{}".format(build_platform, build_type)
+ if only_platforms and platform not in only_platforms:
+ continue
- post_task = copy.deepcopy(prototype)
- post_task['build-label'] = task.label
- post_task['build-platform'] = platform
- post_task['build-task'] = task
- yield post_task
+ post_task = copy.deepcopy(prototype)
+ post_task['build-label'] = task.label
+ post_task['build-platform'] = platform
+ post_task['build-task'] = task
+ yield post_task
+
+
+def load_tasks(kind, path, config, params, loaded_tasks):
+ return transform.transform_inputs(
+ get_inputs(kind, path, config, params, loaded_tasks),
+ kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/repacks.py
+++ b/taskcluster/taskgraph/task/repacks.py
@@ -2,36 +2,39 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from . import transform
-class RepackTask(transform.TransformTask):
+def get_inputs(kind, path, config, params, loaded_tasks):
"""
- A task implementing a l10n repack job. These may depend on build jobs and
- do a repack of them
+ Generate tasks implementing l10n repack jobs. These may depend on build
+ jobs and do a repack of them
"""
+ only_platforms = config.get('only-for-build-platforms')
- @classmethod
- def get_inputs(cls, kind, path, config, params, loaded_tasks):
- only_platforms = config.get('only-for-build-platforms')
-
- for task in loaded_tasks:
- if task.kind not in config.get('kind-dependencies'):
- continue
+ for task in loaded_tasks:
+ if task.kind not in config.get('kind-dependencies'):
+ continue
- build_platform = task.attributes.get('build_platform')
- build_type = task.attributes.get('build_type')
- if not build_platform or not build_type:
- continue
- platform = "{}/{}".format(build_platform, build_type)
- if only_platforms and platform not in only_platforms:
- continue
+ build_platform = task.attributes.get('build_platform')
+ build_type = task.attributes.get('build_type')
+ if not build_platform or not build_type:
+ continue
+ platform = "{}/{}".format(build_platform, build_type)
+ if only_platforms and platform not in only_platforms:
+ continue
+
+ repack_task = {'dependent-task': task}
- repack_task = {'dependent-task': task}
+ if config.get('job-template'):
+ repack_task.update(config.get('job-template'))
+
+ yield repack_task
- if config.get('job-template'):
- repack_task.update(config.get('job-template'))
- yield repack_task
+def load_tasks(kind, path, config, params, loaded_tasks):
+ return transform.transform_inputs(
+ get_inputs(kind, path, config, params, loaded_tasks),
+ kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/signing.py
+++ b/taskcluster/taskgraph/task/signing.py
@@ -2,27 +2,30 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from . import transform
-class SigningTask(transform.TransformTask):
+def get_inputs(kind, path, config, params, loaded_tasks):
+ """
+ Generate tasks implementing signing jobs. These depend on nightly build
+ jobs and sign the artifacts after a build has completed.
"""
- A task implementing a signing job. These depend on nightly build jobs and
- sign the artifacts after a build has completed.
- """
+ if (config.get('kind-dependencies', []) != ["build"] and
+ config.get('kind-dependencies', []) != ["nightly-l10n"]):
+ raise Exception("Signing kinds must depend on builds or l10n repacks")
+ for task in loaded_tasks:
+ if task.kind not in config.get('kind-dependencies'):
+ continue
+ if not task.attributes.get('nightly'):
+ continue
+ signing_task = {'dependent-task': task}
- @classmethod
- def get_inputs(cls, kind, path, config, params, loaded_tasks):
- if (config.get('kind-dependencies', []) != ["build"] and
- config.get('kind-dependencies', []) != ["nightly-l10n"]):
- raise Exception("Signing kinds must depend on builds or l10n repacks")
- for task in loaded_tasks:
- if task.kind not in config.get('kind-dependencies'):
- continue
- if not task.attributes.get('nightly'):
- continue
- signing_task = {'dependent-task': task}
+ yield signing_task
+
- yield signing_task
+def load_tasks(kind, path, config, params, loaded_tasks):
+ return transform.transform_inputs(
+ get_inputs(kind, path, config, params, loaded_tasks),
+ kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/test.py
+++ b/taskcluster/taskgraph/task/test.py
@@ -8,109 +8,113 @@ import copy
import logging
from . import transform
from ..util.yaml import load_yaml
logger = logging.getLogger(__name__)
-class TestTask(transform.TransformTask):
+def get_inputs(kind, path, config, params, loaded_tasks):
"""
- A task implementing a Gecko test.
+ Generate tasks implementing Gecko tests.
"""
- @classmethod
- def get_inputs(cls, kind, path, config, params, loaded_tasks):
+ # the kind on which this one depends
+ if len(config.get('kind-dependencies', [])) != 1:
+ raise Exception(
+ "TestTask kinds must have exactly one item in kind-dependencies")
+ dep_kind = config['kind-dependencies'][0]
- # the kind on which this one depends
- if len(config.get('kind-dependencies', [])) != 1:
- raise Exception("TestTask kinds must have exactly one item in kind-dependencies")
- dep_kind = config['kind-dependencies'][0]
-
- # get build tasks, keyed by build platform
- builds_by_platform = cls.get_builds_by_platform(dep_kind, loaded_tasks)
+ # get build tasks, keyed by build platform
+ builds_by_platform = get_builds_by_platform(dep_kind, loaded_tasks)
- # get the test platforms for those build tasks
- test_platforms_cfg = load_yaml(path, 'test-platforms.yml')
- test_platforms = cls.get_test_platforms(test_platforms_cfg, builds_by_platform)
+ # get the test platforms for those build tasks
+ test_platforms_cfg = load_yaml(path, 'test-platforms.yml')
+ test_platforms = get_test_platforms(test_platforms_cfg, builds_by_platform)
- # expand the test sets for each of those platforms
- test_sets_cfg = load_yaml(path, 'test-sets.yml')
- test_platforms = cls.expand_tests(test_sets_cfg, test_platforms)
+ # expand the test sets for each of those platforms
+ test_sets_cfg = load_yaml(path, 'test-sets.yml')
+ test_platforms = expand_tests(test_sets_cfg, test_platforms)
- # load the test descriptions
- test_descriptions = load_yaml(path, 'tests.yml')
+ # load the test descriptions
+ test_descriptions = load_yaml(path, 'tests.yml')
- # generate all tests for all test platforms
- for test_platform_name, test_platform in test_platforms.iteritems():
- for test_name in test_platform['test-names']:
- test = copy.deepcopy(test_descriptions[test_name])
- test['build-platform'] = test_platform['build-platform']
- test['test-platform'] = test_platform_name
- test['build-label'] = test_platform['build-label']
- test['test-name'] = test_name
- if test_platform['nightly']:
- test.setdefault('attributes', {})['nightly'] = True
+ # generate all tests for all test platforms
+ for test_platform_name, test_platform in test_platforms.iteritems():
+ for test_name in test_platform['test-names']:
+ test = copy.deepcopy(test_descriptions[test_name])
+ test['build-platform'] = test_platform['build-platform']
+ test['test-platform'] = test_platform_name
+ test['build-label'] = test_platform['build-label']
+ test['test-name'] = test_name
+ if test_platform['nightly']:
+ test.setdefault('attributes', {})['nightly'] = True
- logger.debug("Generating tasks for test {} on platform {}".format(
- test_name, test['test-platform']))
- yield test
+ logger.debug("Generating tasks for test {} on platform {}".format(
+ test_name, test['test-platform']))
+ yield test
+
- @classmethod
- def get_builds_by_platform(cls, dep_kind, loaded_tasks):
- """Find the build tasks on which tests will depend, keyed by
- platform/type. Returns a dictionary mapping build platform to task."""
- builds_by_platform = {}
- for task in loaded_tasks:
- if task.kind != dep_kind:
- continue
+def get_builds_by_platform(dep_kind, loaded_tasks):
+ """Find the build tasks on which tests will depend, keyed by
+ platform/type. Returns a dictionary mapping build platform to task."""
+ builds_by_platform = {}
+ for task in loaded_tasks:
+ if task.kind != dep_kind:
+ continue
- build_platform = task.attributes.get('build_platform')
- build_type = task.attributes.get('build_type')
- if not build_platform or not build_type:
- continue
- platform = "{}/{}".format(build_platform, build_type)
- if platform in builds_by_platform:
- raise Exception("multiple build jobs for " + platform)
- builds_by_platform[platform] = task
- return builds_by_platform
+ build_platform = task.attributes.get('build_platform')
+ build_type = task.attributes.get('build_type')
+ if not build_platform or not build_type:
+ continue
+ platform = "{}/{}".format(build_platform, build_type)
+ if platform in builds_by_platform:
+ raise Exception("multiple build jobs for " + platform)
+ builds_by_platform[platform] = task
+ return builds_by_platform
+
- @classmethod
- def get_test_platforms(cls, test_platforms_cfg, builds_by_platform):
- """Get the test platforms for which test tasks should be generated,
- based on the available build platforms. Returns a dictionary mapping
- test platform to {test-set, build-platform, build-label}."""
- test_platforms = {}
- for test_platform, cfg in test_platforms_cfg.iteritems():
- build_platform = cfg['build-platform']
- if build_platform not in builds_by_platform:
- logger.warning(
- "No build task with platform {}; ignoring test platform {}".format(
- build_platform, test_platform))
- continue
- test_platforms[test_platform] = {
- 'nightly': builds_by_platform[build_platform].attributes.get('nightly', False),
- 'build-platform': build_platform,
- 'build-label': builds_by_platform[build_platform].label,
- }
- test_platforms[test_platform].update(cfg)
- return test_platforms
+def get_test_platforms(test_platforms_cfg, builds_by_platform):
+ """Get the test platforms for which test tasks should be generated,
+ based on the available build platforms. Returns a dictionary mapping
+ test platform to {test-set, build-platform, build-label}."""
+ test_platforms = {}
+ for test_platform, cfg in test_platforms_cfg.iteritems():
+ build_platform = cfg['build-platform']
+ if build_platform not in builds_by_platform:
+ logger.warning(
+ "No build task with platform {}; ignoring test platform {}".format(
+ build_platform, test_platform))
+ continue
+ test_platforms[test_platform] = {
+ 'nightly': builds_by_platform[build_platform].attributes.get('nightly', False),
+ 'build-platform': build_platform,
+ 'build-label': builds_by_platform[build_platform].label,
+ }
+ test_platforms[test_platform].update(cfg)
+ return test_platforms
+
- @classmethod
- def expand_tests(cls, test_sets_cfg, test_platforms):
- """Expand the test sets in `test_platforms` out to sets of test names.
- Returns a dictionary like `get_test_platforms`, with an additional
- `test-names` key for each test platform, containing a set of test
- names."""
- rv = {}
- for test_platform, cfg in test_platforms.iteritems():
- test_sets = cfg['test-sets']
- if not set(test_sets) < set(test_sets_cfg):
- raise Exception(
- "Test sets {} for test platform {} are not defined".format(
- ', '.join(test_sets), test_platform))
- test_names = set()
- for test_set in test_sets:
- test_names.update(test_sets_cfg[test_set])
- rv[test_platform] = cfg.copy()
- rv[test_platform]['test-names'] = test_names
- return rv
+def expand_tests(test_sets_cfg, test_platforms):
+ """Expand the test sets in `test_platforms` out to sets of test names.
+ Returns a dictionary like `get_test_platforms`, with an additional
+ `test-names` key for each test platform, containing a set of test
+ names."""
+ rv = {}
+ for test_platform, cfg in test_platforms.iteritems():
+ test_sets = cfg['test-sets']
+ if not set(test_sets) < set(test_sets_cfg):
+ raise Exception(
+ "Test sets {} for test platform {} are not defined".format(
+ ', '.join(test_sets), test_platform))
+ test_names = set()
+ for test_set in test_sets:
+ test_names.update(test_sets_cfg[test_set])
+ rv[test_platform] = cfg.copy()
+ rv[test_platform]['test-names'] = test_names
+ return rv
+
+
+def load_tasks(kind, path, config, params, loaded_tasks):
+ return transform.transform_inputs(
+ get_inputs(kind, path, config, params, loaded_tasks),
+ kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/transform.py
+++ b/taskcluster/taskgraph/task/transform.py
@@ -14,73 +14,80 @@ from ..util.templates import merge
from ..util.yaml import load_yaml
from ..util.seta import is_low_value_task
from ..transforms.base import TransformSequence, TransformConfig
logger = logging.getLogger(__name__)
+def get_inputs(kind, path, config, params, loaded_tasks):
+ """
+ Get the input elements that will be transformed into tasks. The
+ elements themselves are free-form, and become the input to the first
+ transform.
+
+ By default, this reads jobs from the `jobs` key, or from yaml files
+ named by `jobs-from`. The entities are read from mappings, and the
+ keys to those mappings are added in the `name` key of each entity.
+
+ If there is a `job-defaults` config, then every job is merged with it.
+ This provides a simple way to set default values for all jobs of a
+ kind. More complex defaults should be implemented with custom
+ transforms.
+
+ Other kind implementations can use a different get_inputs function to
+ produce inputs and hand them to `transform_inputs`.
+ """
+ def jobs():
+ defaults = config.get('job-defaults')
+ jobs = config.get('jobs', {}).iteritems()
+ jobs_from = itertools.chain.from_iterable(
+ load_yaml(path, filename).iteritems()
+ for filename in config.get('jobs-from', {}))
+ for name, job in itertools.chain(jobs, jobs_from):
+ if defaults:
+ job = merge(defaults, job)
+ yield name, job
+
+ for name, job in jobs():
+ job['name'] = name
+ logger.debug("Generating tasks for {} {}".format(kind, name))
+ yield job
+
+
+def transform_inputs(inputs, kind, path, config, params, loaded_tasks):
+ """
+ Transform a sequence of inputs according to the transform configuration.
+ """
+ transforms = TransformSequence()
+ for xform_path in config['transforms']:
+ transform = find_object(xform_path)
+ transforms.add(transform)
+
+ # perform the transformations
+ trans_config = TransformConfig(kind, path, config, params)
+ tasks = [TransformTask(kind, t) for t in transforms(trans_config, inputs)]
+ return tasks
+
+
+def load_tasks(kind, path, config, params, loaded_tasks):
+ return transform_inputs(
+ get_inputs(kind, path, config, params, loaded_tasks),
+ kind, path, config, params, loaded_tasks)
+
+
class TransformTask(base.Task):
"""
Tasks of this class are generated by applying transformations to a sequence
of input entities. By default, it gets those inputs from YAML data in the
kind directory, but subclasses may override `get_inputs` to produce them in
some other way.
"""
- @classmethod
- def get_inputs(cls, kind, path, config, params, loaded_tasks):
- """
- Get the input elements that will be transformed into tasks. The
- elements themselves are free-form, and become the input to the first
- transform.
-
- By default, this reads jobs from the `jobs` key, or from yaml files
- named by `jobs-from`. The entities are read from mappings, and the
- keys to those mappings are added in the `name` key of each entity.
-
- If there is a `job-defaults` config, then every job is merged with it.
- This provides a simple way to set default values for all jobs of a
- kind. More complex defaults should be implemented with custom
- transforms.
-
- This method can be overridden in subclasses that need to perform more
- complex calculations to generate the list of inputs.
- """
- def jobs():
- defaults = config.get('job-defaults')
- jobs = config.get('jobs', {}).iteritems()
- jobs_from = itertools.chain.from_iterable(
- load_yaml(path, filename).iteritems()
- for filename in config.get('jobs-from', {}))
- for name, job in itertools.chain(jobs, jobs_from):
- if defaults:
- job = merge(defaults, job)
- yield name, job
-
- for name, job in jobs():
- job['name'] = name
- logger.debug("Generating tasks for {} {}".format(kind, name))
- yield job
-
- @classmethod
- def load_tasks(cls, kind, path, config, params, loaded_tasks):
- inputs = cls.get_inputs(kind, path, config, params, loaded_tasks)
-
- transforms = TransformSequence()
- for xform_path in config['transforms']:
- transform = find_object(xform_path)
- transforms.add(transform)
-
- # perform the transformations
- trans_config = TransformConfig(kind, path, config, params)
- tasks = [cls(kind, t) for t in transforms(trans_config, inputs)]
- return tasks
-
def __init__(self, kind, task):
self.dependencies = task['dependencies']
self.when = task['when']
super(TransformTask, self).__init__(kind, task['label'],
task['attributes'], task['task'],
index_paths=task.get('index-paths'))
def get_dependencies(self, taskgraph):
--- a/taskcluster/taskgraph/test/test_generator.py
+++ b/taskcluster/taskgraph/test/test_generator.py
@@ -1,52 +1,52 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import unittest
-from ..generator import TaskGraphGenerator, Kind
-from .. import graph, target_tasks as target_tasks_mod
-from ..task import base
+from taskgraph.generator import TaskGraphGenerator, Kind
+from taskgraph import graph, target_tasks as target_tasks_mod
+from taskgraph.task import base
from mozunit import main
class FakeTask(base.Task):
def __init__(self, **kwargs):
self.i = kwargs.pop('i')
super(FakeTask, self).__init__(**kwargs)
- @classmethod
- def load_tasks(cls, kind, path, config, parameters, loaded_tasks):
- return [cls(kind=kind,
- label='{}-t-{}'.format(kind, i),
- attributes={'_tasknum': str(i)},
- task={},
- i=i)
- for i in range(3)]
-
def get_dependencies(self, full_task_set):
i = self.i
if i > 0:
return [('{}-t-{}'.format(self.kind, i - 1), 'prev')]
else:
return []
def optimize(self, params):
return False, None
+def fake_loader(kind, path, config, parameters, loaded_tasks):
+ return [FakeTask(kind=kind,
+ label='{}-t-{}'.format(kind, i),
+ attributes={'_tasknum': str(i)},
+ task={},
+ i=i)
+ for i in range(3)]
+
+
class FakeKind(Kind):
- def _get_impl_class(self):
- return FakeTask
+ def _get_loader(self):
+ return fake_loader
def load_tasks(self, parameters, loaded_tasks):
FakeKind.loaded_kinds.append(self.name)
return super(FakeKind, self).load_tasks(parameters, loaded_tasks)
class WithFakeKind(TaskGraphGenerator):
@@ -128,10 +128,11 @@ class TestGenerator(unittest.TestCase):
tid = self.tgg.label_to_taskid
self.assertEqual(
self.tgg.optimized_task_graph.graph,
graph.Graph({tid['_fake-t-0'], tid['_fake-t-1'], tid['_fake-t-2']}, {
(tid['_fake-t-1'], tid['_fake-t-0'], 'prev'),
(tid['_fake-t-2'], tid['_fake-t-1'], 'prev'),
}))
+
if __name__ == '__main__':
main()