--- a/taskcluster/ci/android-stuff/kind.yml
+++ b/taskcluster/ci/android-stuff/kind.yml
@@ -1,17 +1,17 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# The name of this kind should suggest it's not meant to be permanent. This is
# a temporary place to generate these tasks in Bug 1286075 until they are
# rewritten in a better way.
-loader: taskgraph.task.transform:load_tasks
+loader: taskgraph.task.transform:loader
transforms:
- taskgraph.transforms.android_stuff:transforms
- taskgraph.transforms.task:transforms
jobs:
android-api-15-gradle-dependencies:
description: "Android armv7 API 15+ gradle dependencies"
--- a/taskcluster/ci/artifact-build/kind.yml
+++ b/taskcluster/ci/artifact-build/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.transform:load_tasks
+loader: taskgraph.task.transform:loader
transforms:
- taskgraph.transforms.build_attrs:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
jobs:
linux64-artifact/opt:
--- a/taskcluster/ci/balrog/kind.yml
+++ b/taskcluster/ci/balrog/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.balrog:load_tasks
+loader: taskgraph.task.balrog:loader
transforms:
- taskgraph.transforms.balrog:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- beetmover
- beetmover-l10n
--- a/taskcluster/ci/beetmover-checksums/kind.yml
+++ b/taskcluster/ci/beetmover-checksums/kind.yml
@@ -1,12 +1,12 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.beetmover_checksums:load_tasks
+loader: taskgraph.task.beetmover_checksums:loader
transforms:
- taskgraph.transforms.beetmover_checksums:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- checksums-signing
--- a/taskcluster/ci/beetmover-l10n/kind.yml
+++ b/taskcluster/ci/beetmover-l10n/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.beetmover:load_tasks
+loader: taskgraph.task.beetmover:loader
transforms:
- taskgraph.transforms.beetmover_l10n:transforms
- taskgraph.transforms.beetmover:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- nightly-l10n-signing
--- a/taskcluster/ci/beetmover/kind.yml
+++ b/taskcluster/ci/beetmover/kind.yml
@@ -1,12 +1,12 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.beetmover:load_tasks
+loader: taskgraph.task.beetmover:loader
transforms:
- taskgraph.transforms.beetmover:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- build-signing
--- a/taskcluster/ci/build-signing/kind.yml
+++ b/taskcluster/ci/build-signing/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.signing:load_tasks
+loader: taskgraph.task.signing:loader
transforms:
- taskgraph.transforms.build_signing:transforms
- taskgraph.transforms.signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- build
--- a/taskcluster/ci/build/kind.yml
+++ b/taskcluster/ci/build/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.transform:load_tasks
+loader: taskgraph.task.transform:loader
transforms:
- taskgraph.transforms.build:transforms
- taskgraph.transforms.build_attrs:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
jobs-from:
--- a/taskcluster/ci/checksums-signing/kind.yml
+++ b/taskcluster/ci/checksums-signing/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.checksums_signing:load_tasks
+loader: taskgraph.task.checksums_signing:loader
transforms:
- taskgraph.transforms.checksums_signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- beetmover
- beetmover-l10n
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.transform:load_tasks
+loader: taskgraph.task.transform:loader
transforms:
- taskgraph.transforms.docker_image:transforms
- taskgraph.transforms.task:transforms
# make a task for each docker-image we might want. For the moment, since we
# write artifacts for each, these are whitelisted, but ideally that will change
# (to use subdirectory clones of the proper directory), at which point we can
--- a/taskcluster/ci/hazard/kind.yml
+++ b/taskcluster/ci/hazard/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.transform:load_tasks
+loader: taskgraph.task.transform:loader
transforms:
- taskgraph.transforms.build_attrs:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
job-defaults:
treeherder:
--- a/taskcluster/ci/l10n/kind.yml
+++ b/taskcluster/ci/l10n/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.repacks:load_tasks
+loader: taskgraph.task.repacks:loader
transforms:
- taskgraph.transforms.l10n:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
--- a/taskcluster/ci/nightly-l10n-signing/kind.yml
+++ b/taskcluster/ci/nightly-l10n-signing/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.signing:load_tasks
+loader: taskgraph.task.signing:loader
transforms:
- taskgraph.transforms.nightly_l10n_signing:transforms
- taskgraph.transforms.signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- nightly-l10n
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.repacks:load_tasks
+loader: taskgraph.task.repacks:loader
transforms:
- taskgraph.transforms.l10n:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- build
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.transform:load_tasks
+loader: taskgraph.task.transform:loader
transforms:
- taskgraph.transforms.source_test:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
jobs-from:
- python-tests.yml
--- a/taskcluster/ci/spidermonkey/kind.yml
+++ b/taskcluster/ci/spidermonkey/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.transform:load_tasks
+loader: taskgraph.task.transform:loader
transforms:
- taskgraph.transforms.build_attrs:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
job-defaults:
treeherder:
--- a/taskcluster/ci/static-analysis/kind.yml
+++ b/taskcluster/ci/static-analysis/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.transform:load_tasks
+loader: taskgraph.task.transform:loader
transforms:
- taskgraph.transforms.build_attrs:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
job-defaults:
index:
--- a/taskcluster/ci/test/kind.yml
+++ b/taskcluster/ci/test/kind.yml
@@ -1,9 +1,9 @@
-loader: taskgraph.task.test:load_tasks
+loader: taskgraph.task.test:loader
kind-dependencies:
- build
transforms:
- taskgraph.transforms.tests:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
--- a/taskcluster/ci/toolchain/kind.yml
+++ b/taskcluster/ci/toolchain/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.transform:load_tasks
+loader: taskgraph.task.transform:loader
transforms:
- taskgraph.transforms.build_attrs:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
jobs-from:
- linux.yml
--- a/taskcluster/ci/upload-symbols/kind.yml
+++ b/taskcluster/ci/upload-symbols/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.post_build:load_tasks
+loader: taskgraph.task.post_build:loader
transforms:
- taskgraph.transforms.upload_symbols:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- build
--- a/taskcluster/ci/valgrind/kind.yml
+++ b/taskcluster/ci/valgrind/kind.yml
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.task.transform:load_tasks
+loader: taskgraph.task.transform:loader
transforms:
- taskgraph.transforms.build_attrs:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
jobs:
linux64-valgrind/opt:
--- a/taskcluster/docs/loading.rst
+++ b/taskcluster/docs/loading.rst
@@ -1,56 +1,42 @@
Loading Tasks
=============
The full task graph generation involves creating tasks for each kind. Kinds
are ordered to satisfy ``kind-dependencies``, and then the ``loader`` specified
in ``kind.yml`` is used to load the tasks for that kind. It should point to
a Python function like::
- def load_tasks(cls, kind, path, config, parameters, loaded_tasks):
+ def loader(cls, kind, path, config, parameters, loaded_tasks):
pass
The ``kind`` is the name of the kind; the configuration for that kind
named this class.
The ``path`` is the path to the configuration directory for the kind. This
can be used to load extra data, templates, etc.
The ``parameters`` give details on which to base the task generation. See
:ref:`parameters` for details.
At the time this method is called, all kinds on which this kind depends
(that is, specified in the ``kind-dependencies`` key in ``config``)
have already loaded their tasks, and those tasks are available in
the list ``loaded_tasks``.
-The return value is a list of Task instances.
-
-TransformTask
--------------
-
-Most kinds generate their tasks by starting with a set of items describing the
-jobs that should be performed and transforming them into task definitions.
-This is the familiar ``transforms`` key in ``kind.yml`` and is further
-documented in :doc:`transforms`.
+The return value is a list of inputs to the transforms listed in the kind's
+``transforms`` property. The specific format for the input depends on the first
+transform - whatever it expects. The final transform should be
+``taskgraph.transform.task:transforms``, which produces the output format the
+task-graph generation infrastructure expects.
-Such kinds generally specify their tasks in a common format: either based on a
-``jobs`` property in ``kind.yml``, or on YAML files listed in ``jobs-from``.
-This is handled by the ``TransformTask`` class in
-``taskcluster/taskgraph/task/transform.py``.
-
-For kinds producing tasks that depend on other tasks -- for example, signing
-tasks depend on build tasks -- ``TransformTask`` has a ``get_inputs`` method
-that can be overridden in subclasses and written to return a set of items based
-on tasks that already exist. You can see a nice example of this behavior in
-``taskcluster/taskgraph/task/post_build.py``.
-
-For more information on how all of this works, consult the docstrings and
-comments in the source code itself.
+The ``transforms`` key in ``kind.yml`` is further documented in
+:doc:`transforms`. For more information on how all of this works, consult the
+docstrings and comments in the source code itself.
Try option syntax
-----------------
The ``parse-commit`` optional field specified in ``kind.yml`` links to a
function to parse the command line options in the ``--message`` mach parameter.
Currently, the only valid value is ``taskgraph.try_option_syntax:parse_message``.
The parsed arguments are stored in ``config.config['args']``, it corresponds
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -6,18 +6,20 @@ from __future__ import absolute_import,
import logging
import os
import yaml
import copy
from . import filter_tasks
from .graph import Graph
from .taskgraph import TaskGraph
+from .task.base import Task
from .optimize import optimize_task_graph
from .util.python_path import find_object
+from .transforms.base import TransformSequence, TransformConfig
from .util.verify import (
verify_docs,
verify_task_graph_symbol,
verify_gecko_v2_routes,
)
logger = logging.getLogger(__name__)
@@ -41,17 +43,33 @@ class Kind(object):
config = copy.deepcopy(self.config)
if 'parse-commit' in self.config:
parse_commit = find_object(config['parse-commit'])
config['args'] = parse_commit(parameters['message'])
else:
config['args'] = None
- return loader(self.name, self.path, config, parameters, loaded_tasks)
+ inputs = loader(self.name, self.path, config, parameters, loaded_tasks)
+
+ transforms = TransformSequence()
+ for xform_path in config['transforms']:
+ transform = find_object(xform_path)
+ transforms.add(transform)
+
+ # perform the transformations on the loaded inputs
+ trans_config = TransformConfig(self.name, self.path, config, parameters)
+ tasks = [Task(self.name,
+ label=task_dict['label'],
+ attributes=task_dict['attributes'],
+ task=task_dict['task'],
+ optimizations=task_dict.get('optimizations'),
+ dependencies=task_dict.get('dependencies'))
+ for task_dict in transforms(trans_config, inputs)]
+ return tasks
class TaskGraphGenerator(object):
"""
The central controller for taskgraph. This handles all phases of graph
generation. The task is generated from all of the kinds defined in
subdirectories of the generator's root directory.
--- a/taskcluster/taskgraph/task/balrog.py
+++ b/taskcluster/taskgraph/task/balrog.py
@@ -1,32 +1,24 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
-from . import transform
-
-def get_inputs(kind, path, config, params, loaded_tasks):
+def loader(kind, path, config, params, loaded_tasks):
"""
Load tasks implementing balrog submission jobs. These depend on beetmover
jobs and submit the update to balrog as available after the files are moved
into place
"""
if config.get('kind-dependencies', []) != ["beetmover", "beetmover-l10n"]:
raise Exception("Balrog kinds must depend on beetmover kinds")
for task in loaded_tasks:
if not task.attributes.get('nightly'):
continue
if task.kind not in config.get('kind-dependencies', []):
continue
beetmover_task = {}
beetmover_task['dependent-task'] = task
yield beetmover_task
-
-
-def load_tasks(kind, path, config, params, loaded_tasks):
- return transform.transform_inputs(
- get_inputs(kind, path, config, params, loaded_tasks),
- kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/beetmover.py
+++ b/taskcluster/taskgraph/task/beetmover.py
@@ -1,32 +1,24 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
-from . import transform
-
-def get_inputs(kind, path, config, params, loaded_tasks):
+def loader(kind, path, config, params, loaded_tasks):
"""
Generate inputs implementing beetmover jobs. These depend on nightly build
and signing jobs and transfer the artifacts to S3 after build and signing
are completed.
"""
if config.get('kind-dependencies', []) != ["build-signing"] and \
config.get('kind-dependencies', []) != ["nightly-l10n-signing"]:
raise Exception("Beetmover kinds must depend on builds or signing builds")
for task in loaded_tasks:
if not task.attributes.get('nightly'):
continue
if task.kind not in config.get('kind-dependencies'):
continue
beetmover_task = {'dependent-task': task}
yield beetmover_task
-
-
-def load_tasks(kind, path, config, params, loaded_tasks):
- return transform.transform_inputs(
- get_inputs(kind, path, config, params, loaded_tasks),
- kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/beetmover_checksums.py
+++ b/taskcluster/taskgraph/task/beetmover_checksums.py
@@ -1,31 +1,23 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
-from . import transform
-
-def get_inputs(kind, path, config, params, loaded_tasks):
+def loader(kind, path, config, params, loaded_tasks):
"""
A task implementing a beetmover job specific for checksums.These depend on
the checksums signing jobs and transfer the checksums files to S3 after
it's being generated and signed.
"""
if config.get('kind-dependencies', []) != ["checksums-signing"]:
raise Exception("Beetmover checksums tasks depend on checksums signing tasks")
for task in loaded_tasks:
if not task.attributes.get('nightly'):
continue
if task.kind not in config.get('kind-dependencies'):
continue
beetmover_checksums_task = {'dependent-task': task}
yield beetmover_checksums_task
-
-
-def load_tasks(kind, path, config, params, loaded_tasks):
- return transform.transform_inputs(
- get_inputs(kind, path, config, params, loaded_tasks),
- kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/checksums_signing.py
+++ b/taskcluster/taskgraph/task/checksums_signing.py
@@ -1,31 +1,23 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
-from . import transform
-
-def get_inputs(kind, path, config, params, loaded_tasks):
+def loader(kind, path, config, params, loaded_tasks):
"""
Generate tasks implementing checksums signing jobs. These depend on beetmover jobs
and sign the checksums after its being generated by beetmover
"""
if (config.get('kind-dependencies', []) != ["beetmover", "beetmover-l10n"]):
raise Exception("Checksums signing tasks must depend on beetmover tasks")
for task in loaded_tasks:
if not task.attributes.get('nightly'):
continue
if task.kind not in config.get('kind-dependencies'):
continue
checksums_signing_task = {'dependent-task': task}
yield checksums_signing_task
-
-
-def load_tasks(kind, path, config, params, loaded_tasks):
- return transform.transform_inputs(
- get_inputs(kind, path, config, params, loaded_tasks),
- kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/post_build.py
+++ b/taskcluster/taskgraph/task/post_build.py
@@ -2,23 +2,22 @@
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import copy
import logging
-from . import transform
from ..util.yaml import load_yaml
logger = logging.getLogger(__name__)
-def get_inputs(kind, path, config, params, loaded_tasks):
+def loader(kind, path, config, params, loaded_tasks):
"""
Generate tasks implementing post-build jobs. These depend on builds and perform
various followup tasks after a that build has completed.
The `only-for-build-platforms` kind configuration, if specified, will limit
the build platforms for which a post-build task will be created.
The `job-template' kind configuration points to a yaml file which will
@@ -44,14 +43,8 @@ def get_inputs(kind, path, config, param
if only_platforms and platform not in only_platforms:
continue
post_task = copy.deepcopy(prototype)
post_task['build-label'] = task.label
post_task['build-platform'] = platform
post_task['build-task'] = task
yield post_task
-
-
-def load_tasks(kind, path, config, params, loaded_tasks):
- return transform.transform_inputs(
- get_inputs(kind, path, config, params, loaded_tasks),
- kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/repacks.py
+++ b/taskcluster/taskgraph/task/repacks.py
@@ -1,18 +1,16 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
-from . import transform
-
-def get_inputs(kind, path, config, params, loaded_tasks):
+def loader(kind, path, config, params, loaded_tasks):
"""
Generate tasks implementing l10n repack jobs. These may depend on build
jobs and do a repack of them
"""
only_platforms = config.get('only-for-build-platforms')
for task in loaded_tasks:
if task.kind not in config.get('kind-dependencies'):
@@ -27,14 +25,8 @@ def get_inputs(kind, path, config, param
continue
repack_task = {'dependent-task': task}
if config.get('job-template'):
repack_task.update(config.get('job-template'))
yield repack_task
-
-
-def load_tasks(kind, path, config, params, loaded_tasks):
- return transform.transform_inputs(
- get_inputs(kind, path, config, params, loaded_tasks),
- kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/signing.py
+++ b/taskcluster/taskgraph/task/signing.py
@@ -1,31 +1,23 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
-from . import transform
-
-def get_inputs(kind, path, config, params, loaded_tasks):
+def loader(kind, path, config, params, loaded_tasks):
"""
Generate tasks implementing signing jobs. These depend on nightly build
jobs and sign the artifacts after a build has completed.
"""
if (config.get('kind-dependencies', []) != ["build"] and
config.get('kind-dependencies', []) != ["nightly-l10n"]):
raise Exception("Signing kinds must depend on builds or l10n repacks")
for task in loaded_tasks:
if task.kind not in config.get('kind-dependencies'):
continue
if not task.attributes.get('nightly'):
continue
signing_task = {'dependent-task': task}
yield signing_task
-
-
-def load_tasks(kind, path, config, params, loaded_tasks):
- return transform.transform_inputs(
- get_inputs(kind, path, config, params, loaded_tasks),
- kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/test.py
+++ b/taskcluster/taskgraph/task/test.py
@@ -2,23 +2,22 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import copy
import logging
-from . import transform
from ..util.yaml import load_yaml
logger = logging.getLogger(__name__)
-def get_inputs(kind, path, config, params, loaded_tasks):
+def loader(kind, path, config, params, loaded_tasks):
"""
Generate tasks implementing Gecko tests.
"""
# the kind on which this one depends
if len(config.get('kind-dependencies', [])) != 1:
raise Exception(
"Test kinds must have exactly one item in kind-dependencies")
@@ -107,14 +106,8 @@ def expand_tests(test_sets_cfg, test_pla
"Test sets {} for test platform {} are not defined".format(
', '.join(test_sets), test_platform))
test_names = set()
for test_set in test_sets:
test_names.update(test_sets_cfg[test_set])
rv[test_platform] = cfg.copy()
rv[test_platform]['test-names'] = test_names
return rv
-
-
-def load_tasks(kind, path, config, params, loaded_tasks):
- return transform.transform_inputs(
- get_inputs(kind, path, config, params, loaded_tasks),
- kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/task/transform.py
+++ b/taskcluster/taskgraph/task/transform.py
@@ -2,42 +2,38 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import logging
import itertools
-from . import base
-from ..util.python_path import find_object
from ..util.templates import merge
from ..util.yaml import load_yaml
-from ..transforms.base import TransformSequence, TransformConfig
-
logger = logging.getLogger(__name__)
-def get_inputs(kind, path, config, params, loaded_tasks):
+def loader(kind, path, config, params, loaded_tasks):
"""
- Get the input elements that will be transformed into tasks. The
- elements themselves are free-form, and become the input to the first
- transform.
+ Get the input elements that will be transformed into tasks in a generic
+ way. The elements themselves are free-form, and become the input to the
+ first transform.
By default, this reads jobs from the `jobs` key, or from yaml files
named by `jobs-from`. The entities are read from mappings, and the
keys to those mappings are added in the `name` key of each entity.
If there is a `job-defaults` config, then every job is merged with it.
This provides a simple way to set default values for all jobs of a
kind. More complex defaults should be implemented with custom
transforms.
- Other kind implementations can use a different get_inputs function to
+ Other kind implementations can use a different loader function to
produce inputs and hand them to `transform_inputs`.
"""
def jobs():
defaults = config.get('job-defaults')
jobs = config.get('jobs', {}).iteritems()
jobs_from = itertools.chain.from_iterable(
load_yaml(path, filename).iteritems()
for filename in config.get('jobs-from', {}))
@@ -45,35 +41,8 @@ def get_inputs(kind, path, config, param
if defaults:
job = merge(defaults, job)
yield name, job
for name, job in jobs():
job['name'] = name
logger.debug("Generating tasks for {} {}".format(kind, name))
yield job
-
-
-def transform_inputs(inputs, kind, path, config, params, loaded_tasks):
- """
- Transform a sequence of inputs according to the transform configuration.
- """
- transforms = TransformSequence()
- for xform_path in config['transforms']:
- transform = find_object(xform_path)
- transforms.add(transform)
-
- # perform the transformations
- trans_config = TransformConfig(kind, path, config, params)
- tasks = [base.Task(kind,
- label=task_dict['label'],
- attributes=task_dict['attributes'],
- task=task_dict['task'],
- optimizations=task_dict.get('optimizations'),
- dependencies=task_dict.get('dependencies'))
- for task_dict in transforms(trans_config, inputs)]
- return tasks
-
-
-def load_tasks(kind, path, config, params, loaded_tasks):
- return transform_inputs(
- get_inputs(kind, path, config, params, loaded_tasks),
- kind, path, config, params, loaded_tasks)
--- a/taskcluster/taskgraph/test/test_generator.py
+++ b/taskcluster/taskgraph/test/test_generator.py
@@ -3,64 +3,51 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import unittest
from taskgraph.generator import TaskGraphGenerator, Kind
from taskgraph import graph, target_tasks as target_tasks_mod
-from taskgraph.task import base
from mozunit import main
-class FakeTask(base.Task):
-
- def __init__(self, **kwargs):
- self.i = kwargs.pop('i')
-
- i = self.i
- if i > 0:
- dependencies = {'prev': '{}-t-{}'.format(kwargs['kind'], i - 1)}
- else:
- dependencies = {}
- kwargs['dependencies'] = dependencies
-
- super(FakeTask, self).__init__(**kwargs)
-
- def optimize(self, params):
- return False, None
-
-
def fake_loader(kind, path, config, parameters, loaded_tasks):
- return [FakeTask(kind=kind,
- label='{}-t-{}'.format(kind, i),
- attributes={'_tasknum': str(i)},
- task={},
- i=i)
- for i in range(3)]
+ for i in range(3):
+ dependencies = {}
+ if i >= 1:
+ dependencies['prev'] = '{}-t-{}'.format(kind, i-1)
+ yield {'kind': kind,
+ 'label': '{}-t-{}'.format(kind, i),
+ 'attributes': {'_tasknum': str(i)},
+ 'task': {'i': i},
+ 'dependencies': dependencies}
class FakeKind(Kind):
def _get_loader(self):
return fake_loader
def load_tasks(self, parameters, loaded_tasks):
FakeKind.loaded_kinds.append(self.name)
return super(FakeKind, self).load_tasks(parameters, loaded_tasks)
class WithFakeKind(TaskGraphGenerator):
def _load_kinds(self):
for kind_name, deps in self.parameters['_kinds']:
- yield FakeKind(
- kind_name, '/fake',
- {'kind-dependencies': deps} if deps else {})
+ config = {
+ 'transforms': [],
+ }
+ if deps:
+ config['kind-dependencies'] = deps
+ yield FakeKind(kind_name, '/fake', config)
class TestGenerator(unittest.TestCase):
def maketgg(self, target_tasks=None, kinds=[('_fake', [])]):
FakeKind.loaded_kinds = []
self.target_tasks = target_tasks or []