--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -151,91 +151,16 @@ class MachCommands(MachCommandBase):
import taskgraph.decision
try:
self.setup_logging()
return taskgraph.decision.taskgraph_decision(options)
except Exception:
traceback.print_exc()
sys.exit(1)
- @SubCommand('taskgraph', 'action-task',
- description="Run the add-tasks task. DEPRECATED! Use 'add-tasks' instead.")
- @CommandArgument('--root', '-r',
- default='taskcluster/ci',
- help="root of the taskgraph definition relative to topsrcdir")
- @CommandArgument('--decision-id',
- required=True,
- help="Decision Task ID of the reference decision task")
- @CommandArgument('--task-labels',
- required=True,
- help='Comma separated list of task labels to be scheduled')
- def taskgraph_action(self, **options):
- """Run the action task: Generates a task graph using the set of labels
- provided in the task-labels parameter. It uses the full-task file of
- the gecko decision task."""
-
- import taskgraph.action
- try:
- self.setup_logging()
- return taskgraph.action.add_tasks(options['decision_id'],
- options['task_labels'].split(','))
- except Exception:
- traceback.print_exc()
- sys.exit(1)
-
- @SubCommand('taskgraph', 'add-tasks',
- description="Run the add-tasks task")
- @CommandArgument('--root', '-r',
- default='taskcluster/ci',
- help="root of the taskgraph definition relative to topsrcdir")
- @CommandArgument('--decision-id',
- required=True,
- help="Decision Task ID of the reference decision task")
- @CommandArgument('--task-labels',
- required=True,
- help='Comma separated list of task labels to be scheduled')
- def taskgraph_add_tasks(self, **options):
- """Run the action task: Generates a task graph using the set of labels
- provided in the task-labels parameter. It uses the full-task file of
- the gecko decision task."""
-
- import taskgraph.action
- try:
- self.setup_logging()
- return taskgraph.action.add_tasks(options['decision_id'],
- options['task_labels'].split(','))
- except Exception:
- traceback.print_exc()
- sys.exit(1)
-
- @SubCommand('taskgraph', 'backfill',
- description="Run the backfill task")
- @CommandArgument('--root', '-r',
- default='taskcluster/ci',
- help="root of the taskgraph definition relative to topsrcdir")
- @CommandArgument('--project',
- required=True,
- help="Project of the jobs that need to be backfilled.")
- @CommandArgument('--job-id',
- required=True,
- help="Id of the job to be backfilled.")
- def taskgraph_backfill(self, **options):
- """Run the backfill task: Given a job in a project, it will
- add that job type to any previous revisions in treeherder
- until either a hard limit is met or a green version of that
- job is found."""
-
- import taskgraph.action
- try:
- self.setup_logging()
- return taskgraph.action.backfill(options['project'], options['job_id'])
- except Exception:
- traceback.print_exc()
- sys.exit(1)
-
@SubCommand('taskgraph', 'cron',
description="Run the cron task")
@CommandArgument('--base-repository',
required=True,
help='URL for "base" repository to clone')
@CommandArgument('--head-repository',
required=True,
help='URL for "head" repository to fetch')
@@ -262,40 +187,16 @@ class MachCommands(MachCommandBase):
import taskgraph.cron
try:
self.setup_logging()
return taskgraph.cron.taskgraph_cron(options)
except Exception:
traceback.print_exc()
sys.exit(1)
- @SubCommand('taskgraph', 'add-talos',
- description="Run the add-talos task")
- @CommandArgument('--root', '-r',
- default='taskcluster/ci',
- help="root of the taskgraph definition relative to topsrcdir")
- @CommandArgument('--decision-task-id',
- required=True,
- help="Id of the decision task that is part of the push to be talos'd")
- @CommandArgument('--times',
- required=False,
- default=1,
- type=int,
- help="Number of times to add each job.")
- def taskgraph_add_talos(self, **options):
- """Add all talos jobs for a push."""
-
- import taskgraph.action
- try:
- self.setup_logging()
- return taskgraph.action.add_talos(options['decision_task_id'], options['times'])
- except Exception:
- traceback.print_exc()
- sys.exit(1)
-
@SubCommand('taskgraph', 'action-callback',
description='Run action callback used by action tasks')
def action_callback(self, **options):
import taskgraph.actions
try:
self.setup_logging()
task_group_id = os.environ.get('ACTION_TASK_GROUP_ID', None)
deleted file mode 100644
--- a/taskcluster/taskgraph/action.py
+++ /dev/null
@@ -1,146 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import logging
-import requests
-
-from .create import create_tasks
-from .decision import write_artifact
-from .optimize import optimize_task_graph
-from .taskgraph import TaskGraph
-from .util.taskcluster import get_artifact
-
-
-logger = logging.getLogger(__name__)
-TREEHERDER_URL = "https://treeherder.mozilla.org/api"
-
-# We set this to 5 for now because this is what SETA sets the
-# count to for every repository/job. If this is ever changed,
-# we'll need to have an API added to Treeherder to let us query
-# how far back we should look.
-MAX_BACKFILL_RESULTSETS = 5
-
-
-def add_tasks(decision_task_id, task_labels, prefix=''):
- """
- Run the add-tasks task. This function implements `mach taskgraph add-tasks`,
- and is responsible for
-
- * creating taskgraph of tasks asked for in parameters with respect to
- a given gecko decision task and schedule these jobs.
- """
- # read in the full graph for reference
- full_task_json = get_artifact(decision_task_id, "public/full-task-graph.json")
- decision_params = get_artifact(decision_task_id, "public/parameters.yml")
- all_tasks, full_task_graph = TaskGraph.from_json(full_task_json)
-
- target_tasks = set(task_labels)
- target_graph = full_task_graph.graph.transitive_closure(target_tasks)
- target_task_graph = TaskGraph(
- {l: all_tasks[l] for l in target_graph.nodes},
- target_graph)
-
- existing_tasks = get_artifact(decision_task_id, "public/label-to-taskid.json")
-
- # We don't want to optimize target tasks since they have been requested by user
- # Hence we put `target_tasks under` `do_not_optimize`
- optimized_graph, label_to_taskid = optimize_task_graph(target_task_graph=target_task_graph,
- params=decision_params,
- do_not_optimize=target_tasks,
- existing_tasks=existing_tasks)
-
- # write out the optimized task graph to describe what will actually happen,
- # and the map of labels to taskids
- write_artifact('{}task-graph.json'.format(prefix), optimized_graph.to_json())
- write_artifact('{}label-to-taskid.json'.format(prefix), label_to_taskid)
- # actually create the graph
- create_tasks(optimized_graph, label_to_taskid, decision_params)
-
-
-def backfill(project, job_id):
- """
- Run the backfill task. This function implements `mach taskgraph backfill-task`,
- and is responsible for
-
- * Scheduling backfill jobs from a given treeherder resultset backwards until either
- a successful job is found or `N` jobs have been scheduled.
- """
- s = requests.Session()
- s.headers.update({"User-Agent": "gecko-intree-backfill-task"})
-
- job = s.get(url="{}/project/{}/jobs/{}/".format(TREEHERDER_URL, project, job_id)).json()
-
- job_type_name = job['job_type_name']
-
- if job['build_system_type'] != 'taskcluster':
- if 'Created by BBB for task' not in job['reason']:
- logger.warning("Invalid build system type! Must be a Taskcluster job. Aborting.")
- return
- task_id = job['reason'].split(' ')[-1]
- task = requests.get("https://queue.taskcluster.net/v1/task/{}".format(task_id)).json()
- job_type_name = task['metadata']['name']
-
- filters = dict((k, job[k]) for k in ("build_platform_id", "platform_option", "job_type_id"))
-
- resultset_url = "{}/project/{}/resultset/".format(TREEHERDER_URL, project)
- params = {"id__lt": job["result_set_id"], "count": MAX_BACKFILL_RESULTSETS}
- results = s.get(url=resultset_url, params=params).json()["results"]
- resultsets = [resultset["id"] for resultset in results]
-
- for decision in load_decisions(s, project, resultsets, filters):
- add_tasks(decision, [job_type_name], '{}-'.format(decision))
-
-
-def add_talos(decision_task_id, times=1):
- """
- Run the add-talos task. This function implements `mach taskgraph add-talos`,
- and is responsible for
-
- * Adding all talos jobs to a push.
- """
- full_task_json = get_artifact(decision_task_id, "public/full-task-graph.json")
- task_labels = [
- label for label, task in full_task_json.iteritems()
- if "talos_try_name" in task['attributes']
- ]
- for time in xrange(times):
- add_tasks(decision_task_id, task_labels, '{}-'.format(time))
-
-
-def load_decisions(s, project, resultsets, filters):
- """
- Given a project, a list of revisions, and a dict of filters, return
- a list of taskIds from decision tasks.
- """
- project_url = "{}/project/{}/jobs/".format(TREEHERDER_URL, project)
- decisions = []
- decision_ids = []
-
- for resultset in resultsets:
- unfiltered = []
- offset = 0
- jobs_per_call = 250
- while True:
- params = {"push_id": resultset, "count": jobs_per_call, "offset": offset}
- results = s.get(url=project_url, params=params).json()["results"]
- unfiltered += results
- if (len(results) < jobs_per_call):
- break
- offset += jobs_per_call
- filtered = [j for j in unfiltered if all([j[k] == filters[k] for k in filters])]
- if filtered and all([j["result"] == "success" for j in filtered]):
- logger.info("Push found with all green jobs for this type. Continuing.")
- break
- decisions += [t for t in unfiltered if t["job_type_name"] == "Gecko Decision Task"]
-
- for decision in decisions:
- job_url = project_url + '{}/'.format(decision["id"])
- taskcluster_metadata = s.get(url=job_url).json()["taskcluster_metadata"]
- decision_ids.append(taskcluster_metadata["task_id"])
-
- return decision_ids
deleted file mode 100644
--- a/taskcluster/taskgraph/action.yml
+++ /dev/null
@@ -1,68 +0,0 @@
----
-created: '{{now}}'
-deadline: '{{#from_now}}1 day{{/from_now}}'
-expires: '{{#from_now}}14 day{{/from_now}}'
-metadata:
- owner: mozilla-taskcluster-maintenance@mozilla.com
- source: 'https://hg.mozilla.org/{{project}}/file/{{head_rev}}/taskcluster/taskgraph/action.yml'
- name: "[tc] Action Task"
- description: Helps schedule new jobs without new push
-
-workerType: "gecko-{{level}}-decision"
-provisionerId: "aws-provisioner-v1"
-schedulerId: "gecko-level-{{level}}"
-
-tags:
- createdForUser: {{owner}}
-
-scopes:
- - {{repo_scope}}
-
-routes:
- - "tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}"
- - "tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}"
-
-payload:
- env:
- GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
- GECKO_HEAD_REPOSITORY: '{{{head_repository}}}'
- GECKO_HEAD_REF: '{{head_ref}}'
- GECKO_HEAD_REV: '{{head_rev}}'
- HG_STORE_PATH: /builds/worker/checkouts/hg-store
-
- cache:
- level-{{level}}-checkouts: /builds/worker/checkouts
-
- features:
- taskclusterProxy: true
-
- # Note: This task is built server side without the context or tooling that
- # exist in tree so we must hard code the version
- image: 'taskcluster/decision:2.0.0@sha256:4039fd878e5700b326d4a636e28c595c053fbcb53909c1db84ad1f513cf644ef'
-
- # Virtually no network or other potentially risky operations happen as part
- # of the task timeout aside from the initial clone. We intentionally have
- # set this to a lower value _all_ decision tasks should use a root
- # repository which is cached.
- maxRunTime: 1800
-
- command:
- - /builds/worker/bin/run-task
- - '--vcs-checkout=/builds/worker/checkouts/gecko'
- - '--'
- - bash
- - -cx
- - >
- cd /builds/worker/checkouts/gecko &&
- ln -s /builds/worker/artifacts artifacts &&
- ./mach --log-no-times taskgraph {{action}} {{action_args}}
-
- artifacts:
- 'public':
- type: 'directory'
- path: '/builds/worker/artifacts'
- expires: '{{#from_now}}7 days{{/from_now}}'
-
-extra:
- treeherder:
- symbol: A
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -3,35 +3,27 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import os
import json
import logging
-import re
import time
import yaml
from .generator import TaskGraphGenerator
from .create import create_tasks
from .parameters import Parameters
from .taskgraph import TaskGraph
from .try_option_syntax import parse_message
from .actions import render_actions_json
from taskgraph.util.partials import populate_release_history
-from . import GECKO
-
-from taskgraph.util.templates import Templates
-from taskgraph.util.time import (
- json_time_from_now,
- current_json_time,
-)
logger = logging.getLogger(__name__)
ARTIFACTS_DIR = 'artifacts'
# For each project, this gives a set of parameters specific to the project.
# See `taskcluster/docs/parameters.rst` for information on parameters.
PER_PROJECT_PARAMETERS = {
@@ -109,19 +101,16 @@ def taskgraph_decision(options):
# create a TaskGraphGenerator instance
tgg = TaskGraphGenerator(
root_dir=options['root'],
parameters=parameters)
# write out the parameters used to generate this graph
write_artifact('parameters.yml', dict(**parameters))
- # write out the yml file for action tasks
- write_artifact('action.yml', get_action_yml(parameters))
-
# write out the public/actions.json file
write_artifact('actions.json', render_actions_json(parameters))
# write out the full graph for reference
full_task_json = tgg.full_task_graph.to_json()
write_artifact('full-task-graph.json', full_task_json)
# this is just a test to check whether the from_json() function is working
@@ -244,29 +233,8 @@ def write_artifact(filename, data):
if filename.endswith('.yml'):
with open(path, 'w') as f:
yaml.safe_dump(data, f, allow_unicode=True, default_flow_style=False)
elif filename.endswith('.json'):
with open(path, 'w') as f:
json.dump(data, f, sort_keys=True, indent=2, separators=(',', ': '))
else:
raise TypeError("Don't know how to write to {}".format(filename))
-
-
-def get_action_yml(parameters):
- # NOTE: when deleting this function, delete taskcluster/taskgraph/util/templates.py too
- templates = Templates(os.path.join(GECKO, "taskcluster/taskgraph"))
- action_parameters = parameters.copy()
-
- match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$', action_parameters['head_repository'])
- if not match:
- raise Exception('Unrecognized head_repository')
- repo_scope = 'assume:repo:{}/{}:*'.format(
- match.group(1), match.group(2))
-
- action_parameters.update({
- "action": "{{action}}",
- "action_args": "{{action_args}}",
- "repo_scope": repo_scope,
- "from_now": json_time_from_now,
- "now": current_json_time()
- })
- return templates.load('action.yml', action_parameters)
--- a/taskcluster/taskgraph/test/test_util_templates.py
+++ b/taskcluster/taskgraph/test/test_util_templates.py
@@ -1,193 +1,21 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import unittest
import mozunit
-import textwrap
from taskgraph.util.templates import (
merge_to,
- merge,
- Templates,
- TemplatesException
+ merge
)
-files = {}
-files['/fixtures/circular.yml'] = textwrap.dedent("""\
- $inherits:
- from: 'circular_ref.yml'
- variables:
- woot: 'inherit'
- """)
-
-files['/fixtures/inherit.yml'] = textwrap.dedent("""\
- $inherits:
- from: 'templates.yml'
- variables:
- woot: 'inherit'
- """)
-
-files['/fixtures/extend_child.yml'] = textwrap.dedent("""\
- list: ['1', '2', '3']
- was_list: ['1']
- obj:
- level: 1
- deeper:
- woot: 'bar'
- list: ['baz']
- """)
-
-files['/fixtures/circular_ref.yml'] = textwrap.dedent("""\
- $inherits:
- from: 'circular.yml'
- """)
-
-files['/fixtures/child_pass.yml'] = textwrap.dedent("""\
- values:
- - {{a}}
- - {{b}}
- - {{c}}
- """)
-
-files['/fixtures/inherit_pass.yml'] = textwrap.dedent("""\
- $inherits:
- from: 'child_pass.yml'
- variables:
- a: 'a'
- b: 'b'
- c: 'c'
- """)
-
-files['/fixtures/deep/2.yml'] = textwrap.dedent("""\
- $inherits:
- from: deep/1.yml
-
- """)
-
-files['/fixtures/deep/3.yml'] = textwrap.dedent("""\
- $inherits:
- from: deep/2.yml
-
- """)
-
-files['/fixtures/deep/4.yml'] = textwrap.dedent("""\
- $inherits:
- from: deep/3.yml
- """)
-
-files['/fixtures/deep/1.yml'] = textwrap.dedent("""\
- variable: {{value}}
- """)
-
-files['/fixtures/simple.yml'] = textwrap.dedent("""\
- is_simple: true
- """)
-
-files['/fixtures/templates.yml'] = textwrap.dedent("""\
- content: 'content'
- variable: '{{woot}}'
- """)
-
-files['/fixtures/extend_parent.yml'] = textwrap.dedent("""\
- $inherits:
- from: 'extend_child.yml'
-
- list: ['4']
- was_list:
- replaced: true
- obj:
- level: 2
- from_parent: true
- deeper:
- list: ['bar']
- """)
-
-
-class TemplatesTest(unittest.TestCase):
-
- def setUp(self):
- self.mocked_open = mozunit.MockedOpen(files)
- self.mocked_open.__enter__()
- self.subject = Templates('/fixtures')
-
- def tearDown(self):
- self.mocked_open.__exit__(None, None, None)
-
- def test_invalid_path(self):
- with self.assertRaisesRegexp(TemplatesException, 'must be a directory'):
- Templates('/zomg/not/a/dir')
-
- def test_no_templates(self):
- content = self.subject.load('simple.yml', {})
- self.assertEquals(content, {
- 'is_simple': True
- })
-
- def test_with_templates(self):
- content = self.subject.load('templates.yml', {
- 'woot': 'bar'
- })
-
- self.assertEquals(content, {
- 'content': 'content',
- 'variable': 'bar'
- })
-
- def test_inheritance(self):
- '''
- The simple single pass inheritance case.
- '''
- content = self.subject.load('inherit.yml', {})
- self.assertEqual(content, {
- 'content': 'content',
- 'variable': 'inherit'
- })
-
- def test_inheritance_implicat_pass(self):
- '''
- Implicitly pass parameters from the child to the ancestor.
- '''
- content = self.subject.load('inherit_pass.yml', {
- 'a': 'overriden'
- })
-
- self.assertEqual(content, {'values': ['overriden', 'b', 'c']})
-
- def test_inheritance_circular(self):
- '''
- Circular reference handling.
- '''
- with self.assertRaisesRegexp(TemplatesException, 'circular'):
- self.subject.load('circular.yml', {})
-
- def test_deep_inheritance(self):
- content = self.subject.load('deep/4.yml', {
- 'value': 'myvalue'
- })
- self.assertEqual(content, {'variable': 'myvalue'})
-
- def test_inheritance_with_simple_extensions(self):
- content = self.subject.load('extend_parent.yml', {})
- self.assertEquals(content, {
- 'list': ['1', '2', '3', '4'],
- 'obj': {
- 'from_parent': True,
- 'deeper': {
- 'woot': 'bar',
- 'list': ['baz', 'bar']
- },
- 'level': 2,
- },
- 'was_list': {'replaced': True}
- })
-
class MergeTest(unittest.TestCase):
def test_merge_to_dicts(self):
source = {'a': 1, 'b': 2}
dest = {'b': '20', 'c': 30}
expected = {
'a': 1, # source only
--- a/taskcluster/taskgraph/util/templates.py
+++ b/taskcluster/taskgraph/util/templates.py
@@ -1,23 +1,16 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
-import os
-
-import pystache
-import yaml
import copy
-# Key used in template inheritance...
-INHERITS_KEY = '$inherits'
-
def merge_to(source, dest):
'''
Merge dict and arrays (override scalar values)
Keys from source override keys from dest, and elements from lists in source
are appended to lists in dest.
@@ -51,105 +44,8 @@ def merge(*objects):
objects later in the list taking precedence. From an inheritance
perspective, "parents" should be listed before "children".
Returns the result without modifying any arguments.
'''
if len(objects) == 1:
return copy.deepcopy(objects[0])
return merge_to(objects[-1], merge(*objects[:-1]))
-
-
-class TemplatesException(Exception):
- pass
-
-
-class Templates():
- '''
- The taskcluster integration makes heavy use of yaml to describe tasks this
- class handles the loading/rendering.
- '''
-
- def __init__(self, root):
- '''
- Initialize the template render.
-
- :param str root: Root path where to load yaml files.
- '''
- if not root:
- raise TemplatesException('Root is required')
-
- if not os.path.isdir(root):
- raise TemplatesException('Root must be a directory')
-
- self.root = root
-
- def _inherits(self, path, obj, properties, seen):
- blueprint = obj.pop(INHERITS_KEY)
- seen.add(path)
-
- # Resolve the path here so we can detect circular references.
- template = self.resolve_path(blueprint.get('from'))
- variables = blueprint.get('variables', {})
-
- # Passed parameters override anything in the task itself.
- for key in properties:
- variables[key] = properties[key]
-
- if not template:
- msg = '"{}" inheritance template missing'.format(path)
- raise TemplatesException(msg)
-
- if template in seen:
- msg = 'Error while handling "{}" in "{}" circular template' + \
- 'inheritance seen \n {}'
- raise TemplatesException(msg.format(path, template, seen))
-
- try:
- out = self.load(template, variables, seen)
- except TemplatesException as e:
- msg = 'Error expanding parent ("{}") of "{}" original error {}'
- raise TemplatesException(msg.format(template, path, str(e)))
-
- # Anything left in obj is merged into final results (and overrides)
- return merge_to(obj, out)
-
- def render(self, path, content, parameters, seen):
- '''
- Renders a given yaml string.
-
- :param str path: used to prevent infinite recursion in inheritance.
- :param str content: Of yaml file.
- :param dict parameters: For mustache templates.
- :param set seen: Seen files (used for inheritance)
- '''
- content = pystache.render(content, parameters)
- result = yaml.load(content)
-
- # In addition to the usual template logic done by mustache we also
- # handle special '$inherit' dict keys.
- if isinstance(result, dict) and INHERITS_KEY in result:
- return self._inherits(path, result, parameters, seen)
-
- return result
-
- def resolve_path(self, path):
- return os.path.join(self.root, path)
-
- def load(self, path, parameters=None, seen=None):
- '''
- Load an render the given yaml path.
-
- :param str path: Location of yaml file to load (relative to root).
- :param dict parameters: To template yaml file with.
- '''
- seen = seen or set()
-
- if not path:
- raise TemplatesException('path is required')
-
- path = self.resolve_path(path)
-
- if not os.path.isfile(path):
- raise TemplatesException('"{}" is not a file'.format(path))
-
- content = open(path).read()
- return self.render(path, content, parameters, seen)