Bug 1334167: use run-on-projects to parallel task graph generation; r?Callek
MozReview-Commit-ID: EQMuh4hN9Ya
--- a/.cron.yml
+++ b/.cron.yml
@@ -4,26 +4,26 @@
jobs:
- name: nightly-desktop
job:
type: decision-task
treeherder-symbol: Nd
triggered-by: nightly
target-tasks-method: nightly_linux
- projects:
+ run-on-projects:
- mozilla-central
- date
when:
- {hour: 16, minute: 0}
- name: nightly-android
job:
type: decision-task
treeherder-symbol: Na
triggered-by: nightly
target-tasks-method: nightly_fennec
- projects:
+ run-on-projects:
- mozilla-central
- date
when:
- {hour: 16, minute: 0}
--- a/taskcluster/docs/cron.rst
+++ b/taskcluster/docs/cron.rst
@@ -1,14 +1,24 @@
Periodic Taskgraphs
===================
The cron functionality allows in-tree scheduling of task graphs that run
periodically, instead of on a push.
+Cron.yml
+--------
+
+In the root of the Gecko directory, you will find `.cron.yml`. This defines
+the periodic tasks ("cron jobs") run for Gecko. Each specifies a name, what to
+do, and some parameters to determine when the cron job should occur.
+
+See ``taskcluster/taskgraph/cron/schema.py`` for details on the format and
+meaning of this file.
+
How It Works
------------
The `TaskCluster Hooks Service <https://tools.taskcluster.net/hooks>`_ has a
hook configured for each repository supporting periodic task graphs. The hook
runs every 15 minutes, and the resulting task is referred to as a "cron task".
That cron task runs `./mach taskgraph cron` in a checkout of the Gecko source
tree.
--- a/taskcluster/taskgraph/cron/__init__.py
+++ b/taskcluster/taskgraph/cron/__init__.py
@@ -16,16 +16,17 @@ import requests
import yaml
from . import decision, schema
from .util import (
match_utc,
calculate_head_rev
)
from ..create import create_task
+from taskgraph.util.attributes import match_run_on_projects
# Functions to handle each `job.type` in `.cron.yml`. These are called with
# the contents of the `job` property from `.cron.yml` and should return a
# sequence of (taskId, task) tuples which will subsequently be fed to
# createTask.
JOB_TYPES = {
'decision-task': decision.run_decision_task,
}
@@ -45,19 +46,19 @@ def get_session():
def load_jobs():
with open(os.path.join(GECKO, '.cron.yml'), 'rb') as f:
cron_yml = yaml.load(f)
schema.validate(cron_yml)
return {j['name']: j for j in cron_yml['jobs']}
def should_run(job, params):
- if 'projects' in job:
- if not any(p == params['project'] for p in job['projects']):
- return False
+ run_on_projects = job.get('run-on-projects', ['all'])
+ if not match_run_on_projects(params['project'], run_on_projects):
+ return False
if not any(match_utc(params, hour=sched.get('hour'), minute=sched.get('minute'))
for sched in job.get('when', [])):
return False
return True
def run_job(job_name, job, params):
params['job_name'] = job_name
--- a/taskcluster/taskgraph/cron/schema.py
+++ b/taskcluster/taskgraph/cron/schema.py
@@ -33,18 +33,20 @@ cron_yml_schema = Schema({
# --target-tasks-method './mach taskgraph decision' argument
'target-tasks-method': basestring,
}),
# when to run it
# Optional set of projects on which this job should run; if omitted, this will
- # run on all projects for which cron tasks are set up
- 'projects': [basestring],
+ # run on all projects for which cron tasks are set up. This works just like the
+ # `run_on_projects` attribute, where strings like "release" and "integration" are
+ # expanded to cover multiple repositories. (taskcluster/docs/attributes.rst)
+ 'run-on-projects': [basestring],
# Array of times at which this task should run. These *must* be a multiple of
# 15 minutes, the minimum scheduling interval.
'when': [{'hour': int, 'minute': All(int, even_15_minutes)}],
}],
})
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -1,28 +1,17 @@
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph import try_option_syntax
-
-INTEGRATION_PROJECTS = set([
- 'mozilla-inbound',
- 'autoland',
-])
-
-RELEASE_PROJECTS = set([
- 'mozilla-central',
- 'mozilla-aurora',
- 'mozilla-beta',
- 'mozilla-release',
-])
+from taskgraph.util.attributes import match_run_on_projects
_target_task_methods = {}
def _target_task(name):
def wrap(func):
_target_task_methods[name] = func
return func
@@ -64,26 +53,17 @@ def target_tasks_try_option_syntax(full_
@_target_task('default')
def target_tasks_default(full_task_graph, parameters):
"""Target the tasks which have indicated they should be run on this project
via the `run_on_projects` attributes."""
def filter(task):
run_on_projects = set(task.attributes.get('run_on_projects', []))
- if 'all' in run_on_projects:
- return True
- project = parameters['project']
- if 'integration' in run_on_projects:
- if project in INTEGRATION_PROJECTS:
- return True
- if 'release' in run_on_projects:
- if project in RELEASE_PROJECTS:
- return True
- return project in run_on_projects
+ return match_run_on_projects(parameters['project'], run_on_projects)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('ash_tasks')
def target_tasks_ash(full_task_graph, parameters):
"""Target tasks that only run on the ash branch."""
def filter(task):
platform = task.attributes.get('build_platform')
--- a/taskcluster/taskgraph/test/test_util_attributes.py
+++ b/taskcluster/taskgraph/test/test_util_attributes.py
@@ -1,16 +1,19 @@
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import unittest
-from taskgraph.util.attributes import attrmatch
+from taskgraph.util.attributes import (
+ attrmatch,
+ match_run_on_projects,
+)
class Attrmatch(unittest.TestCase):
def test_trivial_match(self):
"""Given no conditions, anything matches"""
self.assertTrue(attrmatch({}))
@@ -38,8 +41,55 @@ class Attrmatch(unittest.TestCase):
self.assertTrue(attrmatch({'att': 10}, att=even))
self.assertFalse(attrmatch({'att': 11}, att=even))
def test_all_matches_required(self):
"""If only one attribute does not match, the result is False"""
self.assertFalse(attrmatch({'a': 1}, a=1, b=2, c=3))
self.assertFalse(attrmatch({'a': 1, 'b': 2}, a=1, b=2, c=3))
self.assertTrue(attrmatch({'a': 1, 'b': 2, 'c': 3}, a=1, b=2, c=3))
+
+
+class MatchRunOnProjects(unittest.TestCase):
+
+ def test_empty(self):
+ self.assertFalse(match_run_on_projects('try', []))
+
+ def test_all(self):
+ self.assertTrue(match_run_on_projects('try', ['all']))
+ self.assertTrue(match_run_on_projects('larch', ['all']))
+ self.assertTrue(match_run_on_projects('autoland', ['all']))
+ self.assertTrue(match_run_on_projects('mozilla-inbound', ['all']))
+ self.assertTrue(match_run_on_projects('mozilla-central', ['all']))
+ self.assertTrue(match_run_on_projects('mozilla-aurora', ['all']))
+ self.assertTrue(match_run_on_projects('mozilla-beta', ['all']))
+ self.assertTrue(match_run_on_projects('mozilla-release', ['all']))
+
+ def test_release(self):
+ self.assertFalse(match_run_on_projects('try', ['release']))
+ self.assertFalse(match_run_on_projects('larch', ['release']))
+ self.assertFalse(match_run_on_projects('autoland', ['release']))
+ self.assertFalse(match_run_on_projects('mozilla-inbound', ['release']))
+ self.assertTrue(match_run_on_projects('mozilla-central', ['release']))
+ self.assertTrue(match_run_on_projects('mozilla-aurora', ['release']))
+ self.assertTrue(match_run_on_projects('mozilla-beta', ['release']))
+ self.assertTrue(match_run_on_projects('mozilla-release', ['release']))
+
+ def test_integration(self):
+ self.assertFalse(match_run_on_projects('try', ['integration']))
+ self.assertFalse(match_run_on_projects('larch', ['integration']))
+ self.assertTrue(match_run_on_projects('autoland', ['integration']))
+ self.assertTrue(match_run_on_projects('mozilla-inbound', ['integration']))
+ self.assertFalse(match_run_on_projects('mozilla-central', ['integration']))
+ self.assertFalse(match_run_on_projects('mozilla-aurora', ['integration']))
+ self.assertFalse(match_run_on_projects('mozilla-beta', ['integration']))
+ self.assertFalse(match_run_on_projects('mozilla-integration', ['integration']))
+
+ def test_combo(self):
+ self.assertTrue(match_run_on_projects('try', ['release', 'try', 'date']))
+ self.assertFalse(match_run_on_projects('larch', ['release', 'try', 'date']))
+ self.assertTrue(match_run_on_projects('date', ['release', 'try', 'date']))
+ self.assertFalse(match_run_on_projects('autoland', ['release', 'try', 'date']))
+ self.assertFalse(match_run_on_projects('mozilla-inbound', ['release', 'try', 'date']))
+ self.assertTrue(match_run_on_projects('mozilla-central', ['release', 'try', 'date']))
+ self.assertTrue(match_run_on_projects('mozilla-aurora', ['release', 'try', 'date']))
+ self.assertTrue(match_run_on_projects('mozilla-beta', ['release', 'try', 'date']))
+ self.assertTrue(match_run_on_projects('mozilla-release', ['release', 'try', 'date']))
--- a/taskcluster/taskgraph/util/attributes.py
+++ b/taskcluster/taskgraph/util/attributes.py
@@ -1,12 +1,24 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+INTEGRATION_PROJECTS = set([
+ 'mozilla-inbound',
+ 'autoland',
+])
+
+RELEASE_PROJECTS = set([
+ 'mozilla-central',
+ 'mozilla-aurora',
+ 'mozilla-beta',
+ 'mozilla-release',
+])
+
def attrmatch(attributes, **kwargs):
"""Determine whether the given set of task attributes matches. The
conditions are given as keyword arguments, where each keyword names an
attribute. The keyword value can be a literal, a set, or a callable. A
literal must match the attribute exactly. Given a set, the attribute value
must be in the set. A callable is called with the attribute value. If an
attribute is specified as a keyword argument but not present in the
@@ -19,8 +31,23 @@ def attrmatch(attributes, **kwargs):
if attval not in kwval:
return False
elif callable(kwval):
if not kwval(attval):
return False
elif kwval != attributes[kwkey]:
return False
return True
+
+
+def match_run_on_projects(project, run_on_projects):
+ """Determine whether the given project is included in the `run-on-projects`
+ parameter, applying expansions for things like "integration" mentioned in
+ the attribute documentation."""
+ if 'all' in run_on_projects:
+ return True
+ if 'integration' in run_on_projects:
+ if project in INTEGRATION_PROJECTS:
+ return True
+ if 'release' in run_on_projects:
+ if project in RELEASE_PROJECTS:
+ return True
+ return project in run_on_projects