Bug 1334167: use run-on-projects to parallel task graph generation; r?Callek draft
authorDustin J. Mitchell <dustin@mozilla.com>
Tue, 31 Jan 2017 19:49:18 +0000
changeset 469654 d79cd3fe68afcdd7f4165abe1a59e9d3e29f939d
parent 469653 c94a2a2fa6fbbd12f334d93cc7bf1d856f77f608
child 469655 753a78516ce4678500a11cabe4781496ee989b16
push id43793
push userdmitchell@mozilla.com
push dateThu, 02 Feb 2017 14:41:57 +0000
reviewersCallek
bugs1334167
milestone54.0a1
Bug 1334167: use run-on-projects to parallel task graph generation; r?Callek MozReview-Commit-ID: EQMuh4hN9Ya
.cron.yml
taskcluster/docs/cron.rst
taskcluster/taskgraph/cron/__init__.py
taskcluster/taskgraph/cron/schema.py
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/test/test_util_attributes.py
taskcluster/taskgraph/util/attributes.py
--- a/.cron.yml
+++ b/.cron.yml
@@ -4,26 +4,26 @@
 
 jobs:
     - name: nightly-desktop
       job:
           type: decision-task
           treeherder-symbol: Nd
           triggered-by: nightly
           target-tasks-method: nightly_linux
-      projects:
+      run-on-projects:
           - mozilla-central
           - date
       when:
           - {hour: 16, minute: 0}
 
     - name: nightly-android
       job:
           type: decision-task
           treeherder-symbol: Na
           triggered-by: nightly
           target-tasks-method: nightly_fennec
-      projects:
+      run-on-projects:
           - mozilla-central
           - date
       when:
           - {hour: 16, minute: 0}
 
--- a/taskcluster/docs/cron.rst
+++ b/taskcluster/docs/cron.rst
@@ -1,14 +1,24 @@
 Periodic Taskgraphs
 ===================
 
 The cron functionality allows in-tree scheduling of task graphs that run
 periodically, instead of on a push.
 
+Cron.yml
+--------
+
+In the root of the Gecko directory, you will find `.cron.yml`.  This defines
+the periodic tasks ("cron jobs") run for Gecko.  Each specifies a name, what to
+do, and some parameters to determine when the cron job should occur.
+
+See ``taskcluster/taskgraph/cron/schema.py`` for details on the format and
+meaning of this file.
+
 How It Works
 ------------
 
 The `TaskCluster Hooks Service <https://tools.taskcluster.net/hooks>`_ has a
 hook configured for each repository supporting periodic task graphs.  The hook
 runs every 15 minutes, and the resulting task is referred to as a "cron task".
 That cron task runs `./mach taskgraph cron` in a checkout of the Gecko source
 tree.
--- a/taskcluster/taskgraph/cron/__init__.py
+++ b/taskcluster/taskgraph/cron/__init__.py
@@ -16,16 +16,17 @@ import requests
 import yaml
 
 from . import decision, schema
 from .util import (
     match_utc,
     calculate_head_rev
 )
 from ..create import create_task
+from taskgraph.util.attributes import match_run_on_projects
 
 # Functions to handle each `job.type` in `.cron.yml`.  These are called with
 # the contents of the `job` property from `.cron.yml` and should return a
 # sequence of (taskId, task) tuples which will subsequently be fed to
 # createTask.
 JOB_TYPES = {
     'decision-task': decision.run_decision_task,
 }
@@ -45,19 +46,19 @@ def get_session():
 def load_jobs():
     with open(os.path.join(GECKO, '.cron.yml'), 'rb') as f:
         cron_yml = yaml.load(f)
     schema.validate(cron_yml)
     return {j['name']: j for j in cron_yml['jobs']}
 
 
 def should_run(job, params):
-    if 'projects' in job:
-        if not any(p == params['project'] for p in job['projects']):
-            return False
+    run_on_projects = job.get('run-on-projects', ['all'])
+    if not match_run_on_projects(params['project'], run_on_projects):
+        return False
     if not any(match_utc(params, hour=sched.get('hour'), minute=sched.get('minute'))
                for sched in job.get('when', [])):
         return False
     return True
 
 
 def run_job(job_name, job, params):
     params['job_name'] = job_name
--- a/taskcluster/taskgraph/cron/schema.py
+++ b/taskcluster/taskgraph/cron/schema.py
@@ -33,18 +33,20 @@ cron_yml_schema = Schema({
 
             # --target-tasks-method './mach taskgraph decision' argument
             'target-tasks-method': basestring,
         }),
 
         # when to run it
 
         # Optional set of projects on which this job should run; if omitted, this will
-        # run on all projects for which cron tasks are set up
-        'projects': [basestring],
+        # run on all projects for which cron tasks are set up.  This works just like the
+        # `run_on_projects` attribute, where strings like "release" and "integration" are
+        # expanded to cover multiple repositories.  (taskcluster/docs/attributes.rst)
+        'run-on-projects': [basestring],
 
         # Array of times at which this task should run.  These *must* be a multiple of
         # 15 minutes, the minimum scheduling interval.
         'when': [{'hour': int, 'minute': All(int, even_15_minutes)}],
     }],
 })
 
 
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -1,28 +1,17 @@
 # -*- coding: utf-8 -*-
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 from taskgraph import try_option_syntax
-
-INTEGRATION_PROJECTS = set([
-    'mozilla-inbound',
-    'autoland',
-])
-
-RELEASE_PROJECTS = set([
-    'mozilla-central',
-    'mozilla-aurora',
-    'mozilla-beta',
-    'mozilla-release',
-])
+from taskgraph.util.attributes import match_run_on_projects
 
 _target_task_methods = {}
 
 
 def _target_task(name):
     def wrap(func):
         _target_task_methods[name] = func
         return func
@@ -64,26 +53,17 @@ def target_tasks_try_option_syntax(full_
 
 
 @_target_task('default')
 def target_tasks_default(full_task_graph, parameters):
     """Target the tasks which have indicated they should be run on this project
     via the `run_on_projects` attributes."""
     def filter(task):
         run_on_projects = set(task.attributes.get('run_on_projects', []))
-        if 'all' in run_on_projects:
-            return True
-        project = parameters['project']
-        if 'integration' in run_on_projects:
-            if project in INTEGRATION_PROJECTS:
-                return True
-        if 'release' in run_on_projects:
-            if project in RELEASE_PROJECTS:
-                return True
-        return project in run_on_projects
+        return match_run_on_projects(parameters['project'], run_on_projects)
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('ash_tasks')
 def target_tasks_ash(full_task_graph, parameters):
     """Target tasks that only run on the ash branch."""
     def filter(task):
         platform = task.attributes.get('build_platform')
--- a/taskcluster/taskgraph/test/test_util_attributes.py
+++ b/taskcluster/taskgraph/test/test_util_attributes.py
@@ -1,16 +1,19 @@
 # -*- coding: utf-8 -*-
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import unittest
-from taskgraph.util.attributes import attrmatch
+from taskgraph.util.attributes import (
+    attrmatch,
+    match_run_on_projects,
+)
 
 
 class Attrmatch(unittest.TestCase):
 
     def test_trivial_match(self):
         """Given no conditions, anything matches"""
         self.assertTrue(attrmatch({}))
 
@@ -38,8 +41,55 @@ class Attrmatch(unittest.TestCase):
         self.assertTrue(attrmatch({'att': 10}, att=even))
         self.assertFalse(attrmatch({'att': 11}, att=even))
 
     def test_all_matches_required(self):
         """If only one attribute does not match, the result is False"""
         self.assertFalse(attrmatch({'a': 1}, a=1, b=2, c=3))
         self.assertFalse(attrmatch({'a': 1, 'b': 2}, a=1, b=2, c=3))
         self.assertTrue(attrmatch({'a': 1, 'b': 2, 'c': 3}, a=1, b=2, c=3))
+
+
+class MatchRunOnProjects(unittest.TestCase):
+
+    def test_empty(self):
+        self.assertFalse(match_run_on_projects('try', []))
+
+    def test_all(self):
+        self.assertTrue(match_run_on_projects('try', ['all']))
+        self.assertTrue(match_run_on_projects('larch', ['all']))
+        self.assertTrue(match_run_on_projects('autoland', ['all']))
+        self.assertTrue(match_run_on_projects('mozilla-inbound', ['all']))
+        self.assertTrue(match_run_on_projects('mozilla-central', ['all']))
+        self.assertTrue(match_run_on_projects('mozilla-aurora', ['all']))
+        self.assertTrue(match_run_on_projects('mozilla-beta', ['all']))
+        self.assertTrue(match_run_on_projects('mozilla-release', ['all']))
+
+    def test_release(self):
+        self.assertFalse(match_run_on_projects('try', ['release']))
+        self.assertFalse(match_run_on_projects('larch', ['release']))
+        self.assertFalse(match_run_on_projects('autoland', ['release']))
+        self.assertFalse(match_run_on_projects('mozilla-inbound', ['release']))
+        self.assertTrue(match_run_on_projects('mozilla-central', ['release']))
+        self.assertTrue(match_run_on_projects('mozilla-aurora', ['release']))
+        self.assertTrue(match_run_on_projects('mozilla-beta', ['release']))
+        self.assertTrue(match_run_on_projects('mozilla-release', ['release']))
+
+    def test_integration(self):
+        self.assertFalse(match_run_on_projects('try', ['integration']))
+        self.assertFalse(match_run_on_projects('larch', ['integration']))
+        self.assertTrue(match_run_on_projects('autoland', ['integration']))
+        self.assertTrue(match_run_on_projects('mozilla-inbound', ['integration']))
+        self.assertFalse(match_run_on_projects('mozilla-central', ['integration']))
+        self.assertFalse(match_run_on_projects('mozilla-aurora', ['integration']))
+        self.assertFalse(match_run_on_projects('mozilla-beta', ['integration']))
+        self.assertFalse(match_run_on_projects('mozilla-integration', ['integration']))
+
+    def test_combo(self):
+        self.assertTrue(match_run_on_projects('try', ['release', 'try', 'date']))
+        self.assertFalse(match_run_on_projects('larch', ['release', 'try', 'date']))
+        self.assertTrue(match_run_on_projects('date', ['release', 'try', 'date']))
+        self.assertFalse(match_run_on_projects('autoland', ['release', 'try', 'date']))
+        self.assertFalse(match_run_on_projects('mozilla-inbound', ['release', 'try', 'date']))
+        self.assertTrue(match_run_on_projects('mozilla-central', ['release', 'try', 'date']))
+        self.assertTrue(match_run_on_projects('mozilla-aurora', ['release', 'try', 'date']))
+        self.assertTrue(match_run_on_projects('mozilla-beta', ['release', 'try', 'date']))
+        self.assertTrue(match_run_on_projects('mozilla-release', ['release', 'try', 'date']))
--- a/taskcluster/taskgraph/util/attributes.py
+++ b/taskcluster/taskgraph/util/attributes.py
@@ -1,12 +1,24 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+INTEGRATION_PROJECTS = set([
+    'mozilla-inbound',
+    'autoland',
+])
+
+RELEASE_PROJECTS = set([
+    'mozilla-central',
+    'mozilla-aurora',
+    'mozilla-beta',
+    'mozilla-release',
+])
+
 
 def attrmatch(attributes, **kwargs):
     """Determine whether the given set of task attributes matches.  The
     conditions are given as keyword arguments, where each keyword names an
     attribute.  The keyword value can be a literal, a set, or a callable.  A
     literal must match the attribute exactly.  Given a set, the attribute value
     must be in the set.  A callable is called with the attribute value.  If an
     attribute is specified as a keyword argument but not present in the
@@ -19,8 +31,23 @@ def attrmatch(attributes, **kwargs):
             if attval not in kwval:
                 return False
         elif callable(kwval):
             if not kwval(attval):
                 return False
         elif kwval != attributes[kwkey]:
             return False
     return True
+
+
+def match_run_on_projects(project, run_on_projects):
+    """Determine whether the given project is included in the `run-on-projects`
+    parameter, applying expansions for things like "integration" mentioned in
+    the attribute documentation."""
+    if 'all' in run_on_projects:
+        return True
+    if 'integration' in run_on_projects:
+        if project in INTEGRATION_PROJECTS:
+            return True
+    if 'release' in run_on_projects:
+        if project in RELEASE_PROJECTS:
+            return True
+    return project in run_on_projects