Bug 1286075: fix target task generation, including try; r=Callek draft
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 12 Sep 2016 18:41:58 +0000
changeset 412741 f69e076ee8f41d937ec2ede1dd5dfde3b3f33448
parent 412740 698c2ad78bc6bc706eb3826c6d7f749f8c8d97ba
child 412742 9a313c0a8c98cc248dbbcc635e04d4389315a99f
push id29252
push userdmitchell@mozilla.com
push dateMon, 12 Sep 2016 19:16:39 +0000
reviewersCallek
bugs1286075
milestone51.0a1
Bug 1286075: fix target task generation, including try; r=Callek This uses the run_on_projects attribute introduced earlier for most branches, adjusts the `ash` method to handle that branch as the legacy implementation did, and updates try syntax to match builds as well as tests. In the process, this enables optimizing target tasks, meaning that tasks specifically requested in the try syntax might be optimized. While this is probably not ideal, it matches the existing behavior of try (where `-j all` is the default but all jobs are set to run only when certain files have been modified). This change can be reverted later, in a more advanced version of try. MozReview-Commit-ID: 5FYeUTAsafr
taskcluster/taskgraph/decision.py
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/task/legacy.py
taskcluster/taskgraph/test/test_target_tasks.py
taskcluster/taskgraph/try_option_syntax.py
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -27,29 +27,30 @@ logger = logging.getLogger(__name__)
 ARTIFACTS_DIR = 'artifacts'
 GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
 
 # For each project, this gives a set of parameters specific to the project.
 # See `taskcluster/docs/parameters.rst` for information on parameters.
 PER_PROJECT_PARAMETERS = {
     'try': {
         'target_tasks_method': 'try_option_syntax',
-        # for try, if a task was specified as a target, it should
-        # not be optimized away
-        'optimize_target_tasks': False,
+        # Always perform optimization.  This makes it difficult to use try
+        # pushes to run a task that would otherwise be optimized, but is a
+        # compromise to avoid essentially disabling optimization in try.
+        'optimize_target_tasks': True,
     },
 
     'ash': {
         'target_tasks_method': 'ash_tasks',
         'optimize_target_tasks': True,
     },
 
     # the default parameters are used for projects that do not match above.
     'default': {
-        'target_tasks_method': 'all_builds_and_tests',
+        'target_tasks_method': 'default',
         'optimize_target_tasks': True,
     }
 }
 
 
 def taskgraph_decision(options):
     """
     Run the decision task.  This function implements `mach taskgraph decision`,
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -1,22 +1,27 @@
 # -*- coding: utf-8 -*-
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 from taskgraph import try_option_syntax
-from taskgraph.util.attributes import attrmatch
+
+INTEGRATION_PROJECTS = set([
+    'mozilla-inbound',
+    'autoland',
+])
 
-BUILD_AND_TEST_KINDS = set([
-    'legacy',  # builds
-    'desktop-test',
-    'android-test',
+RELEASE_PROJECTS = set([
+    'mozilla-central',
+    'mozilla-aurora',
+    'mozilla-beta',
+    'mozilla-release',
 ])
 
 _target_task_methods = {}
 
 
 def _target_task(name):
     def wrap(func):
         _target_task_methods[name] = func
@@ -50,42 +55,53 @@ def target_tasks_try_option_syntax(full_
         for l in target_tasks_labels:
             task = full_task_graph[l]
             if 'unittest_suite' in task.attributes:
                 task.attributes['task_duplicates'] = options.trigger_tests
 
     return target_tasks_labels
 
 
-@_target_task('all_builds_and_tests')
-def target_tasks_all_builds_and_tests(full_task_graph, parameters):
-    """Trivially target all build and test tasks.  This is used for
-    branches where we want to build "everyting", but "everything"
-    does not include uninteresting things like docker images"""
+@_target_task('all_builds_and_tests')  # (old name)
+@_target_task('default')
+def target_tasks_default(full_task_graph, parameters):
+    """Target the tasks which have indicated they should be run on this project
+    via the `run_on_projects` attributes."""
     def filter(task):
-        return t.attributes.get('kind') in BUILD_AND_TEST_KINDS
+        run_on_projects = set(t.attributes.get('run_on_projects', []))
+        if 'all' in run_on_projects:
+            return True
+        project = parameters['project']
+        if 'integration' in run_on_projects:
+            if project in INTEGRATION_PROJECTS:
+                return True
+        if 'release' in run_on_projects:
+            if project in RELEASE_PROJECTS:
+                return True
+        return project in run_on_projects
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('ash_tasks')
-def target_tasks_ash_tasks(full_task_graph, parameters):
-    """Special case for builds on ash."""
+def target_tasks_ash(full_task_graph, parameters):
+    """Target tasks that only run on the ash branch."""
     def filter(task):
-        # NOTE: on the ash branch, update taskcluster/ci/desktop-test/tests.yml to
-        # run the M-dt-e10s tasks
-        attrs = t.attributes
-        if attrs.get('kind') not in BUILD_AND_TEST_KINDS:
+        platform = task.attributes.get('build_platform')
+        # only select platforms
+        if platform not in ('linux64', 'linux64-asan', 'linux64-pgo'):
             return False
-        if not attrmatch(attrs, build_platform=set([
-            'linux64',
-            'linux64-asan',
-            'linux64-pgo',
-        ])):
+        # and none of this linux64-asan/debug stuff
+        if platform == 'linux64-asan' and task.attributes['build_type'] == 'debug':
             return False
-        if not attrmatch(attrs, e10s=True):
+        # no non-et10s tests
+        if task.attributes.get('unittest_suite') or task.attributes.get('talos_siute'):
+            if not task.attributes.get('e10s'):
+                return False
+        # don't upload symbols
+        if task.attributes['kind'] == 'upload-symbols':
             return False
         return True
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('nightly_fennec')
 def target_tasks_nightly(full_task_graph, parameters):
     """Select the set of tasks required for a nightly build of fennec. The
--- a/taskcluster/taskgraph/task/legacy.py
+++ b/taskcluster/taskgraph/task/legacy.py
@@ -510,17 +510,21 @@ class LegacyTask(base.Task):
                                             build_parameters['head_rev'],
                                             build_parameters['pushlog_id'])
             decorate_task_json_routes(build_task['task'],
                                       json_routes,
                                       build_parameters)
 
             # Ensure each build graph is valid after construction.
             validate_build_task(build_task)
-            attributes = build_task['attributes'] = {'kind': 'legacy', 'legacy_kind': 'build'}
+            attributes = build_task['attributes'] = {
+                'kind': 'legacy',
+                'legacy_kind': 'build',
+                'run_on_projects': ['all'],
+            }
             if 'build_name' in build:
                 attributes['build_platform'] = build['build_name']
             if 'build_type' in task_extra:
                 attributes['build_type'] = {'dbg': 'debug'}.get(task_extra['build_type'],
                                                                 task_extra['build_type'])
             if build.get('is_job'):
                 attributes['job'] = build['build_name']
                 attributes['legacy_kind'] = 'job'
--- a/taskcluster/taskgraph/test/test_target_tasks.py
+++ b/taskcluster/taskgraph/test/test_target_tasks.py
@@ -25,24 +25,49 @@ class FakeTryOptionSyntax(object):
 
 class TestTargetTasks(unittest.TestCase):
 
     def test_from_parameters(self):
         method = target_tasks.get_method('from_parameters')
         self.assertEqual(method(None, {'target_tasks': ['a', 'b']}),
                          ['a', 'b'])
 
-    def test_all_builds_and_tests(self):
-        method = target_tasks.get_method('all_builds_and_tests')
+    def default_matches(self, run_on_projects, project):
+        method = target_tasks.get_method('default')
         graph = TaskGraph(tasks={
-            'a': TestTask(kind='legacy', label='a'),
-            'b': TestTask(kind='legacy', label='b'),
-            'boring': TestTask(kind='docker', label='boring'),
-        }, graph=Graph(nodes={'a', 'b', 'boring'}, edges=set()))
-        self.assertEqual(sorted(method(graph, {})), sorted(['a', 'b']))
+            'a': TestTask(kind='build', label='a',
+                          attributes={'run_on_projects': run_on_projects}),
+        }, graph=Graph(nodes={'a'}, edges=set()))
+        parameters = {'project': project}
+        return 'a' in method(graph, parameters)
+
+    def test_default_all(self):
+        """run_on_projects=[all] includes release, integration, and other projects"""
+        self.assertTrue(self.default_matches(['all'], 'mozilla-central'))
+        self.assertTrue(self.default_matches(['all'], 'mozilla-inbound'))
+        self.assertTrue(self.default_matches(['all'], 'mozilla-aurora'))
+        self.assertTrue(self.default_matches(['all'], 'baobab'))
+
+    def test_default_integration(self):
+        """run_on_projects=[integration] includes integration projects"""
+        self.assertFalse(self.default_matches(['integration'], 'mozilla-central'))
+        self.assertTrue(self.default_matches(['integration'], 'mozilla-inbound'))
+        self.assertFalse(self.default_matches(['integration'], 'baobab'))
+
+    def test_default_relesae(self):
+        """run_on_projects=[release] includes release projects"""
+        self.assertTrue(self.default_matches(['release'], 'mozilla-central'))
+        self.assertFalse(self.default_matches(['release'], 'mozilla-inbound'))
+        self.assertFalse(self.default_matches(['release'], 'baobab'))
+
+    def test_default_nothing(self):
+        """run_on_projects=[] includes nothing"""
+        self.assertFalse(self.default_matches([], 'mozilla-central'))
+        self.assertFalse(self.default_matches([], 'mozilla-inbound'))
+        self.assertFalse(self.default_matches([], 'baobab'))
 
     def test_try_option_syntax(self):
         tasks = {
             'a': TestTask(kind=None, label='a'),
             'b': TestTask(kind=None, label='b', attributes={'at-at': 'yep'}),
         }
         graph = Graph(nodes=set('ab'), edges=set())
         tg = TaskGraph(tasks, graph)
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -16,16 +16,24 @@ TRY_DELIMITER = 'try:'
 
 # The build type aliases are very cryptic and only used in try flags these are
 # mappings from the single char alias to a longer more recognizable form.
 BUILD_TYPE_ALIASES = {
     'o': 'opt',
     'd': 'debug'
 }
 
+# consider anything in this whitelist of kinds to be governed by -b/-p
+BUILD_KINDS = set([
+])
+
+# anything in this list is governed by -j
+JOB_KINDS = set([
+])
+
 
 # mapping from shortcut name (usable with -u) to a boolean function identifying
 # matching test names
 def alias_prefix(prefix):
     return lambda name: name.startswith(prefix)
 
 
 def alias_contains(infix):
@@ -119,17 +127,18 @@ UNITTEST_PLATFORM_PRETTY_NAMES = {
     # 'Windows 8':  [..TODO..],
     # 'Windows XP': [..TODO..],
     # 'win32': [..TODO..],
     # 'win64': [..TODO..],
 }
 
 # We have a few platforms for which we want to do some "extra" builds, or at
 # least build-ish things.  Sort of.  Anyway, these other things are implemented
-# as different "platforms".
+# as different "platforms".  These do *not* automatically ride along with "-p
+# all"
 RIDEALONG_BUILDS = {
     'android-api-15': [
         'android-api-15-l10n',
     ],
     'linux': [
         'linux-l10n',
     ],
     'linux64': [
@@ -240,17 +249,17 @@ class TryOptionSyntax(object):
             return None
 
         results = []
         for build in platform_arg.split(','):
             results.append(build)
             if build in RIDEALONG_BUILDS:
                 results.extend(RIDEALONG_BUILDS[build])
                 logger.info("platform %s triggers ridealong builds %s" %
-                            (build, RIDEALONG_BUILDS[build]))
+                            (build, ', '.join(RIDEALONG_BUILDS[build])))
 
         return results
 
     def parse_test_option(self, attr_name, test_arg, full_task_graph):
         '''
 
         Parse a unittest (-u) or talos (-t) option, in the context of a full
         task graph containing available `unittest_try_name` or `talos_try_name`
@@ -503,16 +512,31 @@ class TryOptionSyntax(object):
                 return True
             elif attr('legacy_kind') == 'unittest':
                 return match_test(self.unittests, 'unittest_try_name')
             elif attr('legacy_kind') == 'talos':
                 return match_test(self.talos, 'talos_try_name')
             return False
         elif attr('kind') in ('desktop-test', 'android-test'):
             return match_test(self.unittests, 'unittest_try_name')
+        elif attr('kind') in JOB_KINDS:
+            if self.jobs is None:
+                return True
+            if attr('build_platform') in self.jobs:
+                return True
+        elif attr('kind') in BUILD_KINDS:
+            if attr('build_type') not in self.build_types:
+                return False
+            elif self.platforms is None:
+                # for "-p all", look for try in the 'run_on_projects' attribute
+                return set(['try', 'all']) & set(attr('run_on_projects', []))
+            else:
+                if attr('build_platform') not in self.platforms:
+                    return False
+            return True
         else:
             return False
 
     def __str__(self):
         def none_for_all(list):
             if list is None:
                 return '<all>'
             return ', '.join(str(e) for e in list)