Bug 1352477 - taskgraph ignores run-on-project for fennec-nightlies r=aki draft
authorJohan Lorenzo <jlorenzo@mozilla.com>
Mon, 03 Apr 2017 14:53:08 +0200
changeset 555395 c6c8672be923a03a83ee8d833313bcf0d02c789a
parent 554803 38894655c89e68bcd8f45d31a0d3005f2c2b53db
child 622603 784dd484dbfa95309ad067f6ac176cac72302c1e
push id52234
push userbmo:jlorenzo@mozilla.com
push dateTue, 04 Apr 2017 08:50:35 +0000
reviewersaki
bugs1352477
milestone55.0a1
Bug 1352477 - taskgraph ignores run-on-project for fennec-nightlies r=aki MozReview-Commit-ID: 2cdNTV11gdn
taskcluster/ci/nightly-l10n/kind.yml
taskcluster/ci/upload-symbols/job-template.yml
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/transforms/l10n.py
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -23,16 +23,20 @@ job-template:
       by-build-platform:
          default: Localization
          android-api-15-nightly: Single Locale Repack
    locales-file:
       by-build-platform:
          default: browser/locales/all-locales
          android-api-15-nightly: mobile/locales/l10n-changesets.json
    chunks: 6
+   run-on-projects:
+     - mozilla-central
+     - mozilla-aurora
+     - mozilla-beta
    run-time:
       by-build-platform:
          default: 36000
          android-api-15-nightly: 18000
    tooltool:
       by-build-platform:
          default: public
          android-api-15-nightly: internal
--- a/taskcluster/ci/upload-symbols/job-template.yml
+++ b/taskcluster/ci/upload-symbols/job-template.yml
@@ -1,17 +1,16 @@
 label: # see transforms
 description: Upload Symbols
 dependencies: # see transforms
 expires-after: 7 days
 deadline-after: 24 hours
 run-on-projects:
     - try
-    - mozilla-beta
-    - mozilla-release
+    - release
 worker-type: aws-provisioner-v1/gecko-symbol-upload
 worker:
     implementation: docker-worker
     max-run-time: 600
     command: ["/bin/bash", "bin/upload.sh"]
     docker-image: taskclusterprivate/upload_symbols:0.0.4
     env:
         GECKO_HEAD_REPOSITORY: # see transforms
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -18,24 +18,41 @@ def _target_task(name):
     return wrap
 
 
 def get_method(method):
     """Get a target_task_method to pass to a TaskGraphGenerator."""
     return _target_task_methods[method]
 
 
+def filter_on_nightly(task, parameters):
+    return not task.attributes.get('nightly') or parameters.get('include_nightly')
+
+
 def filter_for_project(task, parameters):
     """Filter tasks by project.  Optionally enable nightlies."""
-    if task.attributes.get('nightly') and not parameters.get('include_nightly'):
-        return False
     run_on_projects = set(task.attributes.get('run_on_projects', []))
     return match_run_on_projects(parameters['project'], run_on_projects)
 
 
+def filter_upload_symbols(task, parameters):
+    # Filters out symbols when there are not part of a nightly or a release build
+    # TODO Remove this too specific filter (bug 1353296)
+    return '-upload-symbols' not in task.label or \
+        task.attributes.get('nightly') or \
+        parameters.get('project') in ('mozilla-beta', 'mozilla-release')
+
+
+def standard_filter(task, parameters):
+    return all(
+        filter_func(task, parameters) for filter_func in
+        (filter_on_nightly, filter_for_project, filter_upload_symbols)
+    )
+
+
 @_target_task('try_option_syntax')
 def target_tasks_try_option_syntax(full_task_graph, parameters):
     """Generate a list of target tasks based on try syntax in
     parameters['message'] and, for context, the full task graph."""
     options = try_option_syntax.TryOptionSyntax(parameters['message'], full_task_graph)
     target_tasks_labels = [t.label for t in full_task_graph.tasks.itervalues()
                            if options.task_matches(t.attributes)]
 
@@ -81,17 +98,17 @@ def target_tasks_try_option_syntax(full_
 
 
 @_target_task('default')
 def target_tasks_default(full_task_graph, parameters):
     """Target the tasks which have indicated they should be run on this project
     via the `run_on_projects` attributes."""
 
     return [l for l, t in full_task_graph.tasks.iteritems()
-            if filter_for_project(t, parameters)]
+            if standard_filter(t, parameters)]
 
 
 @_target_task('ash_tasks')
 def target_tasks_ash(full_task_graph, parameters):
     """Target tasks that only run on the ash branch."""
     def filter(task):
         platform = task.attributes.get('build_platform')
         # Early return if platform is None
@@ -179,24 +196,26 @@ def target_tasks_code_coverage(full_task
         platform = task.attributes.get('test_platform')
         if platform not in ('linux64-ccov', 'linux64-jsdcov'):
             return False
         return True
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('nightly_fennec')
-def target_tasks_nightly(full_task_graph, parameters):
+def target_tasks_nightly_fennec(full_task_graph, parameters):
     """Select the set of tasks required for a nightly build of fennec. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
     def filter(task):
         platform = task.attributes.get('build_platform')
         if platform in ('android-api-15-nightly', 'android-x86-nightly'):
-            return task.attributes.get('nightly', False)
+            if not task.attributes.get('nightly', False):
+                return False
+            return filter_for_project(task, parameters)
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('nightly_linux')
 def target_tasks_nightly_linux(full_task_graph, parameters):
     """Select the set of tasks required for a nightly build of linux. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
@@ -209,17 +228,17 @@ def target_tasks_nightly_linux(full_task
 
 @_target_task('mozilla_beta_tasks')
 def target_tasks_mozilla_beta(full_task_graph, parameters):
     """Select the set of tasks required for a promotable beta or release build
     of linux, plus android CI. The candidates build process involves a pipeline
     of builds and signing, but does not include beetmover or balrog jobs."""
 
     def filter(task):
-        if not filter_for_project(task, parameters):
+        if not standard_filter(task, parameters):
             return False
         platform = task.attributes.get('build_platform')
         if platform in ('linux64-pgo', 'linux-pgo', 'win32-pgo', 'win64-pgo',
                         'android-api-15-nightly', 'android-x86-nightly',
                         'win32', 'win64', 'macosx64'):
             return False
         if platform in ('linux64', 'linux'):
             if task.attributes['build_type'] == 'opt':
@@ -243,17 +262,17 @@ def target_tasks_mozilla_release(full_ta
     return target_tasks_mozilla_beta(full_task_graph, parameters)
 
 
 @_target_task('candidates_fennec')
 def target_tasks_candidates_fennec(full_task_graph, parameters):
     """Select the set of tasks required for a candidates build of fennec. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
-    filtered_for_project = target_tasks_nightly(full_task_graph, parameters)
+    filtered_for_project = target_tasks_nightly_fennec(full_task_graph, parameters)
 
     def filter(task):
         if task.kind not in ['balrog']:
             return task.attributes.get('nightly', False)
 
     return [l for l in filtered_for_project if filter(full_task_graph[l])]
 
 
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -74,16 +74,18 @@ l10n_description_schema = Schema({
         Required('job-name'): _by_platform(basestring),
 
         # Type of index
         Optional('type'): basestring,
     },
     # Description of the localized task
     Required('description'): _by_platform(basestring),
 
+    Optional('run-on-projects'): job_description_schema['run-on-projects'],
+
     # task object of the dependent task
     Required('dependent-task'): object,
 
     # worker-type to utilize
     Required('worker-type'): _by_platform(basestring),
 
     # File which contains the used locales
     Required('locales-file'): _by_platform(basestring),
@@ -361,17 +363,17 @@ def make_job_description(config, jobs):
             },
             'attributes': job['attributes'],
             'treeherder': {
                 'kind': 'build',
                 'tier': job['treeherder']['tier'],
                 'symbol': job['treeherder']['symbol'],
                 'platform': job['treeherder']['platform'],
             },
-            'run-on-projects': [],
+            'run-on-projects': job.get('run-on-projects') if job.get('run-on-projects') else [],
         }
 
         if job.get('index'):
             job_description['index'] = {
                 'product': job['index']['product'],
                 'job-name': job['index']['job-name'],
                 'type': job['index'].get('type', 'generic'),
             }