Bug 1352477 - taskgraph ignores run-on-project for fennec-nightlies r=aki
MozReview-Commit-ID: 2cdNTV11gdn
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -23,16 +23,20 @@ job-template:
by-build-platform:
default: Localization
android-api-15-nightly: Single Locale Repack
locales-file:
by-build-platform:
default: browser/locales/all-locales
android-api-15-nightly: mobile/locales/l10n-changesets.json
chunks: 6
+ run-on-projects:
+ - mozilla-central
+ - mozilla-aurora
+ - mozilla-beta
run-time:
by-build-platform:
default: 36000
android-api-15-nightly: 18000
tooltool:
by-build-platform:
default: public
android-api-15-nightly: internal
--- a/taskcluster/ci/upload-symbols/job-template.yml
+++ b/taskcluster/ci/upload-symbols/job-template.yml
@@ -1,17 +1,16 @@
label: # see transforms
description: Upload Symbols
dependencies: # see transforms
expires-after: 7 days
deadline-after: 24 hours
run-on-projects:
- try
- - mozilla-beta
- - mozilla-release
+ - release
worker-type: aws-provisioner-v1/gecko-symbol-upload
worker:
implementation: docker-worker
max-run-time: 600
command: ["/bin/bash", "bin/upload.sh"]
docker-image: taskclusterprivate/upload_symbols:0.0.4
env:
GECKO_HEAD_REPOSITORY: # see transforms
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -18,24 +18,41 @@ def _target_task(name):
return wrap
def get_method(method):
"""Get a target_task_method to pass to a TaskGraphGenerator."""
return _target_task_methods[method]
+def filter_on_nightly(task, parameters):
+ return not task.attributes.get('nightly') or parameters.get('include_nightly')
+
+
def filter_for_project(task, parameters):
"""Filter tasks by project. Optionally enable nightlies."""
- if task.attributes.get('nightly') and not parameters.get('include_nightly'):
- return False
run_on_projects = set(task.attributes.get('run_on_projects', []))
return match_run_on_projects(parameters['project'], run_on_projects)
+def filter_upload_symbols(task, parameters):
+ # Filters out symbols when there are not part of a nightly or a release build
+ # TODO Remove this too specific filter (bug 1353296)
+ return '-upload-symbols' not in task.label or \
+ task.attributes.get('nightly') or \
+ parameters.get('project') in ('mozilla-beta', 'mozilla-release')
+
+
+def standard_filter(task, parameters):
+ return all(
+ filter_func(task, parameters) for filter_func in
+ (filter_on_nightly, filter_for_project, filter_upload_symbols)
+ )
+
+
@_target_task('try_option_syntax')
def target_tasks_try_option_syntax(full_task_graph, parameters):
"""Generate a list of target tasks based on try syntax in
parameters['message'] and, for context, the full task graph."""
options = try_option_syntax.TryOptionSyntax(parameters['message'], full_task_graph)
target_tasks_labels = [t.label for t in full_task_graph.tasks.itervalues()
if options.task_matches(t.attributes)]
@@ -81,17 +98,17 @@ def target_tasks_try_option_syntax(full_
@_target_task('default')
def target_tasks_default(full_task_graph, parameters):
"""Target the tasks which have indicated they should be run on this project
via the `run_on_projects` attributes."""
return [l for l, t in full_task_graph.tasks.iteritems()
- if filter_for_project(t, parameters)]
+ if standard_filter(t, parameters)]
@_target_task('ash_tasks')
def target_tasks_ash(full_task_graph, parameters):
"""Target tasks that only run on the ash branch."""
def filter(task):
platform = task.attributes.get('build_platform')
# Early return if platform is None
@@ -179,24 +196,26 @@ def target_tasks_code_coverage(full_task
platform = task.attributes.get('test_platform')
if platform not in ('linux64-ccov', 'linux64-jsdcov'):
return False
return True
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('nightly_fennec')
-def target_tasks_nightly(full_task_graph, parameters):
+def target_tasks_nightly_fennec(full_task_graph, parameters):
"""Select the set of tasks required for a nightly build of fennec. The
nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
def filter(task):
platform = task.attributes.get('build_platform')
if platform in ('android-api-15-nightly', 'android-x86-nightly'):
- return task.attributes.get('nightly', False)
+ if not task.attributes.get('nightly', False):
+ return False
+ return filter_for_project(task, parameters)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('nightly_linux')
def target_tasks_nightly_linux(full_task_graph, parameters):
"""Select the set of tasks required for a nightly build of linux. The
nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
@@ -209,17 +228,17 @@ def target_tasks_nightly_linux(full_task
@_target_task('mozilla_beta_tasks')
def target_tasks_mozilla_beta(full_task_graph, parameters):
"""Select the set of tasks required for a promotable beta or release build
of linux, plus android CI. The candidates build process involves a pipeline
of builds and signing, but does not include beetmover or balrog jobs."""
def filter(task):
- if not filter_for_project(task, parameters):
+ if not standard_filter(task, parameters):
return False
platform = task.attributes.get('build_platform')
if platform in ('linux64-pgo', 'linux-pgo', 'win32-pgo', 'win64-pgo',
'android-api-15-nightly', 'android-x86-nightly',
'win32', 'win64', 'macosx64'):
return False
if platform in ('linux64', 'linux'):
if task.attributes['build_type'] == 'opt':
@@ -243,17 +262,17 @@ def target_tasks_mozilla_release(full_ta
return target_tasks_mozilla_beta(full_task_graph, parameters)
@_target_task('candidates_fennec')
def target_tasks_candidates_fennec(full_task_graph, parameters):
"""Select the set of tasks required for a candidates build of fennec. The
nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
- filtered_for_project = target_tasks_nightly(full_task_graph, parameters)
+ filtered_for_project = target_tasks_nightly_fennec(full_task_graph, parameters)
def filter(task):
if task.kind not in ['balrog']:
return task.attributes.get('nightly', False)
return [l for l in filtered_for_project if filter(full_task_graph[l])]
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -74,16 +74,18 @@ l10n_description_schema = Schema({
Required('job-name'): _by_platform(basestring),
# Type of index
Optional('type'): basestring,
},
# Description of the localized task
Required('description'): _by_platform(basestring),
+ Optional('run-on-projects'): job_description_schema['run-on-projects'],
+
# task object of the dependent task
Required('dependent-task'): object,
# worker-type to utilize
Required('worker-type'): _by_platform(basestring),
# File which contains the used locales
Required('locales-file'): _by_platform(basestring),
@@ -361,17 +363,17 @@ def make_job_description(config, jobs):
},
'attributes': job['attributes'],
'treeherder': {
'kind': 'build',
'tier': job['treeherder']['tier'],
'symbol': job['treeherder']['symbol'],
'platform': job['treeherder']['platform'],
},
- 'run-on-projects': [],
+ 'run-on-projects': job.get('run-on-projects') if job.get('run-on-projects') else [],
}
if job.get('index'):
job_description['index'] = {
'product': job['index']['product'],
'job-name': job['index']['job-name'],
'type': job['index'].get('type', 'generic'),
}