Bug 1302590 - remove nightly-fennec kind. use build kind for nightlies, r?dustin
* fixes up task-reference error where url was using label instead of name
* add nightly attribute for filtering out nightly tasks at target generation
MozReview-Commit-ID: F8MCUWwlFYD
rename from taskcluster/ci/signing/signing.yml
rename to taskcluster/ci/build-signing/android-signing.yml
--- a/taskcluster/ci/signing/signing.yml
+++ b/taskcluster/ci/build-signing/android-signing.yml
@@ -1,18 +1,26 @@
-task:
- provisionerId: "scriptworker-prov-v1"
- workerType: "signing-linux-v1"
- scopes:
- - "project:releng:signing:cert:dep-signing"
- - "project:releng:signing:format:jar"
- created:
- relative-datestamp: "0 seconds"
- deadline:
- relative-datestamp: "24 hours"
- payload:
- unsignedArtifacts: []
- maxRunTime: 600
- metadata:
- name: "Signing Scriptworker Task"
- description: "Testing the signing scriptworker"
- owner: "amiyaguchi@mozilla.com"
- source: "https://tools.taskcluster.net/task-creator/"
+signing-nightly-fennec:
+ task:
+ provisionerId: "scriptworker-prov-v1"
+ workerType: "signing-linux-v1"
+ scopes:
+ - "project:releng:signing:cert:dep-signing"
+ - "project:releng:signing:format:jar"
+ created:
+ relative-datestamp: "0 seconds"
+ deadline:
+ relative-datestamp: "24 hours"
+ payload:
+ unsignedArtifacts: []
+ maxRunTime: 600
+ metadata:
+ name: "Signing Scriptworker Task"
+ description: "Sign Android Build Tasks"
+ owner: "jlund@mozilla.com"
+ source: "https://tools.taskcluster.net/task-creator/"
+ attributes:
+ nightly: true
+ unsigned-task:
+ label: "build-android-api-15-nightly/opt"
+ artifacts:
+ - "public/build/target.apk"
+ - "public/build/en-US/target.apk"
rename from taskcluster/ci/signing/kind.yml
rename to taskcluster/ci/build-signing/kind.yml
--- a/taskcluster/ci/signing/kind.yml
+++ b/taskcluster/ci/build-signing/kind.yml
@@ -1,8 +1,11 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
implementation: 'taskgraph.task.signing:SigningTask'
-signing_path: '.'
+
+jobs-from:
+ - android-signing.yml
+
kind-dependencies:
- - nightly-fennec
+ - build
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -67,16 +67,44 @@ android-api-15/opt:
config:
- builds/releng_base_android_64_builds.py
- disable_signing.py
- platform_supports_post_upload_to_latest.py
script: "mozharness/scripts/fx_desktop_build.py"
custom-build-variant-cfg: api-15
tooltool-downloads: internal
+android-api-15-nightly/opt:
+ description: "Android 4.0 API15+ Nightly"
+ attributes:
+ nightly: true
+ index:
+ product: mobile
+ job-name: android-api-15-nightly-opt
+ treeherder:
+ platform: android-4-0-armv7-api15/opt
+ symbol: tc(N)
+ tier: 2
+ worker-type: aws-provisioner-v1/android-api-15
+ worker:
+ implementation: docker-worker
+ max-run-time: 7200
+ run:
+ using: mozharness
+ actions: [get-secrets build multi-l10n update]
+ config:
+ - builds/releng_base_android_64_builds.py
+ - disable_signing.py
+ - platform_supports_post_upload_to_latest.py
+ - taskcluster_nightly.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ custom-build-variant-cfg: api-15
+ tooltool-downloads: internal
+ run-on-projects: []
+
android-api-15-gradle/opt:
description: "Android 4.0 API15+ (Gradle) Opt"
index:
product: mobile
job-name: android-api-15-gradle-opt
treeherder:
platform: android-4-0-armv7-api15/opt
symbol: tc(Bg)
deleted file mode 100644
--- a/taskcluster/ci/nightly-fennec/android-api-15-nightly-build.yml
+++ /dev/null
@@ -1,66 +0,0 @@
-$inherits:
- from: 'mobile_base.yml'
- variables:
- build_name: 'android'
- build_type: 'opt'
-task:
- metadata:
- name: '[TC] Android armv7 API 15+'
- description: 'Android armv7 API 15+'
-
- workerType: android-api-15
-
- routes:
- - 'index.buildbot.branches.{{project}}.android-api-15'
- - 'index.buildbot.revisions.{{head_rev}}.{{project}}.android-api-15'
-
- scopes:
- - 'docker-worker:cache:level-{{level}}-{{project}}-build-android-api-15-workspace'
- - 'docker-worker:cache:tooltool-cache'
- - 'docker-worker:relengapi-proxy:tooltool.download.internal'
- - 'docker-worker:relengapi-proxy:tooltool.download.public'
-
- payload:
- cache:
- level-{{level}}-{{project}}-build-android-api-15-workspace: '/home/worker/workspace'
- tooltool-cache: '/home/worker/tooltool-cache'
-
- features:
- relengAPIProxy: true
-
- env:
- # inputs to mozharness
- MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
- # TODO: make these additional configuration files go away
- MOZHARNESS_CONFIG: >
- builds/releng_base_android_64_builds.py
- disable_signing.py
- platform_supports_post_upload_to_latest.py
- taskcluster_nightly.py
- MOZHARNESS_ACTIONS: "get-secrets build multi-l10n update"
- MH_CUSTOM_BUILD_VARIANT_CFG: api-15
- MH_BRANCH: {{project}}
- MH_BUILD_POOL: taskcluster
- TOOLTOOL_CACHE: '/home/worker/tooltool-cache'
-
- command: ["/bin/bash", "bin/build.sh"]
-
- extra:
- treeherderEnv:
- - production
- - staging
- treeherder:
- machine:
- # see https://github.com/mozilla/treeherder/blob/master/ui/js/values.js
- platform: android-4-0-armv7-api15
- groupSymbol: tc
- groupName: Submitted by taskcluster
- symbol: B
- tier: 2
- # Rather then enforcing particular conventions we require that all build
- # tasks provide the "build" extra field to specify where the build and tests
- # files are located.
- locations:
- build: 'public/build/target.apk'
- mozharness: 'public/build/mozharness.zip'
- test_packages: 'public/build/target.test_packages.json'
deleted file mode 100644
--- a/taskcluster/ci/nightly-fennec/build.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-# This is the "base" task which contains the common values all builds must
-# provide.
----
-taskId: {{build_slugid}}
-
-task:
- created:
- relative-datestamp: "0 seconds"
- deadline:
- relative-datestamp: "24 hours"
- metadata:
- source: '{{source}}'
- owner: mozilla-taskcluster-maintenance@mozilla.com
-
- tags:
- createdForUser: {{owner}}
-
- provisionerId: aws-provisioner-v1
- schedulerId: task-graph-scheduler
-
- payload:
- # Two hours is long but covers edge cases (and matches bb based infra)
- maxRunTime: 7200
-
- env:
- # Common environment variables for checking out gecko
- GECKO_BASE_REPOSITORY: '{{base_repository}}'
- GECKO_HEAD_REPOSITORY: '{{head_repository}}'
- GECKO_HEAD_REV: '{{head_rev}}'
- GECKO_HEAD_REF: '{{head_ref}}'
- TOOLTOOL_REPO: 'https://github.com/mozilla/build-tooltool'
- TOOLTOOL_REV: 'master'
- MOZ_BUILD_DATE: '{{pushdate}}'
- MOZ_SCM_LEVEL: '{{level}}'
-
- extra:
- build_product: '{{build_product}}'
- build_name: '{{build_name}}'
- build_type: '{{build_type}}'
- index:
- rank: {{rank}}
- treeherder:
- jobKind: build
- groupSymbol: tc
- groupName: Submitted by taskcluster
- symbol: B
deleted file mode 100644
--- a/taskcluster/ci/nightly-fennec/docker_build.yml
+++ /dev/null
@@ -1,27 +0,0 @@
-$inherits:
- from: 'build.yml'
-
-
-task:
- workerType: b2gbuild
-
- routes:
- - 'index.gecko.v1.{{project}}.revision.linux.{{head_rev}}.{{build_name}}.{{build_type}}'
- - 'index.gecko.v1.{{project}}.latest.linux.{{build_name}}.{{build_type}}'
-
- scopes:
- # docker build tasks use tc-vcs so include the scope.
- - 'docker-worker:cache:level-{{level}}-{{project}}-tc-vcs'
-
- payload:
-
- cache:
- level-{{level}}-{{project}}-tc-vcs: '/home/worker/.tc-vcs'
-
- # All docker builds share a common artifact directory for ease of uploading.
- artifacts:
- 'public/build':
- type: directory
- path: '/home/worker/artifacts/'
- expires:
- relative-datestamp: '1 year'
deleted file mode 100644
--- a/taskcluster/ci/nightly-fennec/kind.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-implementation: 'taskgraph.task.nightly_fennec:NightlyFennecTask'
-nightly_fennec_path: '.'
-
deleted file mode 100644
--- a/taskcluster/ci/nightly-fennec/mobile_base.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-$inherits:
- from: 'docker_build.yml'
- variables:
- build_product: 'mobile'
-docker-image: desktop-build
-task:
- payload:
- image:
- type: 'task-image'
- path: 'public/image.tar'
- taskId:
- task-reference: "<docker-image>"
-
deleted file mode 100644
--- a/taskcluster/ci/nightly-fennec/routes.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "routes": [
- "{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
- "{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}.{build_name}-{build_type}",
- "{index}.gecko.v2.{project}.latest.{build_product}.{build_name}-{build_type}"
- ],
- "nightly": [
- "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
- "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.latest.{build_product}.{build_name}-{build_type}",
- "{index}.gecko.v2.{project}.nightly.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
- "{index}.gecko.v2.{project}.nightly.latest.{build_product}.{build_name}-{build_type}"
- ],
- "l10n": [
- "{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
- "{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
- "{index}.gecko.v2.{project}.latest.{build_product}-l10n.{build_name}-{build_type}.{locale}"
- ]
-}
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -111,8 +111,14 @@ e10s
For test suites which distinguish whether they run with or without e10s, this
boolean value identifies this particular run.
image_name
==========
For the ``docker_image`` kind, this attribute contains the docker image name.
+
+nightly
+=======
+
+Signals whether the task is part of a nightly graph. Useful when filtering
+out nightly tasks from full task set at target stage.
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -101,10 +101,11 @@ def target_tasks_ash(full_task_graph, pa
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('nightly_fennec')
def target_tasks_nightly(full_task_graph, parameters):
"""Select the set of tasks required for a nightly build of fennec. The
nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
- return [t.label for t in full_task_graph.tasks.itervalues()
- if t.attributes.get('kind') in ['nightly-fennec', 'signing']]
+ def filter(task):
+ return task.attributes.get('nightly', False)
+ return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
deleted file mode 100644
--- a/taskcluster/taskgraph/task/nightly_fennec.py
+++ /dev/null
@@ -1,201 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-"""
-Temporary placeholder for nightly builds
-"""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import logging
-import json
-import os
-import time
-from collections import namedtuple
-
-from . import base
-from slugid import nice as slugid
-from taskgraph.util.templates import Templates
-from taskgraph.util.docker import docker_image
-
-
-logger = logging.getLogger(__name__)
-GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
-ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
-INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
-TASKID_PLACEHOLDER = 'TaskLabel=={}'
-
-
-def mklabel():
- return TASKID_PLACEHOLDER.format(slugid())
-
-
-def gaia_info():
- '''Fetch details from in tree gaia.json (which links this version of
- gecko->gaia) and construct the usual base/head/ref/rev pairing...'''
- gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json')))
-
- if gaia['git'] is None or \
- gaia['git']['remote'] == '' or \
- gaia['git']['git_revision'] == '' or \
- gaia['git']['branch'] == '':
-
- # Just use the hg params...
- return {
- 'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
- 'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
- 'gaia_ref': gaia['revision'],
- 'gaia_rev': gaia['revision']
- }
-
- else:
- # Use git
- return {
- 'gaia_base_repository': gaia['git']['remote'],
- 'gaia_head_repository': gaia['git']['remote'],
- 'gaia_rev': gaia['git']['git_revision'],
- 'gaia_ref': gaia['git']['branch'],
- }
-
-
-def decorate_task_json_routes(task, json_routes, parameters):
- """Decorate the given task with routes.json routes.
-
- :param dict task: task definition.
- :param json_routes: the list of routes to use from routes.json
- :param parameters: dictionary of parameters to use in route templates
- """
- routes = task.get('routes', [])
- for route in json_routes:
- routes.append(route.format(**parameters))
-
- task['routes'] = routes
-
-
-def query_vcs_info(repository, revision):
- """Query the pushdate and pushid of a repository/revision.
-
- This is intended to be used on hg.mozilla.org/mozilla-central and
- similar. It may or may not work for other hg repositories.
- """
- if not repository or not revision:
- logger.warning('cannot query vcs info because vcs info not provided')
- return None
-
- VCSInfo = namedtuple('VCSInfo', ['pushid', 'pushdate', 'changesets'])
-
- try:
- import requests
- url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'),
- revision)
- logger.debug("Querying version control for metadata: %s", url)
- contents = requests.get(url).json()
-
- changesets = []
- for c in contents['changesets']:
- changesets.append({k: c[k] for k in ('desc', 'files', 'node')})
-
- pushid = contents['changesets'][-1]['pushid']
- pushdate = contents['changesets'][-1]['pushdate'][0]
-
- return VCSInfo(pushid, pushdate, changesets)
-
- except Exception:
- logger.exception("Error querying VCS info for '%s' revision '%s'",
- repository, revision)
- return None
-
-
-class NightlyFennecTask(base.Task):
-
- def __init__(self, *args, **kwargs):
- try:
- self.task_dict = kwargs.pop('task_dict')
- except KeyError:
- pass
- super(NightlyFennecTask, self).__init__(*args, **kwargs)
-
- @classmethod
- def load_tasks(cls, kind, path, config, params, loaded_tasks):
- root = os.path.abspath(os.path.join(path, config[
- 'nightly_fennec_path']))
-
- project = params['project']
-
- # Set up the parameters, including the time of the build
- push_epoch = int(time.time())
- vcs_info = query_vcs_info(params['head_repository'], params['head_rev'])
- changed_files = set()
- if vcs_info:
- push_epoch = vcs_info.pushdate
-
- logger.debug(
- '{} commits influencing task scheduling:'.format(len(vcs_info.changesets)))
- for c in vcs_info.changesets:
- logger.debug("{cset} {desc}".format(
- cset=c['node'][0:12],
- desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
- changed_files |= set(c['files'])
-
- pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(push_epoch))
-
- # Template parameters used when expanding the graph
- parameters = dict(gaia_info().items() + {
- 'index': 'index',
- 'project': project,
- 'pushlog_id': params.get('pushlog_id', 0),
- 'base_repository': params['base_repository'] or params['head_repository'],
- 'docker_image': docker_image,
- 'head_repository': params['head_repository'],
- 'head_ref': params['head_ref'] or params['head_rev'],
- 'head_rev': params['head_rev'],
- 'pushdate': pushdate,
- 'pushtime': pushdate[8:],
- 'year': pushdate[0:4],
- 'month': pushdate[4:6],
- 'day': pushdate[6:8],
- 'rank': push_epoch,
- 'owner': params['owner'],
- 'level': params['level'],
- 'build_slugid': mklabel(),
- 'source': '{repo}file/{rev}/taskcluster/ci/nightly-fennec'.format(
- repo=params['head_repository'], rev=params['head_rev']),
- 'build_name': 'android',
- 'build_type': 'opt',
- 'build_product': 'mobile'
- }.items())
-
- routes_file = os.path.join(root, 'routes.json')
- with open(routes_file) as f:
- contents = json.load(f)
- json_routes = contents['routes']
-
- tasks = []
- templates = Templates(root)
-
- task = templates.load('android-api-15-nightly-build.yml', parameters)
- decorate_task_json_routes(task['task'], json_routes, parameters)
-
- attributes = {'kind': 'nightly-fennec'}
- tasks.append(cls(kind, 'build-nightly-fennec',
- task=task['task'], attributes=attributes,
- task_dict=task))
-
- # Convert to a dictionary of tasks. The process above has invented a
- # taskId for each task, and we use those as the *labels* for the tasks;
- # taskgraph will later assign them new taskIds.
- return tasks
-
- def get_dependencies(self, taskgraph):
- deps = [(label, label) for label in self.task_dict.get('requires', [])]
-
- # add a dependency on an image task, if needed
- if 'docker-image' in self.task_dict:
- deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict),
- 'docker-image'))
-
- return deps
-
- def optimize(self, params):
- return False, None
--- a/taskcluster/taskgraph/task/signing.py
+++ b/taskcluster/taskgraph/task/signing.py
@@ -14,42 +14,39 @@ from taskgraph.util.templates import Tem
logger = logging.getLogger(__name__)
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
class SigningTask(base.Task):
- def __init__(self, *args, **kwargs):
- super(SigningTask, self).__init__(*args, **kwargs)
+ def __init__(self, kind, name, task, attributes):
+ self.unsigned_artifact_label = task['unsigned-task']['label']
+ super(SigningTask, self).__init__(kind, name, task=task['task'],
+ attributes=attributes)
@classmethod
def load_tasks(cls, kind, path, config, params, loaded_tasks):
- root = os.path.abspath(os.path.join(path, config['signing_path']))
-
- # get each nightly-fennec and add its name to this task
- fennec_tasks = [t for t in loaded_tasks if t.attributes.get('kind') == 'nightly-fennec']
+ root = os.path.abspath(path)
tasks = []
- for fennec_task in fennec_tasks:
+ for filename in config.get('jobs-from', []):
templates = Templates(root)
- task = templates.load('signing.yml', {})
+ jobs = templates.load(filename, {})
- artifacts = ['public/build/target.apk',
- 'public/build/en-US/target.apk']
- for artifact in artifacts:
- url = ARTIFACT_URL.format('<build-nightly-fennec>', artifact)
- task['task']['payload']['unsignedArtifacts'].append({
- 'task-reference': url
- })
-
- attributes = {'kind': 'signing'}
- tasks.append(cls(kind, 'signing-nightly-fennec', task=task['task'],
- attributes=attributes))
+ for name, job in jobs.iteritems():
+ for artifact in job['unsigned-task']['artifacts']:
+ url = ARTIFACT_URL.format('<{}>'.format('unsigned-artifact'), artifact)
+ job['task']['payload']['unsignedArtifacts'].append({
+ 'task-reference': url
+ })
+ attributes = job.setdefault('attributes', {})
+ attributes.update({'kind': 'signing'})
+ tasks.append(cls(kind, name, job, attributes=attributes))
return tasks
def get_dependencies(self, taskgraph):
- return [('build-nightly-fennec', 'build-nightly-fennec')]
+ return [(self.unsigned_artifact_label, 'unsigned-artifact')]
def optimize(self, params):
return False, None