--- a/taskcluster/docs/parameters.rst
+++ b/taskcluster/docs/parameters.rst
@@ -139,16 +139,22 @@ Optimization
taskId rather than submitting a new task.
Release Promotion
-----------------
``build_number``
Specify the release promotion build number.
+``version``
+ Specify the version for release tasks.
+
+``app_version``
+ Specify the application version for release tasks. For releases, this is often a less specific version number than ``version``.
+
``next_version``
Specify the next version for version bump tasks.
``desktop_release_type``
The type of desktop release being promoted. One of "beta", "devedition", "esr", "rc",
or "release".
``release_eta``
--- a/taskcluster/taskgraph/__init__.py
+++ b/taskcluster/taskgraph/__init__.py
@@ -3,16 +3,19 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import os
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
+APP_VERSION_PATH = os.path.join(GECKO, "browser", "config", "version.txt")
+VERSION_PATH = os.path.join(GECKO, "browser", "config", "version_display.txt")
+
# Maximum number of dependencies a single task can have
# https://docs.taskcluster.net/reference/platform/taskcluster-queue/references/api#createTask
# specifies 100, but we also optionally add the decision task id as a dep in
# taskgraph.create, so let's set this to 99.
MAX_DEPENDENCIES = 99
# Enable fast task generation for local debugging
# This is normally switched on via the --fast/-F flag to `mach taskgraph`
--- a/taskcluster/taskgraph/actions/release_promotion.py
+++ b/taskcluster/taskgraph/actions/release_promotion.py
@@ -18,72 +18,73 @@ from taskgraph.util.partials import popu
from taskgraph.taskgraph import TaskGraph
from taskgraph.decision import taskgraph_decision
from taskgraph.parameters import Parameters
from taskgraph.util.attributes import RELEASE_PROMOTION_PROJECTS
RELEASE_PROMOTION_CONFIG = {
'promote_fennec': {
'target_tasks_method': 'promote_fennec',
+ 'product': 'fennec',
},
'ship_fennec': {
'target_tasks_method': 'ship_fennec',
+ 'product': 'fennec',
},
'promote_firefox': {
'target_tasks_method': 'promote_firefox',
+ 'product': 'firefox',
},
'push_firefox': {
'target_tasks_method': 'push_firefox',
+ 'product': 'firefox',
},
'ship_firefox': {
'target_tasks_method': 'ship_firefox',
+ 'product': 'firefox',
+ },
+ 'promote_firefox_rc': {
+ 'target_tasks_method': 'promote_firefox',
+ 'product': 'firefox',
+ 'desktop_release_type': 'rc',
+ },
+ 'ship_firefox_rc': {
+ 'target_tasks_method': 'ship_firefox',
+ 'product': 'firefox',
+ 'desktop_release_type': 'rc',
},
'promote_devedition': {
'target_tasks_method': 'promote_devedition',
+ 'product': 'devedition',
},
'push_devedition': {
'target_tasks_method': 'push_devedition',
+ 'product': 'devedition',
},
'ship_devedition': {
'target_tasks_method': 'ship_devedition',
+ 'product': 'devedition',
},
}
VERSION_BUMP_FLAVORS = (
'ship_fennec',
'ship_firefox',
'ship_devedition',
)
UPTAKE_MONITORING_PLATFORMS_FLAVORS = (
'push_firefox',
'push_devedition',
)
PARTIAL_UPDATES_FLAVORS = UPTAKE_MONITORING_PLATFORMS_FLAVORS + (
'promote_firefox',
- 'promote_devedition',
-)
-
-DESKTOP_RELEASE_TYPE_FLAVORS = (
- 'promote_firefox',
- 'push_firefox',
- 'ship_firefox',
+ 'promote_firefox_rc',
'promote_devedition',
- 'push_devedition',
- 'ship_devedition',
-)
-
-
-VALID_DESKTOP_RELEASE_TYPES = (
- 'beta',
- 'devedition',
- 'esr',
- 'release',
- 'rc',
)
def is_release_promotion_available(parameters):
return parameters['project'] in RELEASE_PROMOTION_PROJECTS
@register_callback_action(
@@ -122,22 +123,16 @@ def is_release_promotion_available(param
'is specified, find the `pushlog_id using the '
'revision.'),
},
'release_promotion_flavor': {
'type': 'string',
'description': 'The flavor of release promotion to perform.',
'enum': sorted(RELEASE_PROMOTION_CONFIG.keys()),
},
- 'target_tasks_method': {
- 'type': 'string',
- 'title': 'target task method',
- 'description': ('Optional: the target task method to use to generate '
- 'the new graph.'),
- },
'rebuild_kinds': {
'type': 'array',
'description': ('Optional: an array of kinds to ignore from the previous '
'graph(s).'),
'items': {
'type': 'string',
},
},
@@ -145,36 +140,47 @@ def is_release_promotion_available(param
'type': 'array',
'description': ('Optional: an array of taskIds of decision or action '
'tasks from the previous graph(s) to use to populate '
'our `previous_graph_kinds`.'),
'items': {
'type': 'string',
},
},
+ 'version': {
+ 'type': 'string',
+ 'description': ('Optional: override the version for release promotion. '
+ "Occasionally we'll land a taskgraph fix in a later "
+ 'commit, but want to act on a build from a previous '
+ 'commit. If a version bump has landed in the meantime, '
+ 'relying on the in-tree version will break things.'),
+ 'default': '',
+ },
'next_version': {
'type': 'string',
- 'description': 'Next version.',
+ 'description': ('Next version. Required in the following flavors: '
+ '{}'.format(sorted(VERSION_BUMP_FLAVORS))),
'default': '',
},
# Example:
# 'partial_updates': {
# '38.0': {
# 'buildNumber': 1,
# 'locales': ['de', 'en-GB', 'ru', 'uk', 'zh-TW']
# },
# '37.0': {
# 'buildNumber': 2,
# 'locales': ['de', 'en-GB', 'ru', 'uk']
# }
# }
'partial_updates': {
'type': 'object',
- 'description': 'Partial updates.',
+ 'description': ('Partial updates. Required in the following flavors: '
+ '{}'.format(sorted(PARTIAL_UPDATES_FLAVORS))),
'default': {},
'additionalProperties': {
'type': 'object',
'properties': {
'buildNumber': {
'type': 'number',
},
'locales': {
@@ -202,59 +208,48 @@ def is_release_promotion_available(param
'win64',
'linux',
'linux64',
],
},
'default': [],
},
- 'desktop_release_type': {
- 'type': 'string',
- 'default': '',
- },
-
'release_eta': {
'type': 'string',
'default': '',
},
},
"required": ['release_promotion_flavor', 'build_number'],
}
)
def release_promotion_action(parameters, input, task_group_id, task_id, task):
release_promotion_flavor = input['release_promotion_flavor']
+ promotion_config = RELEASE_PROMOTION_CONFIG[release_promotion_flavor]
release_history = {}
- desktop_release_type = None
+ product = promotion_config['product']
next_version = str(input.get('next_version') or '')
if release_promotion_flavor in VERSION_BUMP_FLAVORS:
# We force str() the input, hence the 'None'
if next_version in ['', 'None']:
raise Exception(
"`next_version` property needs to be provided for %s "
"targets." % ', '.join(VERSION_BUMP_FLAVORS)
)
- if release_promotion_flavor in DESKTOP_RELEASE_TYPE_FLAVORS:
- desktop_release_type = input.get('desktop_release_type', None)
- if desktop_release_type not in VALID_DESKTOP_RELEASE_TYPES:
- raise Exception("`desktop_release_type` must be one of: %s" %
- ", ".join(VALID_DESKTOP_RELEASE_TYPES))
-
+ if product in ('firefox', 'devedition'):
if release_promotion_flavor in PARTIAL_UPDATES_FLAVORS:
partial_updates = json.dumps(input.get('partial_updates', {}))
if partial_updates == "{}":
raise Exception(
"`partial_updates` property needs to be provided for %s "
"targets." % ', '.join(PARTIAL_UPDATES_FLAVORS)
)
- balrog_prefix = 'Firefox'
- if desktop_release_type == 'devedition':
- balrog_prefix = 'Devedition'
+ balrog_prefix = product.title()
os.environ['PARTIAL_UPDATES'] = partial_updates
release_history = populate_release_history(
balrog_prefix, parameters['project'],
partial_updates=input['partial_updates']
)
if release_promotion_flavor in UPTAKE_MONITORING_PLATFORMS_FLAVORS:
uptake_monitoring_platforms = json.dumps(input.get('uptake_monitoring_platforms', []))
@@ -262,19 +257,18 @@ def release_promotion_action(parameters,
raise Exception(
"`uptake_monitoring_platforms` property needs to be provided for %s "
"targets." % ', '.join(UPTAKE_MONITORING_PLATFORMS_FLAVORS)
)
os.environ['UPTAKE_MONITORING_PLATFORMS'] = uptake_monitoring_platforms
promotion_config = RELEASE_PROMOTION_CONFIG[release_promotion_flavor]
- target_tasks_method = input.get(
- 'target_tasks_method',
- promotion_config['target_tasks_method'].format(project=parameters['project'])
+ target_tasks_method = promotion_config['target_tasks_method'].format(
+ project=parameters['project']
)
rebuild_kinds = input.get(
'rebuild_kinds', promotion_config.get('rebuild_kinds', [])
)
do_not_optimize = input.get(
'do_not_optimize', promotion_config.get('do_not_optimize', [])
)
@@ -304,15 +298,17 @@ def release_promotion_action(parameters,
parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds(
combined_full_task_graph, previous_graph_ids, rebuild_kinds
)
parameters['do_not_optimize'] = do_not_optimize
parameters['target_tasks_method'] = target_tasks_method
parameters['build_number'] = int(input['build_number'])
parameters['next_version'] = next_version
parameters['release_history'] = release_history
- parameters['desktop_release_type'] = desktop_release_type
+ parameters['desktop_release_type'] = promotion_config.get('desktop_release_type', '')
parameters['release_eta'] = input.get('release_eta', '')
+ if input['version']:
+ parameters['version'] = input['version']
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({}, parameters=parameters)
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -9,17 +9,17 @@ import os
import json
import logging
import time
import yaml
from .generator import TaskGraphGenerator
from .create import create_tasks
-from .parameters import Parameters
+from .parameters import Parameters, get_version, get_app_version
from .taskgraph import TaskGraph
from .try_option_syntax import parse_message
from .actions import render_actions_json
from taskgraph.util.partials import populate_release_history
logger = logging.getLogger(__name__)
ARTIFACTS_DIR = 'artifacts'
@@ -188,16 +188,18 @@ def get_decision_parameters(options):
# custom filters.
parameters['filters'] = [
'check_servo',
'target_tasks_method',
]
parameters['existing_tasks'] = {}
parameters['do_not_optimize'] = []
parameters['build_number'] = 1
+ parameters['version'] = get_version()
+ parameters['app_version'] = get_app_version()
parameters['next_version'] = None
parameters['desktop_release_type'] = ''
parameters['release_eta'] = ''
# owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
# case, fake it
if '@' not in parameters['owner']:
parameters['owner'] += '@noreply.mozilla.org'
--- a/taskcluster/taskgraph/parameters.py
+++ b/taskcluster/taskgraph/parameters.py
@@ -1,39 +1,51 @@
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
+import functools
import json
import time
import yaml
from datetime import datetime
from mozbuild.util import ReadOnlyDict, memoize
from mozversioncontrol import get_repository_object
-from . import GECKO
+from . import APP_VERSION_PATH, GECKO, VERSION_PATH
class ParameterMismatch(Exception):
"""Raised when a parameters.yml has extra or missing parameters."""
@memoize
def get_head_ref():
return get_repository_object(GECKO).head_ref
+def get_contents(path):
+ with open(path, "r") as fh:
+ contents = fh.readline().rstrip()
+ return contents
+
+
+get_version = functools.partial(get_contents, VERSION_PATH)
+get_app_version = functools.partial(get_contents, APP_VERSION_PATH)
+
+
# Please keep this list sorted and in sync with taskcluster/docs/parameters.rst
# Parameters are of the form: {name: default}
PARAMETERS = {
+ 'app_version': get_app_version(),
'base_repository': 'https://hg.mozilla.org/mozilla-unified',
'build_date': lambda: int(time.time()),
'build_number': 1,
'desktop_release_type': '',
'do_not_optimize': [],
'existing_tasks': {},
'filters': ['check_servo', 'target_tasks_method'],
'head_ref': get_head_ref,
@@ -50,16 +62,17 @@ PARAMETERS = {
'pushdate': lambda: int(time.time()),
'pushlog_id': '0',
'release_eta': '',
'release_history': {},
'target_tasks_method': 'default',
'try_mode': None,
'try_options': None,
'try_task_config': None,
+ 'version': get_version(),
}
COMM_PARAMETERS = {
'comm_base_repository': 'https://hg.mozilla.org/comm-central',
'comm_head_ref': None,
'comm_head_repository': 'https://hg.mozilla.org/comm-central',
'comm_head_rev': None,
}
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -338,20 +338,16 @@ def target_tasks_promote_firefox(full_ta
'release-secondary-final-verify'):
if 'secondary' in task.label:
return False
if task.attributes.get('shipping_product') == 'firefox' and \
task.attributes.get('shipping_phase') == 'promote':
return True
- # TODO: funsize, all but balrog submission
- # TODO: bouncer sub
- # TODO: recompression tasks
-
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('push_firefox')
def target_tasks_push_firefox(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to push a build of firefox to cdns.
Previous build deps will be optimized out via action task."""
filtered_for_candidates = target_tasks_promote_firefox(
@@ -360,21 +356,16 @@ def target_tasks_push_firefox(full_task_
def filter(task):
# Include promotion tasks; these will be optimized out
if task.label in filtered_for_candidates:
return True
if task.attributes.get('shipping_product') == 'firefox' and \
task.attributes.get('shipping_phase') == 'push':
return True
- # TODO: add beetmover push-to-releases
- # TODO: publish to balrog
- # TODO: funsize balrog submission
- # TODO: recompression push-to-releases + balrog
- # TODO: checksums
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('ship_firefox')
def target_tasks_ship_firefox(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to ship firefox.
Previous build deps will be optimized out via action task."""
@@ -384,21 +375,16 @@ def target_tasks_ship_firefox(full_task_
def filter(task):
# Include promotion tasks; these will be optimized out
if task.label in filtered_for_candidates:
return True
if task.attributes.get('shipping_product') == 'firefox' and \
task.attributes.get('shipping_phase') == 'ship':
return True
- # TODO: add beetmover push-to-releases
- # TODO: publish to balrog
- # TODO: funsize balrog submission
- # TODO: recompression push-to-releases + balrog
- # TODO: checksums
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('promote_devedition')
def target_tasks_promote_devedition(full_task_graph, parameters, graph_config):
"""Select the superset of tasks required to promote a beta or release build
of devedition. This should include all non-android mozilla_{beta,release}
@@ -426,21 +412,16 @@ def target_tasks_promote_devedition(full
# shipping_product matches.
if task.label in beta_release_tasks:
return True
if task.attributes.get('shipping_product') == 'devedition' and \
task.attributes.get('shipping_phase') == 'promote':
return True
- # TODO: funsize, all but balrog submission
- # TODO: binary transparency
- # TODO: bouncer sub
- # TODO: recompression tasks
-
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('push_devedition')
def target_tasks_push_devedition(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to push a build of devedition to cdns.
Previous build deps will be optimized out via action task."""
filtered_for_candidates = target_tasks_promote_devedition(
@@ -449,21 +430,16 @@ def target_tasks_push_devedition(full_ta
def filter(task):
# Include promotion tasks; these will be optimized out
if task.label in filtered_for_candidates:
return True
if task.attributes.get('shipping_product') == 'devedition' and \
task.attributes.get('shipping_phase') == 'push':
return True
- # TODO: add beetmover push-to-releases
- # TODO: publish to balrog
- # TODO: funsize balrog submission
- # TODO: recompression push-to-releases + balrog
- # TODO: checksums
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('ship_devedition')
def target_tasks_ship_devedition(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to ship devedition.
Previous build deps will be optimized out via action task."""
@@ -473,21 +449,16 @@ def target_tasks_ship_devedition(full_ta
def filter(task):
# Include promotion tasks; these will be optimized out
if task.label in filtered_for_candidates:
return True
if task.attributes.get('shipping_product') == 'devedition' and \
task.attributes.get('shipping_phase') == 'ship':
return True
- # TODO: add beetmover push-to-releases
- # TODO: publish to balrog
- # TODO: funsize balrog submission
- # TODO: recompression push-to-releases + balrog
- # TODO: checksums
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('promote_fennec')
def target_tasks_promote_fennec(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a candidates build of fennec. The
nightly build process involves a pipeline of builds, signing,
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -17,20 +17,16 @@ Additional configuration is found in the
"""
from __future__ import absolute_import, print_function, unicode_literals
import functools
import json
import os
# constants {{{1
-GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
-VERSION_PATH = os.path.join(GECKO, "browser", "config", "version_display.txt")
-APP_VERSION_PATH = os.path.join(GECKO, "browser", "config", "version.txt")
-
"""Map signing scope aliases to sets of projects.
Currently m-c and DevEdition on m-b use nightly signing; Beta on m-b and m-r
use release signing. These data structures aren't set-up to handle different
scopes on the same repo, so we use a different set of them for DevEdition, and
callers are responsible for using the correct one (by calling the appropriate
helper below). More context on this in https://bugzilla.mozilla.org/show_bug.cgi?id=1358601.
@@ -468,22 +464,18 @@ def get_release_config(config):
uptake_monitoring_platforms = os.environ.get("UPTAKE_MONITORING_PLATFORMS", "[]")
if uptake_monitoring_platforms != "[]" and \
config.kind in ('release-uptake-monitoring',):
uptake_monitoring_platforms = json.loads(uptake_monitoring_platforms)
release_config['platforms'] = ', '.join(uptake_monitoring_platforms)
if release_config['platforms'] == "[]":
del release_config['platforms']
- with open(VERSION_PATH, "r") as fh:
- version = fh.readline().rstrip()
- release_config['version'] = version
- with open(APP_VERSION_PATH, "r") as fh:
- appVersion = fh.readline().rstrip()
- release_config['appVersion'] = appVersion
+ release_config['version'] = str(config.params['version'])
+ release_config['appVersion'] = str(config.params['app_version'])
release_config['next_version'] = str(config.params['next_version'])
release_config['build_number'] = config.params['build_number']
return release_config
def get_signing_cert_scope_per_platform(build_platform, is_nightly, config):
if 'devedition' in build_platform: