--- a/buildfarm/release/release-runner.py
+++ b/buildfarm/release/release-runner.py
@@ -10,36 +10,36 @@ import subprocess
import hashlib
import functools
import shutil
import tempfile
import requests
from os import path
from optparse import OptionParser
from twisted.python.lockfile import FilesystemLock
+from taskcluster import Scheduler, Index, Queue
+from taskcluster.utils import slugId
+import yaml
site.addsitedir(path.join(path.dirname(__file__), "../../lib/python"))
-from kickoff import get_partials, ReleaseRunner, make_task_graph_strict_kwargs, long_revision
-from kickoff import get_l10n_config, get_en_US_config
-from kickoff import email_release_drivers
-from kickoff import bump_version
+from kickoff import (get_partials, ReleaseRunner,
+ make_task_graph_strict_kwargs, long_revision,
+ get_l10n_config, get_en_US_config, email_release_drivers,
+ bump_version)
from kickoff.sanity.base import SanityException, is_candidate_release
from kickoff.sanity.revisions import RevisionsSanitizer
from kickoff.sanity.l10n import L10nSanitizer
from kickoff.sanity.partials import PartialsSanitizer
from kickoff.build_status import are_en_us_builds_completed
from release.info import readBranchConfig
from release.l10n import parsePlainL10nChangesets
from release.versions import getAppVersion
-from taskcluster import Scheduler, Index, Queue
-from taskcluster.utils import slugId
from util.hg import mercurial
from util.retry import retry
-import yaml
log = logging.getLogger(__name__)
# both CHECKSUMS and ALL_FILES have been defined to improve the release sanity
# en-US binaries timing by whitelisting artifacts of interest - bug 1251761
CHECKSUMS = set([
'.checksums',
@@ -48,18 +48,17 @@ CHECKSUMS = set([
ALL_FILES = set([
'.checksums',
'.checksums.asc',
'.complete.mar',
'.exe',
'.dmg',
- 'i686.tar.bz2',
- 'x86_64.tar.bz2',
+ 'tar.bz2',
])
CONFIGS_WORKDIR = 'buildbot-configs'
def check_and_assign_long_revision(release_runner, release):
# Revisions must be checked before trying to get the long one.
RevisionsSanitizer(**release).run()
@@ -273,19 +272,23 @@ def get_hash(path, hash_type="sha512"):
for chunk in iter(functools.partial(f.read, 4096), ''):
h.update(chunk)
return h.hexdigest()
def validate_graph_kwargs(queue, gpg_key_path, **kwargs):
# TODO: to be moved under kickoff soon, once new relpro sanity is in place
# bug 1282959
- platforms = kwargs.get('en_US_config', {}).get('platforms', {})
- for platform in platforms.keys():
- task_id = platforms.get(platform).get('task_id', {})
+ platforms = kwargs['en_US_config']['platforms']
+ for platform in platforms:
+ # FIXME: enable sanity check later for TC platforms
+ if platforms[platform]["signed_task_id"] != platforms[platform]["unsigned_task_id"]:
+ log.warning("Skipping en-US sanity for %s, TC platform", platform)
+ continue
+ task_id = platforms[platform]['signed_task_id']
log.info('Performing release sanity for %s en-US binary', platform)
sanitize_en_US_binary(queue, task_id, gpg_key_path)
log.info("Release sanity for all en-US is now completed!")
def main(options):
log.info('Loading config from %s' % options.config)
@@ -399,35 +402,35 @@ def main(options):
postrelease_bouncer_aliases_enabled = branchConfig[
'postrelease_bouncer_aliases_enabled']
postrelease_mark_as_shipped_enabled = branchConfig[
'postrelease_mark_as_shipped_enabled']
final_verify_channels = release_channels
publish_to_balrog_channels = release_channels
push_to_releases_enabled = True
- ship_it_product_name = release['product']
- tc_product_name = branchConfig['stage_product'][ship_it_product_name]
# XXX: Doesn't work with neither Fennec nor Thunderbird
platforms = branchConfig['release_platforms']
try:
- if not are_en_us_builds_completed(index, release_name=release['name'], submitted_at=release['submittedAt'],
- branch=release['branchShortName'], revision=release[
- 'mozillaRevision'],
- tc_product_name=tc_product_name, platforms=platforms, queue=queue):
+ graph_id = slugId()
+ done = are_en_us_builds_completed(
+ index=index, release_name=release['name'],
+ submitted_at=release['submittedAt'],
+ revision=release['mozillaRevision'],
+ platforms=platforms, queue=queue,
+ tc_task_indexes=branchConfig['tc_indexes'][release['product']])
+ if not done:
log.info(
'Builds are not completed yet, skipping release "%s" for now', release['name'])
rr.update_status(release, 'Waiting for builds to be completed')
continue
log.info('Every build is completed for release: %s',
release['name'])
- graph_id = slugId()
-
rr.update_status(release, 'Generating task graph')
kwargs = {
"public_key": docker_worker_key,
"version": release["version"],
# ESR should not use "esr" suffix here:
"next_version": bump_version(release["version"].replace("esr", "")),
"appVersion": getAppVersion(release["version"]),
@@ -443,23 +446,25 @@ def main(options):
"branch": release['branchShortName'],
"updates_enabled": bool(release["partials"]),
"l10n_config": get_l10n_config(
index=index, product=release[
"product"], branch=release['branchShortName'],
revision=release['mozillaRevision'],
platforms=branchConfig['platforms'],
l10n_platforms=branchConfig['l10n_release_platforms'],
- l10n_changesets=release['l10n_changesets']
+ l10n_changesets=release['l10n_changesets'],
+ tc_task_indexes=branchConfig['tc_indexes'][release['product']],
),
"en_US_config": get_en_US_config(
index=index, product=release[
"product"], branch=release['branchShortName'],
revision=release['mozillaRevision'],
- platforms=branchConfig['release_platforms']
+ platforms=branchConfig['release_platforms'],
+ tc_task_indexes=branchConfig['tc_indexes'][release['product']],
),
"verifyConfigs": {},
"balrog_api_root": branchConfig["balrog_api_root"],
"funsize_balrog_api_root": branchConfig["funsize_balrog_api_root"],
"balrog_username": balrog_username,
"balrog_password": balrog_password,
"beetmover_aws_access_key_id": beetmover_aws_access_key_id,
"beetmover_aws_secret_access_key": beetmover_aws_secret_access_key,
@@ -495,17 +500,17 @@ def main(options):
}
validate_graph_kwargs(queue, gpg_key_path, **kwargs)
graph = make_task_graph_strict_kwargs(**kwargs)
rr.update_status(release, "Submitting task graph")
log.info("Task graph generated!")
import pprint
log.debug(pprint.pformat(graph, indent=4, width=160))
- print scheduler.createTaskGraph(graph_id, graph)
+ print(scheduler.createTaskGraph(graph_id, graph))
rr.mark_as_completed(release)
l10n_url = rr.release_l10n_api.getL10nFullUrl(release['name'])
email_release_drivers(smtp_server=smtp_server, from_=notify_from,
to=notify_to, release=release,
task_group_id=graph_id, l10n_url=l10n_url)
except Exception as exception:
# We explicitly do not raise an error here because there's no
--- a/buildfarm/release/releasetasks_graph_gen.py
+++ b/buildfarm/release/releasetasks_graph_gen.py
@@ -88,23 +88,25 @@ def main(release_runner_config, release_
# in release-runner.py world we have a concept of branchConfig and release (shipit) vars
# todo fix get_en_US_config and en_US_config helper methods to not require both
"l10n_config": get_l10n_config(
index=index, product=release_config["product"],
branch=release_config["branch"],
revision=release_config["mozilla_revision"],
platforms=release_config['platforms'],
l10n_platforms=release_config['l10n_release_platforms'] or {},
- l10n_changesets=release_config["l10n_changesets"]
+ l10n_changesets=release_config["l10n_changesets"],
+ tc_task_indexes=None,
),
"en_US_config": get_en_US_config(
index=index, product=release_config["product"],
branch=release_config["branch"],
revision=release_config["mozilla_revision"],
- platforms=release_config['platforms']
+ platforms=release_config['platforms'],
+ tc_task_indexes=None,
),
"extra_balrog_submitter_params": release_config['extra_balrog_submitter_params'],
"publish_to_balrog_channels": release_config["publish_to_balrog_channels"],
"postrelease_mark_as_shipped_enabled": release_config["postrelease_mark_as_shipped_enabled"],
# TODO: use [] when snaps_enabled is landed
"snap_enabled": release_config.get("snap_enabled", False),
"update_verify_channel": release_config["update_verify_channel"],
"update_verify_requires_cdn_push": release_config["update_verify_requires_cdn_push"],
--- a/lib/python/kickoff/__init__.py
+++ b/lib/python/kickoff/__init__.py
@@ -13,19 +13,20 @@ log = logging.getLogger(__name__)
# temporary regex to filter out anything but mozilla-beta and mozilla-release
# within release promotion. Once migration to release promotion is completed
# for all types of releases, we will backout this filtering
# regex beta tracking bug is 1252333,
# regex release tracking bug is 1263976
# This is now a default
RELEASE_PATTERNS = [
r"Firefox-.*",
-# r"Fennec-.*",
+ # r"Fennec-.*",
]
+
def matches(name, patterns):
return any([re.search(p, name) for p in patterns])
def long_revision(repo, revision):
"""Convert short revision to long using JSON API
>>> long_revision("releases/mozilla-beta", "59f372c35b24")
@@ -75,21 +76,21 @@ class ReleaseRunner(object):
def get_release_l10n(self, release):
return self.release_l10n_api.getL10n(release)
def update_status(self, release, status):
log.info('updating status for %s to %s' % (release['name'], status))
try:
self.release_api.update(release['name'], status=status)
- except requests.HTTPError, e:
+ except requests.HTTPError as e:
log.warning('Caught HTTPError: %s' % e.response.content)
log.warning('status update failed, continuing...', exc_info=True)
- def mark_as_completed(self, release):#, enUSPlatforms):
+ def mark_as_completed(self, release):
log.info('mark as completed %s' % release['name'])
self.release_api.update(release['name'], complete=True,
status='Started')
def mark_as_failed(self, release, why):
log.info('mark as failed %s' % release['name'])
self.release_api.update(release['name'], ready=False, status=why)
@@ -151,48 +152,60 @@ def get_platform_locales(l10n_changesets
if platform == "macosx64":
ignore = "ja"
else:
ignore = "ja-JP-mac"
return [l for l in l10n_changesets.keys() if l != ignore]
-def task_for_revision(index, branch, revision, product, platform):
- return index.findTask(
- "gecko.v2.{branch}.revision.{rev}.{product}.{platform}-opt".format(
- rev=revision, branch=branch, product=product, platform=platform))
-
-
def get_l10n_config(index, product, branch, revision, platforms,
- l10n_platforms, l10n_changesets):
+ l10n_platforms, l10n_changesets, tc_task_indexes):
l10n_platform_configs = {}
for platform in l10n_platforms:
- task = task_for_revision(index, branch, revision, product, platform)
- url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build".format(
- taskid=task["taskId"]
+ route = tc_task_indexes[platform]['signed'].format(rev=revision)
+ signed_task = index.findTask(route)
+ en_us_binary_url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build".format(
+ taskid=signed_task["taskId"]
)
+ if tc_task_indexes[platform]['unsigned'] != tc_task_indexes[platform]['signed']:
+ # TC based builds use different tasks for en-US (signed) and
+ # martools (unsigned)
+ unsigned_route = tc_task_indexes[platform]['unsigned'].format(rev=revision)
+ unsigned_task = index.findTask(unsigned_route)
+ mar_tools_url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build/host/bin".format(
+ taskid=unsigned_task["taskId"]
+ )
+ else:
+ mar_tools_url = en_us_binary_url
+
l10n_platform_configs[platform] = {
"locales": get_platform_locales(l10n_changesets, platform),
- "en_us_binary_url": url,
+ "en_us_binary_url": en_us_binary_url,
+ "mar_tools_url": mar_tools_url,
"chunks": platforms[platform].get("l10n_chunks", 10),
}
return {
"platforms": l10n_platform_configs,
"changesets": l10n_changesets,
}
-def get_en_US_config(index, product, branch, revision, platforms):
+def get_en_US_config(index, product, branch, revision, platforms,
+ tc_task_indexes):
platform_configs = {}
for platform in platforms:
- task = task_for_revision(index, branch, revision, product, platform)
+ signed_route = tc_task_indexes[platform]['signed'].format(rev=revision)
+ signed_task = index.findTask(signed_route)
+ unsigned_route = tc_task_indexes[platform]['unsigned'].format(rev=revision)
+ unsigned_task = index.findTask(unsigned_route)
platform_configs[platform] = {
- "task_id": task["taskId"]
+ "signed_task_id": signed_task["taskId"],
+ "unsigned_task_id": unsigned_task["taskId"],
}
return {
"platforms": platform_configs,
}
# FIXME: the following function should be removed and we should use
--- a/lib/python/kickoff/build_status.py
+++ b/lib/python/kickoff/build_status.py
@@ -1,99 +1,110 @@
from datetime import datetime, timedelta
from dateutil import parser, tz
from taskcluster.exceptions import TaskclusterRestFailure
-from kickoff import task_for_revision
-
import logging
log = logging.getLogger(__name__)
_BUILD_WATCHERS = {}
# TODO: Bug 1300147. Avoid having 7 parameters by using a release object that
# contains only what's needed.
-def are_en_us_builds_completed(index, release_name, submitted_at, branch,
- revision, tc_product_name, platforms, queue):
+def are_en_us_builds_completed(index, release_name, submitted_at, revision,
+ platforms, queue, tc_task_indexes):
try:
watcher = _BUILD_WATCHERS[release_name]
except KeyError:
- watcher = EnUsBuildsWatcher(index, release_name, submitted_at, branch,
- revision, tc_product_name, platforms, queue)
+ watcher = EnUsBuildsWatcher(index, release_name, submitted_at,
+ revision, platforms, queue,
+ tc_task_indexes)
_BUILD_WATCHERS[release_name] = watcher
log.debug('New watcher created for "%s"', release_name)
result = watcher.are_builds_completed()
if result is True:
del _BUILD_WATCHERS[release_name]
- log.debug('Builds for "%s" are completed. Watcher deleted', release_name)
+ log.debug('Builds for "%s" are completed. Watcher deleted',
+ release_name)
return result
class LoggedError(Exception):
def __init__(self, message):
log.exception(message)
Exception.__init__(self, message)
class EnUsBuildsWatcher:
# TODO: Bug 1300147 as well
- def __init__(self, index, release_name, submitted_at, branch, revision,
- tc_product_name, platforms, queue):
+ def __init__(self, index, release_name, submitted_at, revision,
+ platforms, queue, tc_task_indexes):
self.taskcluster_index = index
- self.taskcluster_product_name = tc_product_name
self.release_name = release_name
- self.branch = branch
self.revision = revision
self.task_per_platform = {p: None for p in platforms}
self.queue = queue
+ self.tc_task_indexes = tc_task_indexes
self._timeout_watcher = TimeoutWatcher(start_timestamp=submitted_at)
def are_builds_completed(self):
if self._timeout_watcher.timed_out:
- raise TimeoutWatcher.TimeoutError(self.release_name, self._timeout_watcher.start_timestamp)
+ raise TimeoutWatcher.TimeoutError(
+ self.release_name, self._timeout_watcher.start_timestamp)
self._fetch_completed_tasks()
return len(self._platforms_with_no_task) == 0
def _fetch_completed_tasks(self):
platforms_with_no_task = self._platforms_with_no_task
- log.debug('Release "%s" still has to find tasks for %s', self.release_name, platforms_with_no_task)
+ log.debug('Release "%s" still has to find tasks for %s',
+ self.release_name, platforms_with_no_task)
for platform in platforms_with_no_task:
try:
- # Tasks are always completed if they are referenced in the index
- # https://docs.taskcluster.net/reference/core/index
- task_id = task_for_revision(
- self.taskcluster_index, self.branch, self.revision, self.taskcluster_product_name, platform
- )['taskId']
+ # Tasks are always completed if they are referenced in the
+ # index https://docs.taskcluster.net/reference/core/index
+ # Assuming that the signed tasks are completed after their
+ # unsigned counterparts
+ route = self.tc_task_indexes[platform]['signed'].format(
+ rev=self.revision)
+ task_id = self.taskcluster_index.findTask(route)['taskId']
# Bug 1307326 - consider only tasks indexed with rank > 0
task = self.queue.task(task_id)
- rank = task["extra"]["index"]["rank"]
- if rank == 0:
- log.debug("Ignoring task %s because the rank is set to 0",
- task_id)
+ rank = task["extra"].get("index", {}).get("rank")
+ tier = task["extra"].get("treeherder", {}).get("tier")
+ if rank is None:
+ eligible = tier == 1
+ else:
+ eligible = rank != 0
+ if not eligible:
+ log.debug("Ignoring task %s because rank (%s) or tier (%s)",
+ task_id, rank, tier)
continue
except TaskclusterRestFailure:
- log.debug('Task for platform "%s" is not yet created for release "%s"', platform, self.release_name)
+ log.debug('Task for platform "%s" is not yet created for '
+ 'release "%s"', platform, self.release_name)
continue
- log.debug('Task "%s" was found for release "%s" and platform "%s"', task_id, self.release_name, platform)
+ log.debug('Task "%s" was found for release "%s" and platform "%s"',
+ task_id, self.release_name, platform)
self.task_per_platform[platform] = task_id
@property
def _platforms_with_no_task(self):
- return [platform for platform, task in self.task_per_platform.iteritems() if task is None]
+ return [platform for platform, task in
+ self.task_per_platform.iteritems() if task is None]
class TimeoutWatcher:
MAX_WAITING_TIME = timedelta(days=1)
def __init__(self, start_timestamp):
self.start_timestamp = parser.parse(start_timestamp)
@@ -105,15 +116,17 @@ class TimeoutWatcher:
@property
def timed_out(self):
return self._now() - self.start_timestamp >= self.MAX_WAITING_TIME
class TimeoutError(LoggedError):
def __init__(self, release_name, start_timestamp):
LoggedError.__init__(
self,
- '{} has spent more than {} between the release being submitted on ship-it (at {} [UTC]) and now.'
- .format(release_name, TimeoutWatcher.MAX_WAITING_TIME, start_timestamp)
+ '{} has spent more than {} between the release being submitted'
+ ' on ship-it (at {} [UTC]) and now.'.format(
+ release_name, TimeoutWatcher.MAX_WAITING_TIME,
+ start_timestamp)
)
class AlreadyStartedError(Exception):
# Common error, there's no need to log it.
pass