--- a/buildfarm/release/release-runner.py
+++ b/buildfarm/release/release-runner.py
@@ -13,27 +13,28 @@ import shutil
import tempfile
import requests
from os import path
from optparse import OptionParser
from twisted.python.lockfile import FilesystemLock
site.addsitedir(path.join(path.dirname(__file__), "../../lib/python"))
-from kickoff.api import Releases, Release, ReleaseL10n
+from kickoff import get_partials, ReleaseRunner, make_task_graph_strict_kwargs
+from kickoff import get_l10n_config, get_en_US_config
+from kickoff import email_release_drivers
+from kickoff import bump_version
from release.info import readBranchConfig
from release.l10n import parsePlainL10nChangesets
from release.versions import getAppVersion
-from releasetasks import make_task_graph
from taskcluster import Scheduler, Index, Queue
from taskcluster.utils import slugId
from util.hg import mercurial
from util.retry import retry
from util.file import load_config, get_config
-from util.sendmail import sendmail
log = logging.getLogger(__name__)
# both CHECKSUMS and ALL_FILES have been defined to improve the release sanity
# en-US binaries timing by whitelisting artifacts of interest - bug 1251761
CHECKSUMS = set([
'.checksums',
@@ -47,52 +48,20 @@ ALL_FILES = set([
'.complete.mar',
'.exe',
'.dmg',
'i686.tar.bz2',
'x86_64.tar.bz2',
])
-# temporary regex to filter out anything but mozilla-beta and mozilla-release
-# within release promotion. Once migration to release promotion is completed
-# for all types of releases, we will backout this filtering
-# regex beta tracking bug is 1252333,
-# regex release tracking bug is 1263976
-RELEASE_PATTERNS = [
- r"Firefox-\d+\.0b\d+-build\d+",
- r"Firefox-\d+\.\d+(\.\d+)?-build\d+"
-]
-
-
class SanityException(Exception):
pass
-# FIXME: the following function should be removed and we should use
-# next_version provided by ship-it
-def bump_version(version):
- """Bump last digit"""
- split_by = "."
- digit_index = 2
- if "b" in version:
- split_by = "b"
- digit_index = 1
- v = version.split(split_by)
- if len(v) < digit_index + 1:
- # 45.0 is 45.0.0 actually
- v.append("0")
- v[-1] = str(int(v[-1]) + 1)
- return split_by.join(v)
-
-
-def matches(name, patterns):
- return any([re.search(p, name) for p in patterns])
-
-
def is_candidate_release(channels):
"""determine if this is a candidate release or not
Because ship-it can not tell us if this is a candidate release (yet!), we assume it is when we
have determined, based on version, that we are planning to ship to more than one update_channel
e.g. for candidate releases we have:
1) one channel to test the 'candidate' release with: to 'beta' channel users
2) once verified, we ship to the main channel: to 'release' channel users
@@ -131,187 +100,16 @@ def get_display_version(repo_path, revis
def _get():
req = requests.get(url, timeout=60)
req.raise_for_status()
return req.content.strip()
return retry(_get)
-def long_revision(repo, revision):
- """Convert short revision to long using JSON API
-
- >>> long_revision("releases/mozilla-beta", "59f372c35b24")
- u'59f372c35b2416ac84d6572d64c49227481a8a6c'
-
- >>> long_revision("releases/mozilla-beta", "59f372c35b2416ac84d6572d64c49227481a8a6c")
- u'59f372c35b2416ac84d6572d64c49227481a8a6c'
- """
- url = "https://hg.mozilla.org/{}/json-rev/{}".format(repo, revision)
-
- def _get():
- req = requests.get(url, timeout=60)
- req.raise_for_status()
- return req.json()["node"]
-
- return retry(_get)
-
-
-class ReleaseRunner(object):
- def __init__(self, api_root=None, username=None, password=None,
- timeout=60):
- self.new_releases = []
- self.releases_api = Releases((username, password), api_root=api_root,
- timeout=timeout)
- self.release_api = Release((username, password), api_root=api_root,
- timeout=timeout)
- self.release_l10n_api = ReleaseL10n((username, password),
- api_root=api_root, timeout=timeout)
-
- def get_release_requests(self):
- new_releases = self.releases_api.getReleases()
- if new_releases['releases']:
- new_releases = [self.release_api.getRelease(name) for name in
- new_releases['releases']]
- our_releases = [r for r in new_releases if
- matches(r['name'], RELEASE_PATTERNS)]
- if our_releases:
- # make sure to use long revision
- for r in our_releases:
- r["mozillaRevision"] = long_revision(r["branch"], r["mozillaRevision"])
- self.new_releases = our_releases
- log.info("Releases to handle are %s", self.new_releases)
- return True
- else:
- log.info("No releases to handle in %s", new_releases)
- return False
- else:
- log.info("No new releases: %s" % new_releases)
- return False
-
- def get_release_l10n(self, release):
- return self.release_l10n_api.getL10n(release)
-
- def update_status(self, release, status):
- log.info('updating status for %s to %s' % (release['name'], status))
- try:
- self.release_api.update(release['name'], status=status)
- except requests.HTTPError, e:
- log.warning('Caught HTTPError: %s' % e.response.content)
- log.warning('status update failed, continuing...', exc_info=True)
-
- def mark_as_completed(self, release):#, enUSPlatforms):
- log.info('mark as completed %s' % release['name'])
- self.release_api.update(release['name'], complete=True,
- status='Started')
-
- def mark_as_failed(self, release, why):
- log.info('mark as failed %s' % release['name'])
- self.release_api.update(release['name'], ready=False, status=why)
-
-
-def getPartials(rr, release):
- partials = {}
- for p in release['partials'].split(','):
- partialVersion, buildNumber = p.split('build')
- partial_release_name = '{}-{}-build{}'.format(
- release['product'].capitalize(), partialVersion, buildNumber,
- )
- partials[partialVersion] = {
- 'appVersion': getAppVersion(partialVersion),
- 'buildNumber': buildNumber,
- 'locales': parsePlainL10nChangesets(
- rr.get_release_l10n(partial_release_name)).keys(),
- }
- return partials
-
-
-def email_release_drivers(smtp_server, from_, to, release, graph_id):
- # Send an email to the mailing after the build
-
- content = """\
-A new build has been submitted through ship-it:
-
-Commit: https://hg.mozilla.org/{path}/rev/{revision}
-Task graph: https://tools.taskcluster.net/task-graph-inspector/#{task_graph_id}/
-
-Created by {submitter}
-Started by {starter}
-
-
-""".format(path=release["branch"], revision=release["mozillaRevision"],
- submitter=release["submitter"], starter=release["starter"],
- task_graph_id=graph_id)
-
- comment = release.get("comment")
- if comment:
- content += "Comment:\n" + comment + "\n\n"
-
- # On r-d, we prefix the subject of the email in order to simplify filtering
- if "Fennec" in release["name"]:
- subject_prefix = "[mobile] "
- if "Firefox" in release["name"]:
- subject_prefix = "[desktop] "
-
- subject = subject_prefix + 'Build of %s' % release["name"]
-
- sendmail(from_=from_, to=to, subject=subject, body=content,
- smtp_server=smtp_server)
-
-
-def get_platform_locales(l10n_changesets, platform):
- # hardcode ja/ja-JP-mac exceptions
- if platform == "macosx64":
- ignore = "ja"
- else:
- ignore = "ja-JP-mac"
-
- return [l for l in l10n_changesets.keys() if l != ignore]
-
-
-def get_l10n_config(release, branchConfig, branch, l10n_changesets, index):
- l10n_platforms = {}
- for platform in branchConfig["l10n_release_platforms"]:
- task = index.findTask("buildbot.revisions.{revision}.{branch}.{platform}".format(
- revision=release["mozillaRevision"],
- branch=branch,
- platform=platform,
- ))
- url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build".format(
- taskid=task["taskId"]
- )
- l10n_platforms[platform] = {
- "locales": get_platform_locales(l10n_changesets, platform),
- "en_us_binary_url": url,
- "chunks": branchConfig["platforms"][platform].get("l10n_chunks", 10),
- }
-
- return {
- "platforms": l10n_platforms,
- "changesets": l10n_changesets,
- }
-
-
-def get_en_US_config(release, branchConfig, branch, index):
- platforms = {}
- for platform in branchConfig["release_platforms"]:
- task = index.findTask("buildbot.revisions.{revision}.{branch}.{platform}".format(
- revision=release["mozillaRevision"],
- branch=branch,
- platform=platform,
- ))
- platforms[platform] = {
- "task_id": task["taskId"],
- }
-
- return {
- "platforms": platforms,
- }
-
-
def validate_version(repo_path, revision, version):
actual_version = get_display_version(repo_path, revision)
if version != actual_version:
raise SanityException(
"In-tree version '%s' doesn't match ship-it version '%s'" %
(actual_version, version))
else:
log.info("In-tree version '%s' matches ship-it version '%s'",
@@ -589,50 +387,57 @@ def main(options):
"buildNumber": release["buildNumber"],
"source_enabled": True,
"checksums_enabled": True,
"repo_path": release["branch"],
"revision": release["mozillaRevision"],
"product": release["product"],
# if mozharness_revision is not passed, use 'revision'
"mozharness_changeset": release.get('mh_changeset') or release['mozillaRevision'],
- "partial_updates": getPartials(rr, release),
+ "partial_updates": get_partials(rr, release['partials'], release['product']),
"branch": branch,
"updates_enabled": bool(release["partials"]),
- "l10n_config": get_l10n_config(release, branchConfig, branch, l10n_changesets, index),
- "en_US_config": get_en_US_config(release, branchConfig, branch, index),
+ "l10n_config": get_l10n_config(
+ release['mozillaRevision'], branchConfig['platforms'],
+ branchConfig['l10n_release_platforms'], branch, l10n_changesets, index
+ ),
+ "en_US_config": get_en_US_config(
+ release['mozillaRevision'], branchConfig['release_platforms'], branch, index
+ ),
"verifyConfigs": {},
"balrog_api_root": branchConfig["balrog_api_root"],
"funsize_balrog_api_root": branchConfig["funsize_balrog_api_root"],
"balrog_username": balrog_username,
"balrog_password": balrog_password,
"beetmover_aws_access_key_id": beetmover_aws_access_key_id,
"beetmover_aws_secret_access_key": beetmover_aws_secret_access_key,
# TODO: stagin specific, make them configurable
"signing_class": "release-signing",
"bouncer_enabled": branchConfig["bouncer_enabled"],
+ "updates_builder_enabled": branchConfig["updates_builder_enabled"],
+ "update_verify_enabled": branchConfig["update_verify_enabled"],
"release_channels": release_channels,
"final_verify_channels": final_verify_channels,
+ "final_verify_platforms": branchConfig['release_platforms'],
"signing_pvt_key": signing_pvt_key,
"build_tools_repo_path": branchConfig['build_tools_repo_path'],
"push_to_candidates_enabled": branchConfig['push_to_candidates_enabled'],
"postrelease_bouncer_aliases_enabled": postrelease_bouncer_aliases_enabled,
"tuxedo_server_url": branchConfig['tuxedoServerUrl'],
"postrelease_version_bump_enabled": postrelease_enabled,
"push_to_releases_enabled": push_to_releases_enabled,
"push_to_releases_automatic": branchConfig['push_to_releases_automatic'],
"beetmover_candidates_bucket": branchConfig["beetmover_buckets"][release["product"]],
"partner_repacks_platforms": branchConfig.get("partner_repacks_platforms", []),
"l10n_changesets": l10n_changesets,
+ "extra_balrog_submitter_params": extra_balrog_submitter_params,
}
- if extra_balrog_submitter_params:
- kwargs["extra_balrog_submitter_params"] = extra_balrog_submitter_params
validate_graph_kwargs(queue, gpg_key_path, **kwargs)
- graph = make_task_graph(**kwargs)
+ graph = make_task_graph_strict_kwargs(**kwargs)
rr.update_status(release, "Submitting task graph")
log.info("Task graph generated!")
import pprint
log.debug(pprint.pformat(graph, indent=4, width=160))
print scheduler.createTaskGraph(graph_id, graph)
rr.mark_as_completed(release)
email_release_drivers(smtp_server=smtp_server, from_=notify_from,
new file mode 100644
--- /dev/null
+++ b/buildfarm/release/releasetasks_graph_gen.py
@@ -0,0 +1,206 @@
+#!/usr/bin/env python
+
+import logging
+import os
+from optparse import OptionParser
+import site
+import yaml
+
+site.addsitedir(os.path.join(os.path.dirname(__file__), "../../lib/python"))
+
+from kickoff import get_partials, ReleaseRunner, make_task_graph_strict_kwargs
+from kickoff import get_l10n_config, get_en_US_config
+from kickoff import bump_version
+
+from release.versions import getAppVersion
+from util.file import load_config, get_config
+
+from taskcluster import Scheduler, Index, Queue
+from taskcluster.utils import slugId
+
+log = logging.getLogger(__name__)
+
+
+def main(release_runner_config, release_config, tc_config):
+
+ api_root = release_runner_config.get('api', 'api_root')
+ username = release_runner_config.get('api', 'username')
+ password = release_runner_config.get('api', 'password')
+
+ scheduler = Scheduler(tc_config)
+ index = Index(tc_config)
+
+ rr = ReleaseRunner(api_root=api_root, username=username, password=password)
+ graph_id = slugId()
+ log.info('Generating task graph')
+ kwargs = {
+ # release-runner.ini
+ "signing_pvt_key": release_config['signing_pvt_key'],
+ "public_key": release_config['docker_worker_key'],
+ "balrog_username": release_config['balrog_username'],
+ "balrog_password": release_config['balrog_password'],
+ "beetmover_aws_access_key_id": release_config['beetmover_aws_access_key_id'],
+ "beetmover_aws_secret_access_key": release_config['beetmover_aws_secret_access_key'],
+ "signing_class": "release-signing", # TODO: stagin specific, make them configurable
+
+ # ship-it items
+ "version": release_config["version"],
+ "revision": release_config["mozilla_revision"],
+ "mozharness_changeset": release_config["mozharness_revision"] or release_config["mozilla_revision"],
+ "buildNumber": release_config["build_number"],
+ "l10n_changesets": release_config["l10n_changesets"],
+
+ # was branchConfig items
+ "funsize_balrog_api_root": release_config["funsize_balrog_api_root"],
+ "balrog_api_root": release_config["balrog_api_root"],
+ "build_tools_repo_path": release_config['build_tools_repo_path'],
+ "tuxedo_server_url": release_config['tuxedo_server_url'],
+ "beetmover_candidates_bucket": release_config["beetmover_candidates_bucket"],
+ "bouncer_enabled": release_config["bouncer_enabled"],
+ "updates_builder_enabled": release_config["updates_builder_enabled"],
+ "update_verify_enabled": release_config["update_verify_enabled"],
+ "push_to_candidates_enabled": release_config['push_to_candidates_enabled'],
+ "postrelease_bouncer_aliases_enabled": release_config['postrelease_bouncer_aliases_enabled'],
+ "postrelease_version_bump_enabled": release_config['postrelease_version_bump_enabled'],
+ "push_to_releases_automatic": release_config['push_to_releases_automatic'],
+ "partner_repacks_platforms": release_config["partner_repacks_platforms"],
+
+ "repo_path": release_config["repo_path"],
+ "branch": release_config["branch"],
+ "product": release_config["product"],
+ "release_channels": release_config['channels'],
+ "final_verify_channels": release_config['final_verify_channels'],
+ "final_verify_platforms": release_config['final_verify_platforms'],
+ "source_enabled": release_config["source_enabled"],
+ "checksums_enabled": release_config["checksums_enabled"],
+ "updates_enabled": release_config["updates_enabled"],
+ "push_to_releases_enabled": release_config["push_to_releases_enabled"],
+
+ "verifyConfigs": {},
+ "next_version": bump_version(release_config["version"]),
+ "appVersion": getAppVersion(release_config["version"]),
+ "partial_updates": get_partials(rr, release_config["partials"],
+ release_config['product']),
+ # in release-runner.py world we have a concept of branchConfig and release (shipit) vars
+ # todo fix get_en_US_config and en_US_config helper methods to not require both
+ "l10n_config": get_l10n_config(
+ release_config["mozilla_revision"], release_config['platforms'],
+ release_config['l10n_release_platforms'] or {}, release_config["branch"],
+ release_config["l10n_changesets"], index
+ ),
+ "en_US_config": get_en_US_config(
+ release_config["mozilla_revision"], release_config['release_platforms'] or [],
+ release_config["branch"], index
+ ),
+ "extra_balrog_submitter_params": release_config['extra_balrog_submitter_params']
+ }
+
+ graph = make_task_graph_strict_kwargs(**kwargs)
+ log.info("Submitting task graph")
+ import pprint
+ log.info(pprint.pformat(graph, indent=4, width=160))
+ if not options.dry_run:
+ print scheduler.createTaskGraph(graph_id, graph)
+
+
+def get_items_from_common_tc_task(common_task_id, tc_config):
+ tc_task_items = {}
+ queue = Queue(tc_config)
+ task = queue.task(common_task_id)
+ tc_task_items["version"] = task["extra"]["build_props"]["version"]
+ tc_task_items["build_number"] = task["extra"]["build_props"]["build_number"]
+ tc_task_items["mozilla_revision"] = task["extra"]["build_props"]["revision"]
+ return tc_task_items
+
+
+def get_unique_release_items(options, tc_config):
+ unique_items = {}
+
+ if options.common_task_id:
+ # sometimes, we make a release based on a previous release. e.g. a graph that represents
+ # part 2 of a Firefox Release Candidate release
+ # TODO extract partials, mozharness_revision, and l10n_changesets from common taskgroup
+ unique_items.update(get_items_from_common_tc_task(options.common_task_id, tc_config))
+ else:
+ unique_items['version'] = options.version
+ unique_items['build_number'] = options.build_number
+ unique_items['mozilla_revision'] = options.mozilla_revision
+
+ unique_items['partials'] = options.partials
+ unique_items['mozharness_revision'] = options.mozharness_revision
+ # TODO have ability to pass l10n_changesets whether based on previous release or new one
+ unique_items["l10n_changesets"] = {}
+
+ return unique_items
+
+
+def get_release_items_from_runner_config(release_runner_ini):
+ ini_items = {}
+ ini_items['signing_pvt_key'] = release_runner_ini.get('signing', 'pvt_key')
+ ini_items['docker_worker_key'] = release_runner_ini.get('release-runner', 'docker_worker_key')
+ ini_items['balrog_username'] = release_runner_ini.get("balrog", "username")
+ ini_items['balrog_password'] = release_runner_ini.get("balrog", "password")
+ ini_items['beetmover_aws_access_key_id'] = release_runner_ini.get("beetmover", "aws_access_key_id")
+ ini_items['beetmover_aws_secret_access_key'] = release_runner_ini.get("beetmover", "aws_secret_access_key")
+ ini_items['extra_balrog_submitter_params'] = get_config(release_runner_ini, "balrog",
+ "extra_balrog_submitter_params", None)
+ return ini_items
+
+
+def load_branch_and_product_config(config_file):
+ with open(config_file, 'r') as rc_file:
+ return yaml.load(rc_file)
+
+
+if __name__ == '__main__':
+ parser = OptionParser(__doc__)
+ parser.add_option('--release-runner-ini', dest='release_runner_ini',
+ help='ini file that contains things like sensitive credentials')
+ parser.add_option('--branch-and-product-config', dest='branch_and_product_config',
+ help='config items specific to certain product and branch')
+ parser.add_option('--version', dest='version', help='full version of release, e.g. 46.0b1')
+ parser.add_option('--build-number', dest='build_number', help='build number of release')
+ parser.add_option('--partials', type="string", dest='partials',
+ help='list of partials for the release')
+ parser.add_option('--mozilla-revision', dest='mozilla_revision',
+ help='gecko revision to build ff from')
+ parser.add_option('--mozharness-revision', dest='mozharness_revision',
+ help='gecko revision for mozharness')
+ parser.add_option('--common-task-id', dest='common_task_id',
+ help='a task id of a task that shares the same release info')
+ parser.add_option('--dry-run', dest='dry_run', action='store_true', default=False,
+ help="render the task graph from yaml tmpl but don't submit to taskcluster")
+
+ options = parser.parse_args()[0]
+
+ if not options.release_runner_ini:
+ parser.error('Need to pass a release runner config')
+ if not options.branch_and_product_config:
+ parser.error('Need to pass a branch and product config')
+
+ # load config files
+ release_runner_config = load_config(options.release_runner_ini)
+ tc_config = {
+ "credentials": {
+ "clientId": get_config(release_runner_config, "taskcluster", "client_id", None),
+ "accessToken": get_config(release_runner_config, "taskcluster", "access_token", None),
+ }
+ }
+ branch_product_config = load_branch_and_product_config(options.branch_and_product_config)
+
+ if release_runner_config.getboolean('release-runner', 'verbose'):
+ log_level = logging.DEBUG
+ else:
+ log_level = logging.INFO
+ logging.basicConfig(filename='releasetasks_graph_gen.log',
+ format="%(asctime)s - %(levelname)s - %(message)s",
+ level=log_level)
+
+
+ # create releasetasks graph args from config files
+ releasetasks_kwargs = {}
+ releasetasks_kwargs.update(branch_product_config)
+ releasetasks_kwargs.update(get_release_items_from_runner_config(release_runner_config))
+ releasetasks_kwargs.update(get_unique_release_items(options, tc_config))
+
+ main(release_runner_config, releasetasks_kwargs, tc_config)
--- a/lib/python/kickoff/__init__.py
+++ b/lib/python/kickoff/__init__.py
@@ -0,0 +1,280 @@
+import re
+import requests
+import logging
+
+from kickoff.api import Releases, Release, ReleaseL10n
+from releasetasks import make_task_graph
+from release.l10n import parsePlainL10nChangesets
+from release.versions import getAppVersion
+from util.sendmail import sendmail
+from util.retry import retry
+
+log = logging.getLogger(__name__)
+
+# temporary regex to filter out anything but mozilla-beta and mozilla-release
+# within release promotion. Once migration to release promotion is completed
+# for all types of releases, we will backout this filtering
+# regex beta tracking bug is 1252333,
+# regex release tracking bug is 1263976
+RELEASE_PATTERNS = [
+ r"Firefox-\d+\.0b\d+-build\d+",
+ r"Firefox-\d+\.\d+(\.\d+)?-build\d+"
+]
+
+
+def matches(name, patterns):
+ return any([re.search(p, name) for p in patterns])
+
+
+def long_revision(repo, revision):
+ """Convert short revision to long using JSON API
+
+ >>> long_revision("releases/mozilla-beta", "59f372c35b24")
+ u'59f372c35b2416ac84d6572d64c49227481a8a6c'
+
+ >>> long_revision("releases/mozilla-beta", "59f372c35b2416ac84d6572d64c49227481a8a6c")
+ u'59f372c35b2416ac84d6572d64c49227481a8a6c'
+ """
+ url = "https://hg.mozilla.org/{}/json-rev/{}".format(repo, revision)
+
+ def _get():
+ req = requests.get(url, timeout=60)
+ req.raise_for_status()
+ return req.json()["node"]
+
+ return retry(_get)
+
+
+class ReleaseRunner(object):
+ def __init__(self, api_root=None, username=None, password=None,
+ timeout=60):
+ self.new_releases = []
+ self.releases_api = Releases((username, password), api_root=api_root,
+ timeout=timeout)
+ self.release_api = Release((username, password), api_root=api_root,
+ timeout=timeout)
+ self.release_l10n_api = ReleaseL10n((username, password),
+ api_root=api_root, timeout=timeout)
+
+ def get_release_requests(self):
+ new_releases = self.releases_api.getReleases()
+ if new_releases['releases']:
+ new_releases = [self.release_api.getRelease(name) for name in
+ new_releases['releases']]
+ our_releases = [r for r in new_releases if
+ matches(r['name'], RELEASE_PATTERNS)]
+ if our_releases:
+ # make sure to use long revision
+ for r in our_releases:
+ r["mozillaRevision"] = long_revision(r["branch"], r["mozillaRevision"])
+ self.new_releases = our_releases
+ log.info("Releases to handle are %s", self.new_releases)
+ return True
+ else:
+ log.info("No releases to handle in %s", new_releases)
+ return False
+ else:
+ log.info("No new releases: %s" % new_releases)
+ return False
+
+ def get_release_l10n(self, release):
+ return self.release_l10n_api.getL10n(release)
+
+ def update_status(self, release, status):
+ log.info('updating status for %s to %s' % (release['name'], status))
+ try:
+ self.release_api.update(release['name'], status=status)
+ except requests.HTTPError, e:
+ log.warning('Caught HTTPError: %s' % e.response.content)
+ log.warning('status update failed, continuing...', exc_info=True)
+
+ def mark_as_completed(self, release):#, enUSPlatforms):
+ log.info('mark as completed %s' % release['name'])
+ self.release_api.update(release['name'], complete=True,
+ status='Started')
+
+ def mark_as_failed(self, release, why):
+ log.info('mark as failed %s' % release['name'])
+ self.release_api.update(release['name'], ready=False, status=why)
+
+def email_release_drivers(smtp_server, from_, to, release, graph_id):
+ # Send an email to the mailing after the build
+
+ content = """\
+A new build has been submitted through ship-it:
+
+Commit: https://hg.mozilla.org/{path}/rev/{revision}
+Task graph: https://tools.taskcluster.net/task-graph-inspector/#{task_graph_id}/
+
+Created by {submitter}
+Started by {starter}
+
+
+""".format(path=release["branch"], revision=release["mozillaRevision"],
+ submitter=release["submitter"], starter=release["starter"],
+ task_graph_id=graph_id)
+
+ comment = release.get("comment")
+ if comment:
+ content += "Comment:\n" + comment + "\n\n"
+
+ # On r-d, we prefix the subject of the email in order to simplify filtering
+ if "Fennec" in release["name"]:
+ subject_prefix = "[mobile] "
+ if "Firefox" in release["name"]:
+ subject_prefix = "[desktop] "
+
+ subject = subject_prefix + 'Build of %s' % release["name"]
+
+ sendmail(from_=from_, to=to, subject=subject, body=content,
+ smtp_server=smtp_server)
+
+
+def get_partials(rr, partial_versions, product):
+ partials = {}
+ if not partial_versions:
+ return partials
+ for p in partial_versions.split(','):
+ partialVersion, buildNumber = p.split('build')
+ partial_release_name = '{}-{}-build{}'.format(
+ product.capitalize(), partialVersion, buildNumber,
+ )
+ partials[partialVersion] = {
+ 'appVersion': getAppVersion(partialVersion),
+ 'buildNumber': buildNumber,
+ 'locales': parsePlainL10nChangesets(
+ rr.get_release_l10n(partial_release_name)).keys(),
+ }
+ return partials
+
+
+def get_platform_locales(l10n_changesets, platform):
+ # hardcode ja/ja-JP-mac exceptions
+ if platform == "macosx64":
+ ignore = "ja"
+ else:
+ ignore = "ja-JP-mac"
+
+ return [l for l in l10n_changesets.keys() if l != ignore]
+
+
+def get_l10n_config(mozilla_revision, platforms, l10n_platforms, branch, l10n_changesets, index):
+ l10n_platform_configs = {}
+ for platform in l10n_platforms:
+ task = index.findTask("buildbot.revisions.{revision}.{branch}.{platform}".format(
+ revision=mozilla_revision,
+ branch=branch,
+ platform=platform,
+ ))
+ url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build".format(
+ taskid=task["taskId"]
+ )
+ l10n_platform_configs[platform] = {
+ "locales": get_platform_locales(l10n_changesets, platform),
+ "en_us_binary_url": url,
+ "chunks": platforms[platform].get("l10n_chunks", 10),
+ }
+
+ return {
+ "platforms": l10n_platform_configs,
+ "changesets": l10n_changesets,
+ }
+
+
+def get_en_US_config(mozilla_revision, platforms, branch, index):
+ platform_configs = {}
+ for platform in platforms:
+ task = index.findTask("buildbot.revisions.{revision}.{branch}.{platform}".format(
+ revision=mozilla_revision,
+ branch=branch,
+ platform=platform,
+ ))
+ platform_configs[platform] = {
+ "task_id": task["taskId"],
+ }
+
+ return {
+ "platforms": platform_configs,
+ }
+
+
+# FIXME: the following function should be removed and we should use
+# next_version provided by ship-it
+def bump_version(version):
+ """Bump last digit"""
+ split_by = "."
+ digit_index = 2
+ if "b" in version:
+ split_by = "b"
+ digit_index = 1
+ v = version.split(split_by)
+ if len(v) < digit_index + 1:
+ # 45.0 is 45.0.0 actually
+ v.append("0")
+ v[-1] = str(int(v[-1]) + 1)
+ return split_by.join(v)
+
+
+def make_task_graph_strict_kwargs(appVersion, balrog_api_root, balrog_password, balrog_username,
+ beetmover_aws_access_key_id, beetmover_aws_secret_access_key,
+ beetmover_candidates_bucket, bouncer_enabled, branch, buildNumber,
+ build_tools_repo_path, checksums_enabled, en_US_config,
+ extra_balrog_submitter_params, final_verify_channels,
+ final_verify_platforms, funsize_balrog_api_root, l10n_config,
+ l10n_changesets, mozharness_changeset, next_version,
+ partial_updates, partner_repacks_platforms,
+ postrelease_bouncer_aliases_enabled, postrelease_version_bump_enabled,
+ product, public_key, push_to_candidates_enabled,
+ push_to_releases_automatic, push_to_releases_enabled, release_channels,
+ repo_path, revision, signing_class, signing_pvt_key, source_enabled,
+ tuxedo_server_url, update_verify_enabled, updates_builder_enabled,
+ updates_enabled, verifyConfigs, version):
+ """simple wrapper that sanitizes whatever calls make_task_graph uses universally known kwargs"""
+
+ kwargs = dict(
+ appVersion=appVersion,
+ balrog_api_root=balrog_api_root,
+ balrog_password=balrog_password,
+ balrog_username=balrog_username,
+ beetmover_aws_access_key_id=beetmover_aws_access_key_id,
+ beetmover_aws_secret_access_key=beetmover_aws_secret_access_key,
+ beetmover_candidates_bucket=beetmover_candidates_bucket,
+ bouncer_enabled=bouncer_enabled,
+ branch=branch,
+ buildNumber=buildNumber,
+ build_tools_repo_path=build_tools_repo_path,
+ checksums_enabled=checksums_enabled,
+ en_US_config=en_US_config,
+ final_verify_channels=final_verify_channels,
+ final_verify_platforms=final_verify_platforms,
+ funsize_balrog_api_root=funsize_balrog_api_root,
+ l10n_changesets=l10n_changesets,
+ l10n_config=l10n_config,
+ mozharness_changeset=mozharness_changeset,
+ next_version=next_version,
+ partial_updates=partial_updates,
+ partner_repacks_platforms=partner_repacks_platforms,
+ postrelease_bouncer_aliases_enabled=postrelease_bouncer_aliases_enabled,
+ postrelease_version_bump_enabled=postrelease_version_bump_enabled,
+ product=product,
+ public_key=public_key,
+ push_to_candidates_enabled=push_to_candidates_enabled,
+ push_to_releases_automatic=push_to_releases_automatic,
+ push_to_releases_enabled=push_to_releases_enabled,
+ release_channels=release_channels,
+ repo_path=repo_path,
+ revision=revision,
+ signing_class=signing_class,
+ signing_pvt_key=signing_pvt_key,
+ source_enabled=source_enabled,
+ tuxedo_server_url=tuxedo_server_url,
+ update_verify_enabled=update_verify_enabled,
+ updates_builder_enabled=updates_builder_enabled,
+ updates_enabled=updates_enabled,
+ verifyConfigs=verifyConfigs,
+ version=version
+ )
+ if extra_balrog_submitter_params:
+ kwargs["extra_balrog_submitter_params"] = extra_balrog_submitter_params
+
+ return make_task_graph(**kwargs)