copy from build/sparse-profiles/update-verify
copy to build/sparse-profiles/mozharness
--- a/build/sparse-profiles/update-verify
+++ b/build/sparse-profiles/mozharness
@@ -1,5 +1,4 @@
%include build/sparse-profiles/mach
[include]
-path:build/mozrelease
path:testing/mozharness
--- a/build/sparse-profiles/update-verify
+++ b/build/sparse-profiles/update-verify
@@ -1,5 +1,4 @@
-%include build/sparse-profiles/mach
+%include build/sparse-profiles/mozharness
[include]
path:build/mozrelease
-path:testing/mozharness
--- a/taskcluster/ci/release-balrog-publishing/kind.yml
+++ b/taskcluster/ci/release-balrog-publishing/kind.yml
@@ -7,17 +7,17 @@ loader: taskgraph.loader.transform:loade
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.release_balrog_publishing:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- - release-uptake-monitoring
+ - release-bouncer-check
job-defaults:
description: Schedule publishing in balrog
worker-type: buildbot-bridge/buildbot-bridge
run-on-projects: []
shipping-phase: ship
run:
using: buildbot
--- a/taskcluster/ci/release-bouncer-aliases/kind.yml
+++ b/taskcluster/ci/release-bouncer-aliases/kind.yml
@@ -6,17 +6,17 @@ loader: taskgraph.loader.transform:loade
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- - release-uptake-monitoring
+ - release-bouncer-check
job-defaults:
description: Update bouncer aliases job
worker-type: buildbot-bridge/buildbot-bridge
run-on-projects: []
shipping-phase: ship
run:
using: buildbot
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-bouncer-check/kind.yml
@@ -0,0 +1,74 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+ - beetmover-cdns
+
+transforms:
+ - taskgraph.transforms.release_deps:transforms
+ - taskgraph.transforms.bouncer_check:transforms
+ - taskgraph.transforms.release_notifications:transforms
+ - taskgraph.transforms.job:transforms
+ - taskgraph.transforms.task:transforms
+
+job-defaults:
+ name: bouncer-check
+ description: bouncer check
+ run-on-projects: [] # to make sure this never runs as part of CI
+ shipping-phase: push
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ max-run-time: 1200
+ docker-image: {in-tree: "lint"}
+ run:
+ using: run-task
+ sparse-profile: mozharness
+ attributes:
+ build_platform: linux64
+ build_type: opt
+ treeherder:
+ symbol: Rel(BncChk)
+ kind: test
+ tier: 1
+
+jobs:
+ firefox:
+ shipping-product: firefox
+ index:
+ product: firefox
+ job-name: firefox-bouncer-check
+ run:
+ config:
+ by-project:
+ mozilla-release:
+ - releases/bouncer_firefox_release.py
+ mozilla-beta:
+ - releases/bouncer_firefox_beta.py
+ maple:
+ - releases/dev_bouncer_firefox_beta.py
+ default:
+ - releases/bouncer_firefox_dev.py
+ treeherder:
+ platform: linux64/opt
+
+ devedition:
+ shipping-product: devedition
+ index:
+ product: devedition
+ job-name: devedition-bouncer-check
+ run:
+ config:
+ by-project:
+ mozilla-release:
+ - releases/bouncer_devedition_release.py
+ mozilla-beta:
+ - releases/bouncer_devedition_beta.py
+ maple:
+ - releases/dev_bouncer_devedition_beta.py
+ default:
+ - releases/bouncer_devedition_dev.py
+ treeherder:
+ platform: linux64-devedition/opt
--- a/taskcluster/ci/release-final-verify/kind.yml
+++ b/taskcluster/ci/release-final-verify/kind.yml
@@ -1,16 +1,16 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.transform:loader
kind-dependencies:
- - release-uptake-monitoring
+ - release-bouncer-check
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.final_verify:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.task:transforms
job-defaults:
deleted file mode 100644
--- a/taskcluster/ci/release-uptake-monitoring/kind.yml
+++ /dev/null
@@ -1,62 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-loader: taskgraph.loader.transform:loader
-
-transforms:
- - taskgraph.transforms.release_deps:transforms
- - taskgraph.transforms.job:transforms
- - taskgraph.transforms.release_notifications:transforms
- - taskgraph.transforms.task:transforms
-
-kind-dependencies:
- - beetmover-cdns
-
-job-defaults:
- description: Uptake monitoring job
- worker-type: buildbot-bridge/buildbot-bridge
- run-on-projects: []
- shipping-phase: push
- run:
- using: buildbot
- release-promotion: true
- worker:
- properties:
- tuxedo_server_url:
- by-project:
- mozilla-beta: https://bounceradmin.mozilla.com/api
- mozilla-release: https://bounceradmin.mozilla.com/api
- maple: https://admin-bouncer-releng.stage.mozaws.net/api
- default: http://localhost/api
-
-jobs:
- fennec:
- name: fennec_release_uptake_monitoring
- shipping-product: fennec
- run:
- product: fennec
- buildername: release-{branch}-fennec_uptake_monitoring
- worker:
- properties:
- platforms: "android-api-16, android-x86"
-
- firefox:
- name: firefox_release_uptake_monitoring
- shipping-product: firefox
- run:
- product: firefox
- buildername: release-{branch}-firefox_uptake_monitoring
- worker:
- properties:
- platforms: "linux, linux64, win32, win64, macosx64"
-
- devedition:
- name: devedition_release_uptake_monitoring
- shipping-product: devedition
- run:
- product: devedition
- buildername: release-{branch}-devedition_uptake_monitoring
- worker:
- properties:
- platforms: "linux, linux64, win32, win64, macosx64"
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -268,17 +268,21 @@ release-bouncer-sub
Submits bouncer updates for releases.
release-mark-as-shipped
-----------------------
Marks releases as shipped in Ship-It.
release-bouncer-aliases
------------------------------
-Update Bouncers (download.mozilla.org) "latest" aliases.
+Update Bouncer's (download.mozilla.org) "latest" aliases.
+
+release-bouncer-check
+------------------------------
+Checks Bouncer (download.mozilla.org) uptake.
release-generate-checksums
--------------------------
Generate the per-release checksums along with the summaries and upload it to S3.
release-final-verify
---------------------
Verifies the contents and package of release update MARs.
@@ -298,20 +302,16 @@ Verifies the contents and package of rel
release-secondary-update-verify
---------------------
Verifies the contents and package of release update MARs.
release-updates-builder
-----------------------
Top level Balrog blob submission & patcher/update verify config updates.
-release-uptake-monitoring
--------------------------
-Run uptake monitoring for releases.
-
release-version-bump
--------------------
Bumps to the next version.
release-source
--------------------
Generates source for the release
--- a/taskcluster/taskgraph/actions/release_promotion.py
+++ b/taskcluster/taskgraph/actions/release_promotion.py
@@ -71,25 +71,23 @@ RELEASE_PROMOTION_CONFIG = {
}
VERSION_BUMP_FLAVORS = (
'ship_fennec',
'ship_firefox',
'ship_devedition',
)
-UPTAKE_MONITORING_PLATFORMS_FLAVORS = (
- 'push_firefox',
- 'push_devedition',
-)
-
-PARTIAL_UPDATES_FLAVORS = UPTAKE_MONITORING_PLATFORMS_FLAVORS + (
+PARTIAL_UPDATES_FLAVORS = (
'promote_firefox',
'promote_firefox_rc',
'promote_devedition',
+ 'ship_firefox',
+ 'ship_firefox_rc',
+ 'ship_devedition',
)
def is_release_promotion_available(parameters):
return parameters['project'] in RELEASE_PROMOTION_PROJECTS
@register_callback_action(
@@ -198,31 +196,16 @@ def is_release_promotion_available(param
'required': [
'buildNumber',
'locales',
],
'additionalProperties': False,
}
},
- 'uptake_monitoring_platforms': {
- 'type': 'array',
- 'items': {
- 'type': 'string',
- 'enum': [
- 'macosx',
- 'win32',
- 'win64',
- 'linux',
- 'linux64',
- ],
- },
- 'default': [],
- },
-
'release_eta': {
'type': 'string',
'default': '',
},
},
"required": ['release_promotion_flavor', 'build_number'],
}
)
@@ -251,25 +234,16 @@ def release_promotion_action(parameters,
)
balrog_prefix = product.title()
os.environ['PARTIAL_UPDATES'] = partial_updates
release_history = populate_release_history(
balrog_prefix, parameters['project'],
partial_updates=input['partial_updates']
)
- if release_promotion_flavor in UPTAKE_MONITORING_PLATFORMS_FLAVORS:
- uptake_monitoring_platforms = json.dumps(input.get('uptake_monitoring_platforms', []))
- if partial_updates == "[]":
- raise Exception(
- "`uptake_monitoring_platforms` property needs to be provided for %s "
- "targets." % ', '.join(UPTAKE_MONITORING_PLATFORMS_FLAVORS)
- )
- os.environ['UPTAKE_MONITORING_PLATFORMS'] = uptake_monitoring_platforms
-
promotion_config = RELEASE_PROMOTION_CONFIG[release_promotion_flavor]
target_tasks_method = promotion_config['target_tasks_method'].format(
project=parameters['project']
)
rebuild_kinds = input.get(
'rebuild_kinds', promotion_config.get('rebuild_kinds', [])
)
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/bouncer_check.py
@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+import copy
+import subprocess
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.scriptworker import get_release_config
+from taskgraph.util.schema import (
+ resolve_keyed_by,
+)
+
+import logging
+logger = logging.getLogger(__name__)
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def add_command(config, jobs):
+ release_config = get_release_config(config)
+ version = release_config["version"]
+ for job in jobs:
+ job = copy.deepcopy(job) # don't overwrite dict values here
+ command = [
+ "cd", "/builds/worker/checkouts/gecko", "&&",
+ "./mach", "python",
+ "testing/mozharness/scripts/release/bouncer_check.py",
+ "--version={}".format(version),
+ ]
+ job["run"]["command"] = command
+ yield job
+
+
+@transforms.add
+def add_previous_versions(config, jobs):
+ release_config = get_release_config(config)
+ if not release_config.get("partial_versions"):
+ for job in jobs:
+ yield job
+ else:
+ extra_params = []
+ for partial in release_config["partial_versions"].split(","):
+ extra_params.append("--previous-version={}".format(partial.split("build")[0]))
+
+ for job in jobs:
+ job = copy.deepcopy(job) # don't overwrite dict values here
+ job["run"]["command"].extend(extra_params)
+ yield job
+
+
+@transforms.add
+def handle_keyed_by(config, jobs):
+ """Resolve fields that can be keyed by project, etc."""
+ fields = [
+ "run.config",
+ ]
+ for job in jobs:
+ job = copy.deepcopy(job) # don't overwrite dict values here
+ for field in fields:
+ resolve_keyed_by(item=job, field=field, item_name=job['name'],
+ project=config.params['project'])
+
+ for cfg in job["run"]["config"]:
+ job["run"]["command"].extend(["--config", cfg])
+
+ del job["run"]["config"]
+ yield job
+
+
+@transforms.add
+def command_to_string(config, jobs):
+ """Convert command to string to make it work properly with run-task"""
+ for job in jobs:
+ job = copy.deepcopy(job) # don't overwrite dict values here
+ job["run"]["command"] = subprocess.list2cmdline(job["run"]["command"])
+ yield job
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -450,35 +450,27 @@ def get_release_config(config):
Returns:
dict: containing both `build_number` and `version`. This can be used to
update `task.payload`.
"""
release_config = {}
partial_updates = os.environ.get("PARTIAL_UPDATES", "")
if partial_updates != "" and config.kind in ('release-bouncer-sub',
- 'release-uptake-monitoring',
+ 'release-bouncer-check',
'release-updates-builder',
):
partial_updates = json.loads(partial_updates)
release_config['partial_versions'] = ', '.join([
'{}build{}'.format(v, info['buildNumber'])
for v, info in partial_updates.items()
])
if release_config['partial_versions'] == "{}":
del release_config['partial_versions']
- uptake_monitoring_platforms = os.environ.get("UPTAKE_MONITORING_PLATFORMS", "[]")
- if uptake_monitoring_platforms != "[]" and \
- config.kind in ('release-uptake-monitoring',):
- uptake_monitoring_platforms = json.loads(uptake_monitoring_platforms)
- release_config['platforms'] = ', '.join(uptake_monitoring_platforms)
- if release_config['platforms'] == "[]":
- del release_config['platforms']
-
release_config['version'] = str(config.params['version'])
release_config['appVersion'] = str(config.params['app_version'])
release_config['next_version'] = str(config.params['next_version'])
release_config['build_number'] = config.params['build_number']
return release_config
--- a/testing/mozharness/configs/releases/bouncer_fennec.py
+++ b/testing/mozharness/configs/releases/bouncer_fennec.py
@@ -1,10 +1,11 @@
# lint_ignore=E501
config = {
+ "bouncer_prefix": "https://download.mozilla.org/",
"products": {
"apk": {
"product-name": "Fennec-%(version)s",
"check_uptake": True,
"alias": "fennec-latest",
"ssl-only": True,
"add-locales": False, # Do not add locales to let "multi" work
"paths": {
--- a/testing/mozharness/configs/releases/bouncer_fennec_beta.py
+++ b/testing/mozharness/configs/releases/bouncer_fennec_beta.py
@@ -1,10 +1,11 @@
# lint_ignore=E501
config = {
+ "bouncer_prefix": "https://download.mozilla.org/",
"products": {
"apk": {
"product-name": "Fennec-%(version)s",
"check_uptake": True,
"alias": "fennec-beta-latest",
"ssl-only": True,
"add-locales": False, # Do not add locales to let "multi" work
"paths": {
--- a/testing/mozharness/configs/releases/bouncer_firefox_beta.py
+++ b/testing/mozharness/configs/releases/bouncer_firefox_beta.py
@@ -1,11 +1,12 @@
# lint_ignore=E501
config = {
"shipped-locales-url": "https://hg.mozilla.org/%(repo)s/raw-file/%(revision)s/browser/locales/shipped-locales",
+ "bouncer_prefix": "https://download.mozilla.org/",
"products": {
"installer": {
"product-name": "Firefox-%(version)s",
"check_uptake": True,
"alias": "firefox-beta-latest",
"ssl-only": False,
"add-locales": True,
"paths": {
--- a/testing/mozharness/configs/releases/bouncer_firefox_devedition.py
+++ b/testing/mozharness/configs/releases/bouncer_firefox_devedition.py
@@ -1,11 +1,12 @@
# lint_ignore=E501
config = {
"shipped-locales-url": "https://hg.mozilla.org/%(repo)s/raw-file/%(revision)s/browser/locales/shipped-locales",
+ "bouncer_prefix": "https://download.mozilla.org/",
"products": {
"installer": {
"product-name": "Devedition-%(version)s",
"check_uptake": True,
"alias": "firefox-devedition-latest",
"ssl-only": False,
"add-locales": True,
"paths": {
--- a/testing/mozharness/configs/releases/bouncer_firefox_esr.py
+++ b/testing/mozharness/configs/releases/bouncer_firefox_esr.py
@@ -1,11 +1,12 @@
# lint_ignore=E501
config = {
"shipped-locales-url": "https://hg.mozilla.org/%(repo)s/raw-file/%(revision)s/browser/locales/shipped-locales",
+ "bouncer_prefix": "https://download.mozilla.org/",
"products": {
"installer": {
"product-name": "Firefox-%(version)s",
"check_uptake": True,
"alias": "firefox-esr-latest",
"ssl-only": True,
"add-locales": True,
"paths": {
--- a/testing/mozharness/configs/releases/bouncer_firefox_release.py
+++ b/testing/mozharness/configs/releases/bouncer_firefox_release.py
@@ -1,11 +1,12 @@
# lint_ignore=E501
config = {
"shipped-locales-url": "https://hg.mozilla.org/%(repo)s/raw-file/%(revision)s/browser/locales/shipped-locales",
+ "bouncer_prefix": "https://download.mozilla.org/",
"products": {
"installer": {
"product-name": "Firefox-%(version)s",
"check_uptake": True,
"alias": "firefox-latest",
"ssl-only": False,
"add-locales": True,
"paths": {
--- a/testing/mozharness/configs/releases/bouncer_thunderbird.py
+++ b/testing/mozharness/configs/releases/bouncer_thunderbird.py
@@ -1,11 +1,12 @@
# lint_ignore=E501
config = {
"shipped-locales-url": "https://hg.mozilla.org/%(repo)s/raw-file/%(revision)s/mail/locales/shipped-locales",
+ "bouncer_prefix": "https://download.mozilla.org/",
"products": {
"installer": {
"product-name": "Thunderbird-%(version)s",
"check_uptake": True,
"alias": "thunderbird-latest",
"ssl-only": False,
"add-locales": True,
"paths": {
--- a/testing/mozharness/configs/releases/dev_bouncer_firefox_beta.py
+++ b/testing/mozharness/configs/releases/dev_bouncer_firefox_beta.py
@@ -1,10 +1,11 @@
# lint_ignore=E501
config = {
+ "bouncer_prefix": "https://bouncer-bouncer-releng.stage.mozaws.net/",
"products": {
"installer": {
"product-name": "Firefox-%(version)s",
"check_uptake": True,
"alias": "firefox-beta-latest",
"ssl-only": False,
"add-locales": False,
"paths": {
--- a/testing/mozharness/configs/releases/dev_bouncer_firefox_devedition.py
+++ b/testing/mozharness/configs/releases/dev_bouncer_firefox_devedition.py
@@ -1,11 +1,12 @@
# lint_ignore=E501
config = {
"shipped-locales-url": "https://hg.mozilla.org/%(repo)s/raw-file/%(revision)s/browser/locales/shipped-locales",
+ "bouncer_prefix": "https://bouncer-bouncer-releng.stage.mozaws.net/",
"products": {
"installer": {
"product-name": "Devedition-%(version)s",
"check_uptake": True,
"alias": "firefox-devedition-latest",
"ssl-only": False,
"add-locales": True,
"paths": {
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/scripts/release/bouncer_check.py
@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+# lint_ignore=E501
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+""" bouncer_check.py
+
+A script to check HTTP statuses of Bouncer products to be shipped.
+"""
+
+import os
+import sys
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+
+from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
+from mozharness.base.script import BaseScript
+
+BOUNCER_URL_PATTERN = "{bouncer_prefix}?product={product}&os={os}&lang={lang}"
+
+
+class BouncerCheck(BaseScript, VirtualenvMixin):
+ config_options = [
+ [["--version"], {
+ "dest": "version",
+ "help": "Version of release, eg: 39.0b5",
+ }],
+ [["--previous-version"], {
+ "dest": "prev_versions",
+ "action": "extend",
+ "help": "Previous version(s)",
+ }],
+ [["--locale"], {
+ "dest": "locales",
+ # Intentionally limited for several reasons:
+ # 1) faster to check
+ # 2) do not need to deal with situation when a new locale
+ # introduced and we do not have partials for it yet
+ # 3) it mimics the old Sentry behaviour that worked for ages
+ # 4) no need to handle ja-JP-mac
+ "default": ["en-US", "de", "it", "zh-TW"],
+ "action": "append",
+ "help": "List of locales to check.",
+ }],
+ [["-j", "--parallelization"], {
+ "dest": "parallelization",
+ "default": 20,
+ "type": int,
+ "help": "Number of HTTP sessions running in parallel",
+ }],
+ ] + virtualenv_config_options
+
+ def __init__(self, require_config_file=True):
+ super(BouncerCheck, self).__init__(
+ config_options=self.config_options,
+ require_config_file=require_config_file,
+ config={
+ "virtualenv_modules": [
+ "redo",
+ "requests",
+ "futures==3.1.1",
+ ],
+ "virtualenv_path": "venv",
+ },
+ all_actions=[
+ "create-virtualenv",
+ "activate-virtualenv",
+ "check-bouncer",
+ ],
+ default_actions=[
+ "create-virtualenv",
+ "activate-virtualenv",
+ "check-bouncer",
+ ],
+ )
+
+ def check_url(self, session, url):
+ from redo import retry
+
+ def do_check_url():
+ self.log("Checking {}".format(url))
+ r = session.head(url, verify=True, timeout=10, allow_redirects=True)
+ try:
+ r.raise_for_status()
+ except Exception:
+ self.warning("FAIL: {}, status: {}".format(url, r.status_code))
+ raise
+
+ retry(do_check_url, sleeptime=3, max_sleeptime=10, attempts=3)
+
+ def get_urls(self):
+ for product in self.config["products"].values():
+ if not product["check_uptake"]:
+ continue
+ product_name = product["product-name"] % {"version": self.config["version"]}
+ for path in product["paths"].values():
+ bouncer_platform = path["bouncer-platform"]
+ for locale in self.config["locales"]:
+ url = BOUNCER_URL_PATTERN.format(
+ bouncer_prefix=self.config["bouncer_prefix"],
+ product=product_name,
+ os=bouncer_platform,
+ lang=locale,
+ )
+ yield url
+
+ for product in self.config.get("partials", {}).values():
+ if not product["check_uptake"]:
+ continue
+ for prev_version in self.config.get("prev_versions", []):
+ product_name = product["product-name"] % {"version": self.config["version"],
+ "prev_version": prev_version}
+ for path in product["paths"].values():
+ bouncer_platform = path["bouncer-platform"]
+ for locale in self.config["locales"]:
+ url = BOUNCER_URL_PATTERN.format(
+ bouncer_prefix=self.config["bouncer_prefix"],
+ product=product_name,
+ os=bouncer_platform,
+ lang=locale,
+ )
+ yield url
+
+ def check_bouncer(self):
+ import requests
+ import concurrent.futures as futures
+ session = requests.Session()
+ http_adapter = requests.adapters.HTTPAdapter(
+ pool_connections=self.config["parallelization"],
+ pool_maxsize=self.config["parallelization"])
+ session.mount('https://', http_adapter)
+ session.mount('http://', http_adapter)
+
+ with futures.ThreadPoolExecutor(self.config["parallelization"]) as e:
+ fs = []
+ for url in self.get_urls():
+ fs.append(e.submit(self.check_url, session, url))
+ for f in futures.as_completed(fs):
+ f.result()
+
+
+if __name__ == '__main__':
+ BouncerCheck().run_and_exit()
deleted file mode 100644
--- a/testing/mozharness/scripts/release/uptake_monitoring.py
+++ /dev/null
@@ -1,189 +0,0 @@
-#!/usr/bin/env python
-# lint_ignore=E501
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-""" uptake_monitoring.py
-
-A script to replace the old-fashion way of computing the uptake monitoring
-from the scheduler within the slaves.
-"""
-
-import os
-import sys
-import datetime
-import time
-import xml.dom.minidom
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-
-from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
-from mozharness.base.script import BaseScript
-from mozharness.mozilla.buildbot import BuildbotMixin
-
-
-def get_tuxedo_uptake_url(tuxedo_server_url, related_product, os):
- return '%s/uptake/?product=%s&os=%s' % (tuxedo_server_url,
- related_product, os)
-
-
-class UptakeMonitoring(BaseScript, VirtualenvMixin, BuildbotMixin):
- config_options = virtualenv_config_options
-
- def __init__(self, require_config_file=True):
- super(UptakeMonitoring, self).__init__(
- config_options=self.config_options,
- require_config_file=require_config_file,
- config={
- "virtualenv_modules": [
- "redo",
- "requests",
- ],
-
- "virtualenv_path": "venv",
- "credentials_file": "oauth.txt",
- "buildbot_json_path": "buildprops.json",
- "poll_interval": 60,
- "poll_timeout": 20*60,
- "min_uptake": 10000,
- },
- all_actions=[
- "create-virtualenv",
- "activate-virtualenv",
- "monitor-uptake",
- ],
- default_actions=[
- "create-virtualenv",
- "activate-virtualenv",
- "monitor-uptake",
- ],
- )
-
- def _pre_config_lock(self, rw_config):
- super(UptakeMonitoring, self)._pre_config_lock(rw_config)
- # override properties from buildbot properties here as defined by
- # taskcluster properties
- self.read_buildbot_config()
- if not self.buildbot_config:
- self.warning("Skipping buildbot properties overrides")
- return
- props = self.buildbot_config["properties"]
- for prop in ['tuxedo_server_url', 'version']:
- if props.get(prop):
- self.info("Overriding %s with %s" % (prop, props[prop]))
- self.config[prop] = props.get(prop)
- else:
- self.warning("%s could not be found within buildprops" % prop)
- return
- if props.get('partial_versions'):
- partials = [v.strip() for v in props["partial_versions"].split(",")]
- self.config["partial_versions"] = [v.split("build")[0] for v in partials]
- self.config["platforms"] = [p.strip() for p in
- props["platforms"].split(",")]
-
- def _get_product_uptake(self, tuxedo_server_url, auth,
- related_product, os):
- from redo import retry
- import requests
-
- url = get_tuxedo_uptake_url(tuxedo_server_url, related_product, os)
- self.info("Requesting {} from tuxedo".format(url))
-
- def get_tuxedo_page():
- r = requests.get(url, auth=auth,
- verify=False, timeout=60)
- r.raise_for_status()
- return r.content
-
- def calculateUptake(page):
- doc = xml.dom.minidom.parseString(page)
- uptake_values = []
-
- for element in doc.getElementsByTagName('available'):
- for node in element.childNodes:
- if node.nodeType == xml.dom.minidom.Node.TEXT_NODE and \
- node.data.isdigit():
- uptake_values.append(int(node.data))
- if not uptake_values:
- uptake_values = [0]
- return min(uptake_values)
-
- page = retry(get_tuxedo_page)
- uptake = calculateUptake(page)
- self.info("Current uptake for {} is {}".format(related_product, uptake))
- return uptake
-
- def _get_release_uptake(self, auth):
- assert isinstance(self.config["platforms"], (list, tuple))
-
- # handle the products first
- tuxedo_server_url = self.config["tuxedo_server_url"]
- version = self.config["version"]
- dl = []
-
- for product, info in self.config["products"].iteritems():
- if info.get("check_uptake"):
- product_template = info["product-name"]
- related_product = product_template % {"version": version}
-
- enUS_platforms = set(self.config["platforms"])
- paths_platforms = set(info["paths"].keys())
- platforms = enUS_platforms.intersection(paths_platforms)
-
- for platform in platforms:
- bouncer_platform = info["paths"].get(platform).get('bouncer-platform')
- dl.append(self._get_product_uptake(tuxedo_server_url, auth,
- related_product, bouncer_platform))
- # handle the partials as well
- prev_versions = self.config.get("partial_versions", [])
- for product, info in self.config.get("partials", {}).iteritems():
- if info.get("check_uptake"):
- product_template = info["product-name"]
- for prev_version in prev_versions:
- subs = {
- "version": version,
- "prev_version": prev_version
- }
- related_product = product_template % subs
-
- enUS_platforms = set(self.config["platforms"])
- paths_platforms = set(info["paths"].keys())
- platforms = enUS_platforms.intersection(paths_platforms)
-
- for platform in platforms:
- bouncer_platform = info["paths"].get(platform).get('bouncer-platform')
- dl.append(self._get_product_uptake(tuxedo_server_url, auth,
- related_product, bouncer_platform))
- return min(dl)
-
- def monitor_uptake(self):
- credentials_file = os.path.join(os.getcwd(),
- self.config["credentials_file"])
- credentials = {}
- execfile(credentials_file, credentials)
- auth = (credentials['tuxedoUsername'], credentials['tuxedoPassword'])
- self.info("Starting the loop to determine the uptake monitoring ...")
-
- start_time = datetime.datetime.now()
- while True:
- delta = (datetime.datetime.now() - start_time).seconds
- if delta > self.config["poll_timeout"]:
- self.error("Uptake monitoring sadly timed-out")
- raise Exception("Time-out during uptake monitoring")
-
- uptake = self._get_release_uptake(auth)
- self.info("Current uptake value to check is {}".format(uptake))
-
- if uptake >= self.config["min_uptake"]:
- self.info("Uptake monitoring is complete!")
- break
- else:
- self.info("Mirrors not yet updated, sleeping for a bit ...")
- time.sleep(self.config["poll_interval"])
-
-
-if __name__ == '__main__':
- myScript = UptakeMonitoring()
- myScript.run_and_exit()