Bug 1265579 - Improve relpro sanity with RC partials testcase. r=rail draft
authorMihai Tabara <mtabara@mozilla.com>
Tue, 19 Jul 2016 19:27:14 +0100
changeset 7029 fe96bb6f9f78831548711d85f081fea66bf6dac3
parent 7028 ebcc78b3513075a4bd7720551a69c530897e8b3a
push id94
push userbmo:mtabara@mozilla.com
push dateTue, 19 Jul 2016 18:27:54 +0000
reviewersrail
bugs1265579
Bug 1265579 - Improve relpro sanity with RC partials testcase. r=rail MozReview-Commit-ID: GJ7sRdARFVS
buildfarm/release/release-runner.py
lib/python/kickoff/sanity.py
--- a/buildfarm/release/release-runner.py
+++ b/buildfarm/release/release-runner.py
@@ -17,17 +17,17 @@ from optparse import OptionParser
 from twisted.python.lockfile import FilesystemLock
 
 site.addsitedir(path.join(path.dirname(__file__), "../../lib/python"))
 
 from kickoff import get_partials, ReleaseRunner, make_task_graph_strict_kwargs
 from kickoff import get_l10n_config, get_en_US_config
 from kickoff import email_release_drivers
 from kickoff import bump_version
-from kickoff.sanity import ReleaseSanitizerRunner, SanityException
+from kickoff.sanity import ReleaseSanitizerRunner, SanityException, is_candidate_release
 from release.info import readBranchConfig
 from release.l10n import parsePlainL10nChangesets
 from release.versions import getAppVersion
 from taskcluster import Scheduler, Index, Queue
 from taskcluster.utils import slugId
 from util.hg import mercurial
 from util.retry import retry
 from util.file import load_config, get_config
@@ -49,28 +49,16 @@ ALL_FILES = set([
     '.complete.mar',
     '.exe',
     '.dmg',
     'i686.tar.bz2',
     'x86_64.tar.bz2',
 ])
 
 
-def is_candidate_release(channels):
-    """determine if this is a candidate release or not
-
-    Because ship-it can not tell us if this is a candidate release (yet!), we assume it is when we
-    have determined, based on version, that we are planning to ship to more than one update_channel
-    e.g. for candidate releases we have:
-     1) one channel to test the 'candidate' release with: to 'beta' channel users
-     2) once verified, we ship to the main channel: to 'release' channel users
-    """
-    return len(channels) > 1
-
-
 def update_channels(version, mappings):
     """Return a list of update channels for a version using version mapping
 
     >>> update_channels("40.0", [(r"^\d+\.0$", ["beta", "release"]), (r"^\d+\.\d+\.\d+$", ["release"])])
     ["beta", "release"]
     >>> update_channels("40.0.1", [(r"^\d+\.0$", ["beta", "release"]), (r"^\d+\.\d+\.\d+$", ["release"])])
     ["release"]
 
--- a/lib/python/kickoff/sanity.py
+++ b/lib/python/kickoff/sanity.py
@@ -12,26 +12,28 @@ import logging
 from os import path
 
 import requests
 
 site.addsitedir(path.join(path.dirname(__file__), ".."))
 from util.retry import retry
 from util.hg import make_hg_url
 from release.versions import getL10nDashboardVersion
+from kickoff import matches
 
 log = logging.getLogger(__name__)
 
 CANDIDATES_SHA512_URL_TEMPLATE = "https://archive.mozilla.org/pub/firefox/candidates/{version}-candidates/build{build_number}/SHA512SUMS"
 RELEASES_SHA512_URL_TEMPLATE= "https://archive.mozilla.org/pub/firefox/releases/{version}/SHA512SUMS"
 L10N_DASHBOARD_URL_TEMPLATE = "https://l10n.mozilla.org/shipping/l10n-changesets?ms={milestone}"
 LOCALE_BASE_URL_TEMPLATE = "{hg_l10n_base}/{locale}/raw-file/{revision}"
 SINGLE_LOCALE_CONFIG_URI_TEMPLATE = "testing/mozharness/configs/single_locale/{branch}.py"
 VERSION_DISPLAY_CONFIG_URI = "browser/config/version_display.txt"
 SHIPPED_LOCALES_CONFIG_URI = "browser/locales/shipped-locales"
+BETA_PATTERNS = [r"\d+\.0b\d+"]
 
 
 def make_generic_head_request(page_url):
     """Make generic HEAD request to check page existence"""
     def _get():
         req = requests.head(page_url, timeout=60)
         req.raise_for_status()
 
@@ -50,16 +52,29 @@ def make_generic_get_request(page_url):
 
 def make_hg_get_request(repo_path, revision,
                         filename=None, hg_url='hg.mozilla.org'):
     """Wrapper to make a GET request for a specific URI under hg repo"""
     url = make_hg_url(hg_url, repo_path, revision=revision, filename=filename)
     return make_generic_get_request(url)
 
 
+def is_candidate_release(channels):
+    """Determine if this is a candidate release or not
+
+    Because ship-it can not tell us if this is a candidate release (yet!),
+    we assume it is when we have determined, based on version,
+    that we are planning to ship to more than one update_channel
+    e.g. for candidate releases we have:
+     1) one channel to test the 'candidate' release with: 'beta' channel
+     2) once verified, we ship to the main channel: 'release' channel
+    """
+    return len(channels) > 1
+
+
 def get_l10_dashboard_changeset(version, product):
     """Helper function to retrieve l10n dashboard changesets
 
     >>> get_l10_dashboard_changeset('47.0, 'firefox')
     ach revision_123
     af revision_245
     an revision_456
     ...
@@ -134,16 +149,17 @@ class ReleaseSanitizerTestSuite(OpsMixin
     def __init__(self, **kwargs):
         self.kwargs = kwargs
         self.repo_path = self.kwargs["repo_path"]
         self.revision = self.kwargs["revision"]
         self.version = self.kwargs["version"]
         self.branch = self.kwargs["branch"]
         self.locales = self.kwargs["l10n_changesets"]
         self.product = self.kwargs["product"]
+        self.partial_updates = self.kwargs["partial_updates"]
 
     def sanitize(self, result):
         """Main method to run all the sanity checks. It collects all the
         methods prefixed with 'test_' and runs them accordingly.
         It runs all the test and collects any potential errors in the :result
         object.
         """
         test_methods = [m for m in filter(lambda k: k.startswith("test_"), dir(self))
@@ -248,16 +264,39 @@ class ReleaseSanitizerTestSuite(OpsMixin
 
             err_msg = ("{version}-build{build_number} is a good candidate"
                        " build, but not the one we shipped! URL: {url}").format(
                            version=pversion,
                            build_number=buildno,
                            url=_url)
             self.assertEqual(result, releases_sha, candidate_sha, err_msg)
 
+    def test_partials_release_candidate_validity(self, result):
+        """test_partials method
+        Tests if a RC contains both beta and release in list of partials.
+        We hit this issue in bug 1265579 in which the updates builder failed
+        if partials were all-beta OR if no-beta at all
+        """
+        log.info("Testing RC partials ...")
+        if not is_candidate_release(self.kwargs["release_channels"]):
+            log.info("Skipping this test as we're not dealing with a RC now")
+            return
+
+        ret = [matches(name, BETA_PATTERNS) for name in self.partial_updates]
+        at_least_one_beta = any(ret)
+        all_betas = all(ret) and ret != []
+
+        partials = ["{name}".format(name=p) for p in self.partial_updates]
+        err_msg = "No beta found in the RC list of partials:{l}".format(l=partials)
+        self.assertEqual(result, at_least_one_beta, True, err_msg)
+
+        err_msg = ("All partials in the RC list are betas. At least a non-beta"
+                   " release is needed in {_list}").format(_list=partials)
+        self.assertEqual(result, all_betas, False, err_msg)
+
     def test_l10n_shipped_locales(self, result):
         """test_l10n method
         Tests if the current locales coming from release runner are in fact
         the same as the shipped locales.
         """
         log.info("Testing l10n shipped locales ...")
         try:
             # TODO: mind that we will need something similar for Fennec