Bug 1330191 - Android l10n nightlies fail with abort: --revision must be a SHA-1 fragment 12-40 characters long. r=mshal
authorJustin Wood <Callek@gmail.com>
Wed, 11 Jan 2017 09:35:19 -0500
changeset 459285 472d7c6ccafd34fef8d8c0f25bfdd3dbfe352a60
parent 459273 95c35806449d1e083f453154bff3c4f333d23c7d
child 459497 a771df57a5bcaca1ddbcbe5b5bbc932ab577c501
child 459674 474dfcafbc730695bbabe3ba418d329a15fffbba
push id41169
push userCallek@gmail.com
push dateWed, 11 Jan 2017 17:01:21 +0000
reviewersmshal
bugs1330191
milestone53.0a1
Bug 1330191 - Android l10n nightlies fail with abort: --revision must be a SHA-1 fragment 12-40 characters long. r=mshal MozReview-Commit-ID: IceOh55Lm6g
testing/mozharness/scripts/mobile_l10n.py
--- a/testing/mozharness/scripts/mobile_l10n.py
+++ b/testing/mozharness/scripts/mobile_l10n.py
@@ -5,17 +5,16 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 """mobile_l10n.py
 
 This currently supports nightly and release single locale repacks for
 Android.  This also creates nightly updates.
 """
 
-from copy import deepcopy
 import glob
 import os
 import re
 import subprocess
 import sys
 import time
 import shlex
 
@@ -246,28 +245,61 @@ class MobileSingleLocale(MockMixin, Loca
         output = self._query_make_ident_output()
         for line in output.splitlines():
             m = r.match(line)
             if m:
                 self.buildid = m.groups()[0]
         return self.buildid
 
     def query_revision(self):
-        """Get revision from the objdir.
-        Only valid after setup is run.
+        """ Get the gecko revision in this order of precedence
+              * cached value
+              * command line arg --revision   (development, taskcluster)
+              * buildbot properties           (try with buildbot forced build)
+              * buildbot change               (try with buildbot scheduler)
+              * from the en-US build          (m-c & m-a)
+
+        This will fail the last case if the build hasn't been pulled yet.
         """
         if self.revision:
             return self.revision
-        r = re.compile(r"gecko_revision ([0-9a-f]+\+?)")
+
+        self.read_buildbot_config()
+        config = self.config
+        revision = None
+        if config.get("revision"):
+            revision = config["revision"]
+        elif 'revision' in self.buildbot_properties:
+            revision = self.buildbot_properties['revision']
+        elif (self.buildbot_config and
+                  self.buildbot_config.get('sourcestamp', {}).get('revision')):
+            revision = self.buildbot_config['sourcestamp']['revision']
+        elif self.buildbot_config and self.buildbot_config.get('revision'):
+            revision = self.buildbot_config['revision']
+        elif config.get("update_gecko_source_to_enUS", True):
+            revision = self._query_enUS_revision()
+
+        if not revision:
+            self.fatal("Can't determine revision!")
+        self.revision = str(revision)
+        return self.revision
+
+    def _query_enUS_revision(self):
+        """Get revision from the objdir.
+        Only valid after setup is run.
+       """
+        if self.enUS_revision:
+            return self.enUS_revision
+        r = re.compile(r"^gecko_revision ([0-9a-f]+\+?)$")
         output = self._query_make_ident_output()
         for line in output.splitlines():
-            m = r.match(line)
-            if m:
-                self.revision = m.groups()[0]
-        return self.revision
+            match = r.match(line)
+            if match:
+                self.enUS_revision = match.groups()[0]
+        return self.enUS_revision
 
     def _query_make_variable(self, variable, make_args=None):
         make = self.query_exe('make')
         env = self.query_repack_env()
         dirs = self.query_abs_dirs()
         if make_args is None:
             make_args = []
         # TODO error checking
@@ -359,22 +391,33 @@ class MobileSingleLocale(MockMixin, Loca
 
     def pull(self):
         c = self.config
         dirs = self.query_abs_dirs()
         repos = []
         replace_dict = {}
         if c.get("user_repo_override"):
             replace_dict['user_repo_override'] = c['user_repo_override']
-            # deepcopy() needed because of self.config lock bug :(
-            for repo_dict in deepcopy(c['repos']):
-                repo_dict['repo'] = repo_dict['repo'] % replace_dict
-                repos.append(repo_dict)
-        else:
-            repos = c['repos']
+        # this is OK so early because we get it from buildbot, or
+        # the command line for local dev
+        replace_dict['revision'] = self.query_revision()
+
+        for repository in c['repos']:
+            current_repo = {}
+            for key, value in repository.iteritems():
+                try:
+                    current_repo[key] = value % replace_dict
+                except TypeError:
+                    # pass through non-interpolables, like booleans
+                    current_repo[key] = value
+                except KeyError:
+                    self.error('not all the values in "{0}" can be replaced. Check your configuration'.format(value))
+                    raise
+            repos.append(current_repo)
+        self.info("repositories: %s" % repos)
         self.vcs_checkout_repos(repos, parent_dir=dirs['abs_work_dir'],
                                 tag_override=c.get('tag_override'))
 
     def clone_locales(self):
         self.pull_locale_source()
 
     # list_locales() is defined in LocalesMixin.
 
@@ -432,17 +475,17 @@ class MobileSingleLocale(MockMixin, Loca
                            cwd=dirs['abs_locales_dir'],
                            env=env,
                            error_list=MakefileErrorList,
                            halt_on_failure=True)
 
         # on try we want the source we already have, otherwise update to the
         # same as the en-US binary
         if self.config.get("update_gecko_source_to_enUS", True):
-            revision = self.query_revision()
+            revision = self.query_enUS_revision()
             if not revision:
                 self.fatal("Can't determine revision!")
             hg = self.query_exe("hg")
             # TODO do this through VCSMixin instead of hardcoding hg
             self.run_command_m([hg, "update", "-r", revision],
                                cwd=dirs["abs_mozilla_dir"],
                                env=env,
                                error_list=BaseErrorList,