Bug 1289249 - Make version control interaction generic; r?dustin draft
authorGregory Szorc <gps@mozilla.com>
Thu, 29 Sep 2016 17:05:05 -0700
changeset 419530 9b816b8883143db99041e6e3fbe1711a005a430c
parent 419529 c13957111f2bcdb77cf848f34dcebaa19e7dbff6
child 419531 432eec44058e6003373f615ba5f7140aa335310c
push id30957
push userbmo:gps@mozilla.com
push dateFri, 30 Sep 2016 16:24:29 +0000
reviewersdustin
bugs1289249
milestone52.0a1
Bug 1289249 - Make version control interaction generic; r?dustin Previously, we assumed we only could have a single version control checkout: Gecko/Firefox. The code reflected this by not passing arguments to the vcs_checkout function. Upcoming commits will introduce the need to perform a checkout of the build/tools repository. In preparation for this, refactor the VCS functionality so it is generic and can work on any repo. MozReview-Commit-ID: B0Act9fz2Ee
testing/docker/recipes/run-task
--- a/testing/docker/recipes/run-task
+++ b/testing/docker/recipes/run-task
@@ -70,35 +70,26 @@ def run_and_prefix_output(prefix, args, 
         if data == b'':
             break
 
         print_line(prefix, data)
 
     return p.wait()
 
 
-def vcs_checkout(args):
-    # TODO get VCS parameters from arguments.
-    base_repo = os.environ.get('GECKO_BASE_REPOSITORY')
-
-    # We set the base repository to mozilla-central so tc-vcs doesn't get
-    # confused. Switch to mozilla-unified because robustcheckout works best
-    # with it.
-    if base_repo == 'https://hg.mozilla.org/mozilla-central':
-        base_repo = b'https://hg.mozilla.org/mozilla-unified'
-
+def vcs_checkout(source_repo, dest, base_repo=None, revision=None, branch=None):
     # Specify method to checkout a revision. This defaults to revisions as
     # SHA-1 strings, but also supports symbolic revisions like `tip` via the
     # branch flag.
-    if os.environ.get('GECKO_HEAD_REV'):
+    if revision:
         revision_flag = b'--revision'
-        revision = os.environ['GECKO_HEAD_REV']
-    elif os.environ.get('GECKO_HEAD_REF'):
+        revision_value = revision
+    elif branch:
         revision_flag = b'--branch'
-        revision = os.environ['GECKO_HEAD_REF']
+        revision_value = branch
     else:
         print('revision is not specified for checkout')
         sys.exit(1)
 
     # Obtain certificate fingerprints.
     try:
         print_line(b'vcs', 'fetching hg.mozilla.org fingerprint from %s\n' %
                    FINGERPRINT_URL)
@@ -111,39 +102,46 @@ def vcs_checkout(args):
     try:
         secret = json.loads(secret, encoding='utf-8')
     except ValueError:
         print('invalid JSON in hg fingerprint secret')
         sys.exit(1)
 
     hgmo_fingerprint = secret['secret']['fingerprints'].encode('ascii')
 
-    res = run_and_prefix_output(b'vcs', [
+    args = [
         b'/usr/bin/hg',
         b'--config', b'hostsecurity.hg.mozilla.org:fingerprints=%s' % hgmo_fingerprint,
         b'robustcheckout',
         b'--sharebase', b'/home/worker/hg-shared',
         b'--purge',
-        b'--upstream', base_repo,
-        revision_flag, revision,
-        os.environ['GECKO_HEAD_REPOSITORY'], args.vcs_checkout
-    ], extra_env={b'PYTHONUNBUFFERED': b'1'})
+    ]
+
+    if base_repo:
+        args.extend([b'--upstream', base_repo])
 
+    args.extend([
+        revision_flag, revision_value,
+        source_repo, dest,
+    ])
+
+    res = run_and_prefix_output(b'vcs', args,
+                                extra_env={b'PYTHONUNBUFFERED': b'1'})
     if res:
         sys.exit(res)
 
     # Update the current revision hash and ensure that it is well formed.
     revision = subprocess.check_output(
         [b'/usr/bin/hg', b'log',
          b'--rev', b'.',
          b'--template', b'{node}'],
-        cwd=args.vcs_checkout)
+        cwd=dest)
 
     assert re.match('^[a-f0-9]{40}$', revision)
-    os.environ['GECKO_HEAD_REV'] = revision
+    return revision
 
 
 def main(args):
     print_line(b'setup', b'run-task started\n')
 
     if os.getuid() != 0:
         print('assertion failed: not running as root')
         return 1
@@ -220,45 +218,62 @@ def main(args):
 
         for root, dirs, files in os.walk(path):
             for d in dirs:
                 set_dir_permissions(os.path.join(root, d), uid, gid)
 
             for f in files:
                 os.chown(os.path.join(root, f), uid, gid)
 
-    checkout = args.vcs_checkout
-    if checkout:
+    def prepare_checkout_dir(checkout):
+        if not checkout:
+            return
+
         # Ensure the directory for the source checkout exists.
         try:
             os.makedirs(os.path.dirname(checkout))
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
         # And that it is owned by the appropriate user/group.
         os.chown('/home/worker/hg-shared', uid, gid)
         os.chown(os.path.dirname(checkout), uid, gid)
 
+    prepare_checkout_dir(args.vcs_checkout)
+
     # Drop permissions to requested user.
     # This code is modeled after what `sudo` was observed to do in a Docker
     # container. We do not bother calling setrlimit() because containers have
     # their own limits.
     print_line(b'setup', b'running as %s:%s\n' % (args.user, args.group))
     os.setgroups(gids)
     os.umask(022)
     os.setresgid(gid, gid, gid)
     os.setresuid(uid, uid, uid)
 
     # Checkout the repository, setting the GECKO_HEAD_REV to the current
     # revision hash. Revision hashes have priority over symbolic revisions. We
     # disallow running tasks with symbolic revisions unless they have been
     # resolved by a checkout.
-    if checkout:
-        vcs_checkout(args)
+    if args.vcs_checkout:
+        base_repo = os.environ.get('GECKO_BASE_REPOSITORY')
+        # Some callers set the base repository to mozilla-central for historical
+        # reasons. Switch to mozilla-unified because robustcheckout works best
+        # with it.
+        if base_repo == 'https://hg.mozilla.org/mozilla-central':
+            base_repo = b'https://hg.mozilla.org/mozilla-unified'
+
+        os.environ['GECKO_HEAD_REV'] = vcs_checkout(
+            os.environ['GECKO_HEAD_REPOSITORY'],
+            args.vcs_checkout,
+            base_repo=base_repo,
+            revision=os.environ.get('GECKO_HEAD_REV'),
+            branch=os.environ.get('GECKO_HEAD_REF'))
+
     elif not os.environ.get('GECKO_HEAD_REV') and \
             os.environ.get('GECKO_HEAD_REF'):
         print('task should be defined in terms of non-symbolic revision')
         return 1
 
     return run_and_prefix_output(b'task', task_args)