autoland: support landing of s3 hosted patches (bug 1368516) r?smacleod r?gps draft
authorbyron jones <glob@mozilla.com>
Fri, 11 Aug 2017 14:42:44 +0800
changeset 11694 2044b6101b8b22b6ab3f7572ea52ed4a0b56b377
parent 11693 92eb2fcffbc6022ab88b6b466f5efe4ddc8872ef
child 11695 03db54b2ad8d7ab1733504633daf7a134bebcb9f
push id1791
push userbjones@mozilla.com
push dateTue, 19 Sep 2017 04:21:13 +0000
reviewerssmacleod, gps
bugs1368516
autoland: support landing of s3 hosted patches (bug 1368516) r?smacleod r?gps Add support for downloading a patch from s3 and landing it. MozReview-Commit-ID: KkMltMjv3WF
autoland/autoland/transplant.py
autoland/tests/test-post-autoland-job-from-patch.t
--- a/autoland/autoland/transplant.py
+++ b/autoland/autoland/transplant.py
@@ -1,14 +1,21 @@
-import config
-import hglib
+import io
 import json
 import logging
 import re
 import tempfile
+import urlparse
+
+import boto3
+import hglib
+import requests
+from botocore.exceptions import ClientError
+
+import config
 
 REPO_CONFIG = {}
 
 logger = logging.getLogger('autoland')
 
 
 class HgCommandError(Exception):
     def __init__(self, hg_args, out):
@@ -30,22 +37,22 @@ class Transplant(object):
         assert isinstance(rev, str), "rev arg is not str"
 
         self.tree = tree
         self.destination = destination
         self.source_rev = rev
         self.path = config.get_repo(tree)['path']
 
     def __enter__(self):
-        configs = ['ui.interactive=False']
+        configs = ['ui.interactive=False', 'extensions.purge=']
         self.hg_repo = hglib.open(self.path, encoding='utf-8', configs=configs)
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        self.strip_drafts()
+        self.clean_repo()
         self.hg_repo.close()
 
     def push_try(self, trysyntax):
         # Don't let unicode leak into command arguments.
         assert isinstance(trysyntax, str), "trysyntax arg is not str"
 
         remote_tip = self.update_repo()
 
@@ -89,18 +96,18 @@ class Transplant(object):
         ])
 
         return rev
 
     def update_repo(self):
         # Obtain remote tip. We assume there is only a single head.
         remote_tip = self.get_remote_tip()
 
-        # Strip any lingering draft changesets.
-        self.strip_drafts()
+        # Strip any lingering changes.
+        self.clean_repo()
 
         # Pull from "upstream".
         self.update_from_upstream(remote_tip)
 
         return remote_tip
 
     def apply_changes(self, remote_tip):
         raise NotImplemented('abstract method call: apply_changes')
@@ -121,22 +128,35 @@ class Transplant(object):
         last_result = ''
         for cmd in cmds:
             try:
                 last_result = self.run_hg(cmd)
             except hglib.error.CommandError as e:
                 raise HgCommandError(cmd, e.out)
         return last_result
 
-    def strip_drafts(self):
+    def clean_repo(self):
         # Strip any lingering draft changesets.
         try:
             self.run_hg(['strip', '--no-backup', '-r', 'not public()'])
         except hglib.error.CommandError:
             pass
+        # Clean working directory.
+        try:
+            self.run_hg(['--quiet', 'revert', '--no-backup', '--all'])
+        except hglib.error.CommandError:
+            pass
+        try:
+            self.run_hg(['purge', '--all'])
+        except hglib.error.CommandError:
+            pass
+
+    def dirty_files(self):
+        return self.run_hg(['status', '--modified', '--added', '--removed',
+                            '--deleted', '--unknown', '--ignored'])
 
     def get_remote_tip(self):
         # Obtain remote tip. We assume there is only a single head.
         # Output can contain bookmark or branch name after a space. Only take
         # first component.
         remote_tip = self.run_hg_cmds([
             ['identify', 'upstream', '-r', 'tip']
         ])
@@ -269,17 +289,82 @@ class PatchTransplant(Transplant):
     def __init__(self, tree, destination, rev, patch_urls):
         self.patch_urls = patch_urls
 
         super(PatchTransplant, self).__init__(tree, destination, rev)
 
     def apply_changes(self, remote_tip):
         assert self.patch_urls, 'patch_urls not provided'
 
+        dirty_files = self.dirty_files()
+        if dirty_files:
+            logger.error('repo is not clean: %s' % ' '.join(dirty_files))
+            raise Exception("We're sorry - something has gone wrong while "
+                            "landing your commits. The repository contains "
+                            "unexpected changes. "
+                            "Please file a bug.")
+
+        self.run_hg(['update', remote_tip])
+
         for patch_url in self.patch_urls:
             if patch_url.startswith('s3://'):
-                # Download patch to temp file and import
-                raise Exception('importing patches from s3 not implemented')
+                # Download patch from s3 to a temp file.
+                io_buf = self._download_from_s3(patch_url)
 
             else:
-                self.run_hg(['update', remote_tip])
-                output = self.run_hg(['import', patch_url])
-                logger.info(output)
+                # Download patch directly from url.  Using a temp file here
+                # instead of passing the url to 'hg import' to make
+                # testing's code path closer to production's.
+                io_buf = self._download_from_url(patch_url)
+
+            with tempfile.NamedTemporaryFile() as temp_file:
+                temp_file.write(io_buf.getvalue())
+                temp_file.flush()
+
+                # Apply the patch, with file rename detection (similarity).
+                # Using 95 as the similarity to match automv's default.
+                logger.info(self.run_hg(['import', '-s', '95', temp_file.name]))
+
+    @staticmethod
+    def _download_from_s3(patch_url):
+        # Download from s3 url specified in self.patch_url, returns io.BytesIO.
+        url = urlparse.urlparse(patch_url)
+        bucket = url.hostname
+        key = url.path[1:]
+
+        buckets_config = config.get('patch_url_buckets')
+        if bucket not in buckets_config:
+            logging.error('bucket "%s" not configured in patch_url_buckets'
+                          % bucket)
+            raise Exception('invalid patch_url')
+        bucket_config = buckets_config[bucket]
+
+        if ('aws_access_key_id' not in bucket_config or
+                'aws_secret_access_key' not in bucket_config):
+            logging.error('bucket "%s" is missing aws_access_key_id or '
+                          'aws_secret_access_key' % bucket)
+            raise Exception('invalid patch_url')
+
+        try:
+            s3 = boto3.client(
+                's3',
+                aws_access_key_id=bucket_config['aws_access_key_id'],
+                aws_secret_access_key=bucket_config['aws_secret_access_key'])
+
+            buf = io.BytesIO
+            s3.download_fileobj(bucket, key, buf)
+            return buf
+        except ClientError as e:
+            error_code = int(e.response['Error']['Code'])
+            if error_code == 404:
+                raise Exception('unable to download %s: file not found'
+                                % patch_url)
+            if error_code == 403:
+                raise Exception('unable to download %s: permission denied'
+                                % patch_url)
+            raise
+
+    @staticmethod
+    def _download_from_url(patch_url):
+        # Download from patch_url, returns io.BytesIO.
+        r = requests.get(patch_url, stream=True)
+        r.raise_for_status()
+        return io.BytesIO(r.content)
--- a/autoland/tests/test-post-autoland-job-from-patch.t
+++ b/autoland/tests/test-post-autoland-job-from-patch.t
@@ -46,22 +46,23 @@ Posting a job with bad credentials shoul
   $ mozreview exec autoland tail -n1 /var/log/apache2/error.log
   * WARNING:root:Failed authentication for "blah" from * (glob)
 
 Post a job from http url should fail
 
   $ ottoland post-autoland-job $AUTOLAND_URL test-repo p1 inbound http://localhost:9898 --patch-url http://example.com/p2.patch
   (400, u'{\n  "error": "Bad request: bad patch_url"\n}')
 
-Post a job from s3 url
+Post a job from s3 url.  This should fail because we don't have a mock
+environment for S3.
 
   $ ottoland post-autoland-job $AUTOLAND_URL test-repo p2 inbound http://localhost:9898 --patch-url s3://example-bucket/p1.patch
   (200, u'{\n  "request_id": 1\n}')
   $ ottoland autoland-job-status $AUTOLAND_URL 1 --poll
-  (200, u'{\n  "destination": "inbound", \n  "error_msg": "importing patches from s3 not implemented", \n  "landed": false, \n  "ldap_username": "autolanduser@example.com", \n  "patch_urls": [\n    "s3://example-bucket/p1.patch"\n  ], \n  "result": "", \n  "rev": "p2", \n  "tree": "test-repo"\n}')
+  (200, u'{\n  "destination": "inbound", \n  "error_msg": "unable to download s3://example-bucket/p1.patch: permission denied", \n  "landed": false, \n  "ldap_username": "autolanduser@example.com", \n  "patch_urls": [\n    "s3://example-bucket/p1.patch"\n  ], \n  "result": "", \n  "rev": "p2", \n  "tree": "test-repo"\n}')
 
 Post a job from private ip
 
   $ ottoland post-autoland-job $AUTOLAND_URL test-repo p3 inbound http://localhost:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (200, u'{\n  "request_id": 2\n}')
   $ ottoland autoland-job-status $AUTOLAND_URL 2 --poll
   (200, u'{\n  "destination": "inbound", \n  "error_msg": "", \n  "landed": true, \n  "ldap_username": "autolanduser@example.com", \n  "patch_urls": [\n    "http://$DOCKER_HOSTNAME:$HGPORT/test-repo/raw-rev/4b444b4e2552"\n  ], \n  "result": null, \n  "rev": "p3", \n  "tree": "test-repo"\n}')
   $ mozreview exec autoland hg log /repos/test-repo/ --template '{rev}:{desc\|firstline}:{phase}\\n'
@@ -169,43 +170,52 @@ Post a job with try syntax
   $ ottoland post-autoland-job $AUTOLAND_URL test-repo p6 try http://localhost:9898 --trysyntax "stuff" --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (400, u'{\n  "error": "Bad request: trysyntax is not supported with patch_urls"\n}')
 
 Getting status for an unknown job should return a 404
 
   $ ottoland autoland-job-status $AUTOLAND_URL 42
   (404, u'{\n  "error": "Not found"\n}')
 
+Ensure unexpected files in the repo path are not landed.
+
+  $ mozreview exec autoland touch /repos/test-repo/rogue
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p7 inbound http://localhost:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  (200, u'{\n  "request_id": 5\n}')
+  $ ottoland autoland-job-status $AUTOLAND_URL 5 --poll
+  (200, u'{\n  "destination": "inbound", \n  "error_msg": "", \n  "landed": true, \n  "ldap_username": "autolanduser@example.com", \n  "patch_urls": [\n    "http://$DOCKER_HOSTNAME:$HGPORT/test-repo/raw-rev/e525ec140d61"\n  ], \n  "result": null, \n  "rev": "p7", \n  "tree": "test-repo"\n}')
+  $ mozreview exec autoland hg files --cwd /repos/test-repo
+  foo
+
 Test pingback url whitelist.  localhost, private IPs, and example.com are in
 the whitelist. example.org is not.
 
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p7 inbound1 http://example.com:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
-  (200, u'{\n  "request_id": 5\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p8 inbound2 http://localhost --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p8 inbound http://example.com:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (200, u'{\n  "request_id": 6\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p9 inbound2 http://localhost --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p9 inbound http://localhost --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (200, u'{\n  "request_id": 7\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p10 inbound3 http://127.0.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p10 inbound http://localhost --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (200, u'{\n  "request_id": 8\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p11 inbound4 http://192.168.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p11 inbound http://127.0.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (200, u'{\n  "request_id": 9\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p12 inbound5 http://172.16.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p12 inbound http://192.168.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (200, u'{\n  "request_id": 10\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p13 inbound6 http://10.0.0.1:443 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p13 inbound http://172.16.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (200, u'{\n  "request_id": 11\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p14 inbound7 http://8.8.8.8:443 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p14 inbound http://10.0.0.1:443 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  (200, u'{\n  "request_id": 12\n}')
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p15 inbound http://8.8.8.8:443 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (400, u'{\n  "error": "Bad request: bad pingback_url"\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p15 inbound8 http://example.org:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p16 inbound http://example.org:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (400, u'{\n  "error": "Bad request: bad pingback_url"\n}')
 
 Post the same job twice.  Start with stopping the autoland service to
 guarentee the first request is still in the queue when the second is submitted.
 
   $ PID=`mozreview exec autoland ps x | grep autoland.py | grep -v grep | awk '{ print $1 }'`
   $ mozreview exec autoland kill $PID
-  [1] (?)
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo $REV try http://localhost:9898 --trysyntax "stuff"
-  (200, u'{\n  "request_id": 12\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo $REV try http://localhost:9898 --trysyntax "stuff"
-  (400, u'{\n  "error": "Bad Request: a request to land revision e525ec140d61 to try is already in progress"\n}')
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p17 inbound http://localhost:9898 --trysyntax "stuff"
+  (200, u'{\n  "request_id": 13\n}')
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p17 inbound http://localhost:9898 --trysyntax "stuff"
+  (400, u'{\n  "error": "Bad Request: a request to land revision p17 to inbound is already in progress"\n}')
 
   $ mozreview stop
   stopped 9 containers