autoland: support landing of s3 hosted patches (
bug 1368516) r?smacleod r?gps
Add support for downloading a patch from s3 and landing it.
MozReview-Commit-ID: KkMltMjv3WF
--- a/autoland/autoland/transplant.py
+++ b/autoland/autoland/transplant.py
@@ -1,14 +1,21 @@
-import config
-import hglib
+import io
import json
import logging
import re
import tempfile
+import urlparse
+
+import boto3
+import hglib
+import requests
+from botocore.exceptions import ClientError
+
+import config
REPO_CONFIG = {}
logger = logging.getLogger('autoland')
class HgCommandError(Exception):
def __init__(self, hg_args, out):
@@ -30,22 +37,22 @@ class Transplant(object):
assert isinstance(rev, str), "rev arg is not str"
self.tree = tree
self.destination = destination
self.source_rev = rev
self.path = config.get_repo(tree)['path']
def __enter__(self):
- configs = ['ui.interactive=False']
+ configs = ['ui.interactive=False', 'extensions.purge=']
self.hg_repo = hglib.open(self.path, encoding='utf-8', configs=configs)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
- self.strip_drafts()
+ self.clean_repo()
self.hg_repo.close()
def push_try(self, trysyntax):
# Don't let unicode leak into command arguments.
assert isinstance(trysyntax, str), "trysyntax arg is not str"
remote_tip = self.update_repo()
@@ -89,18 +96,18 @@ class Transplant(object):
])
return rev
def update_repo(self):
# Obtain remote tip. We assume there is only a single head.
remote_tip = self.get_remote_tip()
- # Strip any lingering draft changesets.
- self.strip_drafts()
+ # Strip any lingering changes.
+ self.clean_repo()
# Pull from "upstream".
self.update_from_upstream(remote_tip)
return remote_tip
def apply_changes(self, remote_tip):
raise NotImplemented('abstract method call: apply_changes')
@@ -121,22 +128,35 @@ class Transplant(object):
last_result = ''
for cmd in cmds:
try:
last_result = self.run_hg(cmd)
except hglib.error.CommandError as e:
raise HgCommandError(cmd, e.out)
return last_result
- def strip_drafts(self):
+ def clean_repo(self):
# Strip any lingering draft changesets.
try:
self.run_hg(['strip', '--no-backup', '-r', 'not public()'])
except hglib.error.CommandError:
pass
+ # Clean working directory.
+ try:
+ self.run_hg(['--quiet', 'revert', '--no-backup', '--all'])
+ except hglib.error.CommandError:
+ pass
+ try:
+ self.run_hg(['purge', '--all'])
+ except hglib.error.CommandError:
+ pass
+
+ def dirty_files(self):
+ return self.run_hg(['status', '--modified', '--added', '--removed',
+ '--deleted', '--unknown', '--ignored'])
def get_remote_tip(self):
# Obtain remote tip. We assume there is only a single head.
# Output can contain bookmark or branch name after a space. Only take
# first component.
remote_tip = self.run_hg_cmds([
['identify', 'upstream', '-r', 'tip']
])
@@ -269,17 +289,82 @@ class PatchTransplant(Transplant):
def __init__(self, tree, destination, rev, patch_urls):
self.patch_urls = patch_urls
super(PatchTransplant, self).__init__(tree, destination, rev)
def apply_changes(self, remote_tip):
assert self.patch_urls, 'patch_urls not provided'
+ dirty_files = self.dirty_files()
+ if dirty_files:
+ logger.error('repo is not clean: %s' % ' '.join(dirty_files))
+ raise Exception("We're sorry - something has gone wrong while "
+ "landing your commits. The repository contains "
+ "unexpected changes. "
+ "Please file a bug.")
+
+ self.run_hg(['update', remote_tip])
+
for patch_url in self.patch_urls:
if patch_url.startswith('s3://'):
- # Download patch to temp file and import
- raise Exception('importing patches from s3 not implemented')
+ # Download patch from s3 to a temp file.
+ io_buf = self._download_from_s3(patch_url)
else:
- self.run_hg(['update', remote_tip])
- output = self.run_hg(['import', patch_url])
- logger.info(output)
+ # Download patch directly from url. Using a temp file here
+ # instead of passing the url to 'hg import' to make
+ # testing's code path closer to production's.
+ io_buf = self._download_from_url(patch_url)
+
+ with tempfile.NamedTemporaryFile() as temp_file:
+ temp_file.write(io_buf.getvalue())
+ temp_file.flush()
+
+ # Apply the patch, with file rename detection (similarity).
+ # Using 95 as the similarity to match automv's default.
+ logger.info(self.run_hg(['import', '-s', '95', temp_file.name]))
+
+ @staticmethod
+ def _download_from_s3(patch_url):
+ # Download from s3 url specified in self.patch_url, returns io.BytesIO.
+ url = urlparse.urlparse(patch_url)
+ bucket = url.hostname
+ key = url.path[1:]
+
+ buckets_config = config.get('patch_url_buckets')
+ if bucket not in buckets_config:
+ logging.error('bucket "%s" not configured in patch_url_buckets'
+ % bucket)
+ raise Exception('invalid patch_url')
+ bucket_config = buckets_config[bucket]
+
+ if ('aws_access_key_id' not in bucket_config or
+ 'aws_secret_access_key' not in bucket_config):
+ logging.error('bucket "%s" is missing aws_access_key_id or '
+ 'aws_secret_access_key' % bucket)
+ raise Exception('invalid patch_url')
+
+ try:
+ s3 = boto3.client(
+ 's3',
+ aws_access_key_id=bucket_config['aws_access_key_id'],
+ aws_secret_access_key=bucket_config['aws_secret_access_key'])
+
+ buf = io.BytesIO
+ s3.download_fileobj(bucket, key, buf)
+ return buf
+ except ClientError as e:
+ error_code = int(e.response['Error']['Code'])
+ if error_code == 404:
+ raise Exception('unable to download %s: file not found'
+ % patch_url)
+ if error_code == 403:
+ raise Exception('unable to download %s: permission denied'
+ % patch_url)
+ raise
+
+ @staticmethod
+ def _download_from_url(patch_url):
+ # Download from patch_url, returns io.BytesIO.
+ r = requests.get(patch_url, stream=True)
+ r.raise_for_status()
+ return io.BytesIO(r.content)
--- a/autoland/tests/test-post-autoland-job-from-patch.t
+++ b/autoland/tests/test-post-autoland-job-from-patch.t
@@ -46,22 +46,23 @@ Posting a job with bad credentials shoul
$ mozreview exec autoland tail -n1 /var/log/apache2/error.log
* WARNING:root:Failed authentication for "blah" from * (glob)
Post a job from http url should fail
$ ottoland post-autoland-job $AUTOLAND_URL test-repo p1 inbound http://localhost:9898 --patch-url http://example.com/p2.patch
(400, u'{\n "error": "Bad request: bad patch_url"\n}')
-Post a job from s3 url
+Post a job from s3 url. This should fail because we don't have a mock
+environment for S3.
$ ottoland post-autoland-job $AUTOLAND_URL test-repo p2 inbound http://localhost:9898 --patch-url s3://example-bucket/p1.patch
(200, u'{\n "request_id": 1\n}')
$ ottoland autoland-job-status $AUTOLAND_URL 1 --poll
- (200, u'{\n "destination": "inbound", \n "error_msg": "importing patches from s3 not implemented", \n "landed": false, \n "ldap_username": "autolanduser@example.com", \n "patch_urls": [\n "s3://example-bucket/p1.patch"\n ], \n "result": "", \n "rev": "p2", \n "tree": "test-repo"\n}')
+ (200, u'{\n "destination": "inbound", \n "error_msg": "unable to download s3://example-bucket/p1.patch: permission denied", \n "landed": false, \n "ldap_username": "autolanduser@example.com", \n "patch_urls": [\n "s3://example-bucket/p1.patch"\n ], \n "result": "", \n "rev": "p2", \n "tree": "test-repo"\n}')
Post a job from private ip
$ ottoland post-autoland-job $AUTOLAND_URL test-repo p3 inbound http://localhost:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
(200, u'{\n "request_id": 2\n}')
$ ottoland autoland-job-status $AUTOLAND_URL 2 --poll
(200, u'{\n "destination": "inbound", \n "error_msg": "", \n "landed": true, \n "ldap_username": "autolanduser@example.com", \n "patch_urls": [\n "http://$DOCKER_HOSTNAME:$HGPORT/test-repo/raw-rev/4b444b4e2552"\n ], \n "result": null, \n "rev": "p3", \n "tree": "test-repo"\n}')
$ mozreview exec autoland hg log /repos/test-repo/ --template '{rev}:{desc\|firstline}:{phase}\\n'
@@ -169,43 +170,52 @@ Post a job with try syntax
$ ottoland post-autoland-job $AUTOLAND_URL test-repo p6 try http://localhost:9898 --trysyntax "stuff" --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
(400, u'{\n "error": "Bad request: trysyntax is not supported with patch_urls"\n}')
Getting status for an unknown job should return a 404
$ ottoland autoland-job-status $AUTOLAND_URL 42
(404, u'{\n "error": "Not found"\n}')
+Ensure unexpected files in the repo path are not landed.
+
+ $ mozreview exec autoland touch /repos/test-repo/rogue
+ $ ottoland post-autoland-job $AUTOLAND_URL test-repo p7 inbound http://localhost:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+ (200, u'{\n "request_id": 5\n}')
+ $ ottoland autoland-job-status $AUTOLAND_URL 5 --poll
+ (200, u'{\n "destination": "inbound", \n "error_msg": "", \n "landed": true, \n "ldap_username": "autolanduser@example.com", \n "patch_urls": [\n "http://$DOCKER_HOSTNAME:$HGPORT/test-repo/raw-rev/e525ec140d61"\n ], \n "result": null, \n "rev": "p7", \n "tree": "test-repo"\n}')
+ $ mozreview exec autoland hg files --cwd /repos/test-repo
+ foo
+
Test pingback url whitelist. localhost, private IPs, and example.com are in
the whitelist. example.org is not.
- $ ottoland post-autoland-job $AUTOLAND_URL test-repo p7 inbound1 http://example.com:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
- (200, u'{\n "request_id": 5\n}')
- $ ottoland post-autoland-job $AUTOLAND_URL test-repo p8 inbound2 http://localhost --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+ $ ottoland post-autoland-job $AUTOLAND_URL test-repo p8 inbound http://example.com:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
(200, u'{\n "request_id": 6\n}')
- $ ottoland post-autoland-job $AUTOLAND_URL test-repo p9 inbound2 http://localhost --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+ $ ottoland post-autoland-job $AUTOLAND_URL test-repo p9 inbound http://localhost --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
(200, u'{\n "request_id": 7\n}')
- $ ottoland post-autoland-job $AUTOLAND_URL test-repo p10 inbound3 http://127.0.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+ $ ottoland post-autoland-job $AUTOLAND_URL test-repo p10 inbound http://localhost --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
(200, u'{\n "request_id": 8\n}')
- $ ottoland post-autoland-job $AUTOLAND_URL test-repo p11 inbound4 http://192.168.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+ $ ottoland post-autoland-job $AUTOLAND_URL test-repo p11 inbound http://127.0.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
(200, u'{\n "request_id": 9\n}')
- $ ottoland post-autoland-job $AUTOLAND_URL test-repo p12 inbound5 http://172.16.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+ $ ottoland post-autoland-job $AUTOLAND_URL test-repo p12 inbound http://192.168.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
(200, u'{\n "request_id": 10\n}')
- $ ottoland post-autoland-job $AUTOLAND_URL test-repo p13 inbound6 http://10.0.0.1:443 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+ $ ottoland post-autoland-job $AUTOLAND_URL test-repo p13 inbound http://172.16.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
(200, u'{\n "request_id": 11\n}')
- $ ottoland post-autoland-job $AUTOLAND_URL test-repo p14 inbound7 http://8.8.8.8:443 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+ $ ottoland post-autoland-job $AUTOLAND_URL test-repo p14 inbound http://10.0.0.1:443 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+ (200, u'{\n "request_id": 12\n}')
+ $ ottoland post-autoland-job $AUTOLAND_URL test-repo p15 inbound http://8.8.8.8:443 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
(400, u'{\n "error": "Bad request: bad pingback_url"\n}')
- $ ottoland post-autoland-job $AUTOLAND_URL test-repo p15 inbound8 http://example.org:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+ $ ottoland post-autoland-job $AUTOLAND_URL test-repo p16 inbound http://example.org:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
(400, u'{\n "error": "Bad request: bad pingback_url"\n}')
Post the same job twice. Start with stopping the autoland service to
guarentee the first request is still in the queue when the second is submitted.
$ PID=`mozreview exec autoland ps x | grep autoland.py | grep -v grep | awk '{ print $1 }'`
$ mozreview exec autoland kill $PID
- [1] (?)
- $ ottoland post-autoland-job $AUTOLAND_URL test-repo $REV try http://localhost:9898 --trysyntax "stuff"
- (200, u'{\n "request_id": 12\n}')
- $ ottoland post-autoland-job $AUTOLAND_URL test-repo $REV try http://localhost:9898 --trysyntax "stuff"
- (400, u'{\n "error": "Bad Request: a request to land revision e525ec140d61 to try is already in progress"\n}')
+ $ ottoland post-autoland-job $AUTOLAND_URL test-repo p17 inbound http://localhost:9898 --trysyntax "stuff"
+ (200, u'{\n "request_id": 13\n}')
+ $ ottoland post-autoland-job $AUTOLAND_URL test-repo p17 inbound http://localhost:9898 --trysyntax "stuff"
+ (400, u'{\n "error": "Bad Request: a request to land revision p17 to inbound is already in progress"\n}')
$ mozreview stop
stopped 9 containers