new file mode 100644
--- /dev/null
+++ b/vcssync/mozvcssync/coland.py
@@ -0,0 +1,427 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import os
+import tempfile
+import urlparse
+from subprocess import CalledProcessError
+from email.utils import (formataddr, parseaddr)
+
+import boto3
+import bugsy
+import hglib
+from botocore.exceptions import ClientError
+
+from .github_pr import GitHubPR
+from .sqs import SqsFatalException
+from .util import run_hg, clean_hg_repo, maybe_revision
+
+"""
+Processes a request to Co-Land a Servo+Gecko Change.
+
+Autoland-transplant uses a FIFO SQS queue to deliver the request to the
+servo-sqs-listen daemon, which passes the message to this package for
+processing.
+
+Requests are JSON with the following fields:
+
+"action"
+ Must be "servo-coland"
+
+"autoland-id"
+ Autoland's internal request ID.
+ Allows Autoland to identify responses.
+
+"author"
+ Patch author, used for auditing.
+
+"bug-id"
+ BMO bug ID.
+ Coland will comment on the bug with the Pull Request URL.
+
+"bundle-url"
+ s3:// URL of Mercurial bundle which contains both Servo and Gecko changes.
+
+"destination"
+ ssh:// URL of destination Mercurial repo.
+
+"""
+
+
+logger = logging.getLogger('coland')
+
+
+def validate_config(config):
+ # Ensure co-land configuration is valid.
+
+ required_fields = {'sqs_coland_queue', 'sqs_coland_error_queue',
+ 'sqs_coland_error_queue', 'coland_bucket',
+ 'coland_bucket_aws_key', 'coland_bucket_aws_secret',
+ 'coland_github_name', 'coland_github_token',
+ 'coland_bugzilla_url', 'coland_bugzilla_apikey'}
+ request_fields = set(config.keys())
+ missing_fields = required_fields - request_fields
+ if missing_fields:
+ raise Exception(
+ 'invalid config: missing required field%s: %s'
+ % ('' if len(missing_fields) == 1
+ else 's', ', '.join(sorted(missing_fields))))
+
+ # Sanity check queue names.
+ if not config['sqs_coland_queue'].endswith('.fifo'):
+ raise Exception("'%s' is not a FIFO queue" % config['sqs_coland_queue'])
+ if not config['sqs_coland_error_queue'].endswith('.fifo'):
+ raise Exception("'%s' is not a FIFO queue"
+ % config['sqs_coland_error_queue'])
+
+ # Ensure repo paths exist and are sane; we won't create them.
+ if not os.path.exists(config['integration_path']):
+ raise Exception('failed to find integration_path "%s"'
+ % config['integration_path'])
+ if not os.path.exists('%s/.hg' % config['integration_path']):
+ raise Exception('invalid integration_path "%s"'
+ % config['integration_path'])
+ if not os.path.exists(config['github_path']):
+ raise Exception('failed to find github_path "%s"'
+ % config['github_path'])
+ if not os.path.exists('%s/.git' % config['github_path']):
+ raise Exception('invalid github_path "%s"'
+ % config['github_path'])
+
+ if len(config['coland_github_name'].split('/')) != 2:
+ raise Exception('invalid coland_github_name: "%s" is not in the form '
+ '"user/repo"' % config['coland_github_name'])
+
+ # Fix bugzilla url if required.
+ config['coland_bugzilla_url'] = config['coland_bugzilla_url'].rstrip('/')
+
+
+def validate_request(config, request):
+ # Ensure mandatory fields are provided.
+ required_fields = {'action', 'author', 'autoland-id', 'bug-id',
+ 'bundle-url', 'destination', }
+
+ request_fields = set(request.keys())
+ missing_fields = required_fields - request_fields
+ if missing_fields:
+ raise SqsFatalException(
+ 'missing required field%s: %s'
+ % ('' if len(missing_fields) == 1
+ else 's', ', '.join(sorted(missing_fields))))
+
+ # Validate author, bug-id.
+ if '@' not in parseaddr(request['author'])[1]:
+ raise SqsFatalException('invalid author "%s"' % request['author'])
+ try:
+ request['bug-id'] = int(request['bug-id'])
+ except ValueError:
+ raise SqsFatalException('invalid bug-id "%s"' % request['bug-id'])
+
+ # Validate the S3 bucket.
+ try:
+ url = urlparse.urlparse(request['bundle-url'])
+ except ValueError:
+ raise SqsFatalException('invalid bundle-url "%s": malformed url'
+ % request['bundle-url'])
+
+ if url.scheme != 's3':
+ raise SqsFatalException('invalid bundle-url "%s": not a s3:// url'
+ % request['bundle-url'])
+
+ if url.hostname != config['coland_bucket']:
+ raise SqsFatalException('invalid bundle-url "%s": illegal bucket'
+ % request['bundle-url'])
+
+ # And that the destination is a URL, not just a repo name.
+ try:
+ url = urlparse.urlparse(request['destination'])
+ except ValueError:
+ raise SqsFatalException('invalid destination "%s": malformed url'
+ % request['destination'])
+
+ if url.scheme != 'ssh':
+ raise SqsFatalException('invalid destination "%s": not a ssh:// url'
+ % request['destination'])
+
+ if url.hostname != 'hg.mozilla.org':
+ raise SqsFatalException('invalid destination "%s": illegal hostname'
+ % request['destination'])
+
+
+def prepare_hg_repo(hg_repo, remote_tip):
+ # Checkout latest.
+ run_hg(logger, hg_repo, ['pull', '-r', remote_tip])
+ run_hg(logger, hg_repo, ['update', remote_tip])
+
+ # Remove old rebased commits.
+ rev_selector = 'first(outgoing() and descendants(%s::))' % remote_tip
+ rev = run_hg(logger, hg_repo,
+ ['log', '-T', '{node}', '-r', rev_selector])
+ if rev:
+ run_hg(logger, hg_repo, ['strip', '-r', rev])
+
+
+def get_bug(bugzilla_url, bug_id):
+ try:
+ logger.info('checking visibility of bug %s' % bug_id)
+ return bugsy.Bugsy(bugzilla_url=bugzilla_url).get(bug_id)
+ except bugsy.BugsyException as e:
+ if e.code == 101: # bug_id_does_not_exist
+ raise SqsFatalException('Invalid bug-id %s' % bug_id)
+ if e.code == 102: # bug_access_denied
+ return None
+ raise
+
+
+def download_from_s3(s3_url, aws_key, aws_secret, output_file):
+ # Download from s3 url specified in self.patch_url to a temp file.
+ # Returns the temp filename which must be deleted by the caller.
+
+ url = urlparse.urlparse(s3_url)
+ bucket = url.hostname
+ key = url.path[1:]
+
+ try:
+ s3 = boto3.client('s3',
+ aws_access_key_id=aws_key,
+ aws_secret_access_key=aws_secret)
+ logger.info('downloading %s to %s' % (s3_url, output_file))
+ s3.download_file(bucket, key, output_file)
+ except ClientError as e:
+ error_code = int(e.response['Error']['Code'])
+ if error_code == 404:
+ raise Exception('unable to download %s: file not found' % s3_url)
+ if error_code == 403:
+ raise Exception('unable to download %s: permission denied' % s3_url)
+ raise
+
+
+def extract_bundle_revs(hg_repo, filename):
+ """Return a list of the full SHAs from a hg bundle file"""
+
+ # We cannot use `hg incoming` as a previous failure may have resulted
+ # in the revisions already be in-tree.
+ return [r.strip() for r in
+ run_hg(logger, hg_repo, ['debugbundle', filename]).splitlines()
+ if maybe_revision(r.strip())]
+
+
+def current_heads_set(hg_repo):
+ return set(run_hg(logger, hg_repo,
+ ['heads', '-T', '{node}\n']).splitlines())
+
+
+def get_touched_servo_files(hg_repo, revset):
+ return [f for f in
+ run_hg(logger, hg_repo, ['log', '-T', '{join(files,"\n")}\n',
+ '-r', revset]).splitlines()
+ if f.startswith('servo/')
+ ]
+
+
+def expanded_author(author):
+ # git requires author to be of the form 'Name <email>'. Populate
+ # the name from the email if it's missing instead of failing.
+ author_name, email = parseaddr(author)
+ if not author_name:
+ author = formataddr((email[:email.index('@')], email))
+ logger.warning('"%s" is missing author name, setting to "%s"' %
+ (email, author))
+ return author
+
+
+def create_pr_from_rev(hg_repo, github_pr, rev, pr_args):
+ run_hg(logger, hg_repo, ['update', rev])
+
+ # TODO mess around with encoding
+ repo_path = run_hg(logger, hg_repo, ['root']).rstrip()
+ patch = run_hg(logger, hg_repo, ['diff', '--git', '--cwd', repo_path,
+ '--change', '.', '--root', 'servo/'],
+ log_output=False)
+ with tempfile.NamedTemporaryFile() as temp_file:
+ temp_file.write(patch)
+ temp_file.flush()
+
+ pr_args['reset_branch'] = False
+ pr_args['patch_file'] = temp_file.name
+ try:
+ return github_pr.create_pr_from_patch(**pr_args)
+ except CalledProcessError as e:
+ if 'patch does not apply' in e.output:
+ raise SqsFatalException('Merge conflict while applying '
+ 'co-landing changes to github/servo; '
+ 'please rebase and try again.')
+ raise
+
+
+def abort_rebase(hg_repo):
+ try:
+ run_hg(logger, hg_repo, ['rebase', '--abort'])
+ except hglib.error.CommandError as e:
+ if 'abort: no rebase in progress' not in e.out:
+ raise
+
+
+def rebase_bundle(hg_repo, bundle_revs, remote_tip):
+ # If the bundle added a new head, we need to rebase it.
+ heads = current_heads_set(hg_repo)
+
+ if bundle_revs[-1] in heads and remote_tip in heads:
+ try:
+ run_hg(logger, hg_repo,
+ ['rebase', '--tool', 'internal:merge',
+ '-s', bundle_revs[0], '-d', remote_tip])
+ except hglib.error.CommandError as e:
+ abort_rebase(hg_repo)
+ if 'unresolved conflicts (see hg resolve' in e.out:
+ raise SqsFatalException('Merge conflict while applying '
+ 'co-landing changes to hg/autoland; '
+ 'please rebase and try again.')
+ raise
+
+ bundle_revs = run_hg(logger, hg_repo,
+ ['log', '-T', '{node}\n',
+ '-r', '%s::' % remote_tip]).splitlines()[1:]
+
+ return bundle_revs
+
+
+def delete_github_branch(git, branch_name):
+ # Delete the branch to remove any partial commits/krud.
+ if git.get('branch', '--list', branch_name):
+ logger.info('deleting git branch %s' % branch_name)
+ git.cmd('checkout', 'master')
+ git.cmd('branch', '--delete', '--force', branch_name)
+ # noinspection PyBroadException
+ try:
+ git.cmd('push', 'origin', '--delete', branch_name)
+ except Exception:
+ pass
+
+
+def clean_master(git):
+ git.cmd('checkout', 'master', '--force')
+ git.cmd('reset', '--hard')
+
+
+def create_pr_comment(pr, comment):
+ # Unfortunately github3.py v0.9.6 doesn't implement create_comment.
+ # It's implemented in github3.py v1 however that isn't stable yet.
+ pr._post(pr.comments_url, {'body': comment})
+
+
+def create_bug_comment(bugzilla_url, api_key, bug_id, comment):
+ try:
+ bugzilla = bugsy.Bugsy(bugzilla_url=bugzilla_url, api_key=api_key)
+ bug = bugzilla.get(bug_id)
+ bug.add_comment(comment)
+ except bugsy.BugsyException as e:
+ if e.code == 101: # bug_id_does_not_exist
+ raise SqsFatalException('Invalid bug-id %s' % bug_id)
+ logger.error('Failed to comment on bug %s: %s' % (bug_id, str(e)))
+
+
+def create_from_queue_message(config, message):
+ validate_request(config, message)
+
+ bmo_rest_url = '%s/rest' % config['coland_bugzilla_url'].rstrip('/')
+ bug = get_bug(bmo_rest_url, message['bug-id'])
+ if not bug:
+ raise SqsFatalException('Bug %s is not public' % message['bug-id'])
+
+ repo_cfg = [b'extensions.strip=', b'ui.interactive=False']
+ with hglib.open(config['integration_path'], 'utf-8', repo_cfg) as hg_repo:
+ abort_rebase(hg_repo)
+ clean_hg_repo(logger, config['integration_path'])
+
+ remote_url = run_hg(logger, hg_repo, ['paths', 'default']).rstrip()
+ remote_tip = run_hg(logger, hg_repo,
+ ['identify', remote_url, '-r', 'tip']).rstrip()
+
+ prepare_hg_repo(hg_repo, remote_tip)
+
+ # Expand remote_tip to full hash.
+ remote_tip = run_hg(logger, hg_repo,
+ ['log', '-T', '{node}', '-r', remote_tip])
+
+ bundle_revs = []
+ bundle_file = tempfile.NamedTemporaryFile(delete=False, suffix='.hg')
+ try:
+ bundle_file.close()
+
+ # Download and unbundle.
+ download_from_s3(message['bundle-url'],
+ config['coland_bucket_aws_key'],
+ config['coland_bucket_aws_secret'],
+ bundle_file.name)
+
+ bundle_revs = extract_bundle_revs(hg_repo, bundle_file.name)
+ run_hg(logger, hg_repo, ['unbundle', bundle_file.name])
+ bundle_revs = rebase_bundle(hg_repo, bundle_revs, remote_tip)
+
+ # Ensure servo files have been touched by this commit.
+ if not get_touched_servo_files(hg_repo, '%s::' % bundle_revs[0]):
+ raise SqsFatalException('servo/ not modified')
+
+ # Initialise github.
+ github_pr = GitHubPR(config['coland_github_token'],
+ config['coland_github_name'],
+ config['github_path'])
+
+ # Name each PR branch after the bug ID.
+ branch_name = 'bug-%s' % message['bug-id']
+ existing_pr = github_pr.pr_from_branch(branch_name, state='open')
+
+ # Always work with a clean state.
+ delete_github_branch(github_pr.git, branch_name)
+ clean_master(github_pr.git)
+
+ pr_args = {
+ 'branch_name': branch_name,
+ 'author': expanded_author(message['author']),
+ 'pr_title': 'Gecko Bug %s' % message['bug-id'],
+ 'pr_body': '%s/show_bug.cgi?id=%s\n%s\n\n#gecko-coland %s' %
+ (config['coland_bugzilla_url'], message['bug-id'],
+ bug.summary, message['bundle-url'])
+ }
+ pr_url = None
+
+ for rev in bundle_revs:
+ if not get_touched_servo_files(hg_repo, rev):
+ continue
+
+ logger.info('%s pull-request from %s' %
+ ('updating' if existing_pr else 'creating', rev))
+
+ pr_args['description'] = (
+ run_hg(logger, hg_repo,
+ ['log', '-T', '{desc}', '-r', rev]).rstrip())
+
+ pr = create_pr_from_rev(hg_repo, github_pr, rev, pr_args)
+ pr_url = pr.html_url
+
+ # Bump the priority on fix-up commits.
+ if existing_pr:
+ create_pr_comment(pr, 'replaces %s\n@bors-servo p=1\n' %
+ existing_pr.html_url)
+
+ # Update bugzilla with the PR url.
+ create_bug_comment(bmo_rest_url, config['coland_bugzilla_apikey'],
+ message['bug-id'],
+ 'Servo changes submitted as %s' % pr_url)
+
+ finally:
+ try:
+ os.unlink(bundle_file.name)
+ except OSError:
+ pass
+ try:
+ if bundle_revs:
+ run_hg(logger, hg_repo,
+ ['strip', '--no-backup'] + bundle_revs)
+ except hglib.error.CommandError:
+ logger.warning('ignoring strip failure')