autoland: add landing of patch based requests (bug 1368516) r?smacleod draft
authorbyron jones <glob@mozilla.com>
Mon, 31 Jul 2017 15:25:04 +0800
changeset 11688 39c03e85555f860d26d6a204f74eef14226c9aad
parent 11687 5ef83a3f1a74b9a89233ae3278bd3ec081d677a9
child 11689 97120ba594289d96b466b63999ce8639d71cbafb
child 11693 92eb2fcffbc6022ab88b6b466f5efe4ddc8872ef
push id1790
push userbjones@mozilla.com
push dateTue, 19 Sep 2017 04:17:41 +0000
reviewerssmacleod
bugs1368516
autoland: add landing of patch based requests (bug 1368516) r?smacleod Adds support for landing http[s] hosted patch based requests. S3 will be implemented in a later change. MozReview-Commit-ID: KgLdT8KU5ya
autoland/autoland/autoland.py
autoland/autoland/transplant.py
autoland/tests/test-post-autoland-job-from-patch.t
--- a/autoland/autoland/autoland.py
+++ b/autoland/autoland/autoland.py
@@ -12,17 +12,17 @@ import time
 import traceback
 
 sys.path.insert(0, os.path.normpath(os.path.join(os.path.normpath(
                 os.path.abspath(os.path.dirname(__file__))), '..',
                                                              '..',
                                                              'pylib',
                                                              'mozautomation')))
 
-from transplant import RepoTransplant
+from transplant import (RepoTransplant, PatchTransplant)
 import treestatus
 
 
 # max attempts to transplant before bailing
 MAX_TRANSPLANT_ATTEMPTS = 50
 
 # max updates to post to reviewboard / iteration
 MOZREVIEW_COMMENT_LIMIT = 10
@@ -91,18 +91,17 @@ def handle_pending_transplants(dbconn):
         # to binary because command arguments aren't unicode.
         destination = destination.encode('ascii')
         requester = request['ldap_username']
         tree = request['tree'].encode('ascii')
         rev = request['rev'].encode('ascii')
         trysyntax = request.get('trysyntax', '')
         push_bookmark = request.get('push_bookmark', '').encode('ascii')
         commit_descriptions = request.get('commit_descriptions')
-        patch_urls = map(lambda u: u.encode('ascii'),
-                         request.get('patch_urls', []))
+        patch_urls = [u.encode('ascii') for u in request.get('patch_urls', [])]
 
         repo_config = config.get_repo(tree)
         if not repo_config['tree']:
             # Trees not present on treestatus cannot be closed.
             tree_open = True
         else:
             # When pushing to try we need to check if try is open, not the
             # tree for the source repo.
@@ -118,25 +117,24 @@ def handle_pending_transplants(dbconn):
         attempts = 0
         started = datetime.datetime.now()
         landed = False
         while attempts < MAX_TRANSPLANT_ATTEMPTS:
             logger.info('initiating transplant from tree: %s rev: %s '
                         'to destination: %s, attempt %s' % (
                             tree, rev, destination, attempts + 1))
 
-            if patch_urls:
-                result = 'patch based landings not implemented'
-                landed = False
-                break
-
             os.environ['AUTOLAND_REQUEST_USER'] = requester
             try:
-                with RepoTransplant(tree, destination, rev,
-                                    commit_descriptions) as tp:
+                if patch_urls:
+                    tp = PatchTransplant(tree, destination, rev, patch_urls)
+                else:
+                    tp = RepoTransplant(tree, destination, rev,
+                                        commit_descriptions)
+                with tp:
                     if trysyntax:
                         result = tp.push_try(str(trysyntax))
                     elif push_bookmark:
                         result = tp.push_bookmark(push_bookmark)
                     else:
                         result = tp.push()
                 landed = True
             except Exception as e:
--- a/autoland/autoland/transplant.py
+++ b/autoland/autoland/transplant.py
@@ -258,8 +258,28 @@ class RepoTransplant(Transplant):
             logger.error('unexpected outgoing commits:')
             for commit in outgoing:
                 logger.error('outgoing: %s: %s' % (commit[1], commit[5]))
 
             raise Exception("We're sorry - something has gone wrong while "
                             "rewriting or rebasing your commits. The commits "
                             "being pushed no longer match what was requested. "
                             "Please file a bug.")
+
+
+class PatchTransplant(Transplant):
+    def __init__(self, tree, destination, rev, patch_urls):
+        self.patch_urls = patch_urls
+
+        super(PatchTransplant, self).__init__(tree, destination, rev)
+
+    def apply_changes(self, remote_tip):
+        assert self.patch_urls, 'patch_urls not provided'
+
+        for patch_url in self.patch_urls:
+            if patch_url.startswith('s3://'):
+                # Download patch to temp file and import
+                raise Exception('importing patches from s3 not implemented')
+
+            else:
+                self.run_hg(['update', remote_tip])
+                output = self.run_hg(['import', patch_url])
+                logger.info(output)
--- a/autoland/tests/test-post-autoland-job-from-patch.t
+++ b/autoland/tests/test-post-autoland-job-from-patch.t
@@ -1,151 +1,211 @@
 #require mozreviewdocker
 
   $ . $TESTDIR/hgext/reviewboard/tests/helpers.sh
   $ commonenv
   $ mozreview create-user cthulhu@example.com password 'Cthulhu :cthulhu'
   Created user 6
-
-Create an initial revision.
+  $ bugzilla create-bug TestProduct TestComponent 'First Bug'
+  $ cd client
 
-  $ cd client
-  $ echo foo > foo
-  $ hg commit -A -m 'root commit'
+Create a commit to test
+
+  $ echo initial > foo
+  $ hg commit -A -m 'Bug 1 - some stuff; r?cthulhu'
   adding foo
-  $ hg phase --public -r .
-
-Create a commit to test on Try
-
-  $ bugzilla create-bug TestProduct TestComponent 'First Bug'
-  $ echo initial > foo
-  $ hg commit -m 'Bug 1 - some stuff; r?cthulhu'
   $ hg push --config reviewboard.autopublish=false
   pushing to ssh://$DOCKER_HOSTNAME:$HGPORT6/test-repo
   (adding commit id to 1 changesets)
-  saved backup bundle to $TESTTMP/client/.hg/strip-backup/633b0929fc18-25aef645-addcommitid.hg (glob)
+  saved backup bundle to $TESTTMP/client/.hg/strip-backup/*-addcommitid.hg (glob)
   searching for changes
   remote: adding changesets
   remote: adding manifests
   remote: adding file changes
-  remote: added 2 changesets with 2 changes to 1 files
+  remote: added 1 changesets with 1 changes to 1 files
   remote: recorded push in pushlog
   submitting 1 changesets for review
   
-  changeset:  1:e2507be7827c
+  changeset:  0:4b444b4e2552
   summary:    Bug 1 - some stuff; r?cthulhu
   review:     http://$DOCKER_HOSTNAME:$HGPORT1/r/2 (draft)
   
   review id:  bz://1/mynick
   review url: http://$DOCKER_HOSTNAME:$HGPORT1/r/1 (draft)
   (visit review url to publish these review requests so others can see them)
   $ REV=`hg log -r . --template "{node|short}"`
-  $ REV_NO=`hg log -r . --template "{rev}"`
 
 Ensure Autoland started without errors
 
   $ mozreview exec autoland tail -n 20 /home/autoland/autoland.log
   starting autoland
   * autoland INFO starting autoland (glob)
 
 Posting a job with bad credentials should fail
 
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p$REV_NO try http://localhost:9898 --user blah --password blah --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p0 try http://localhost:9898 --user blah --password blah --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (401, u'Login required')
   $ mozreview exec autoland tail -n1 /var/log/apache2/error.log
   * WARNING:root:Failed authentication for "blah" from * (glob)
 
+Post a job from http url should fail
+
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p1 inbound http://localhost:9898 --patch-url http://example.com/p2.patch
+  (400, u'{\n  "error": "Bad request: bad patch_url"\n}')
+
 Post a job from s3 url
 
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p0 inbound http://localhost:9898 --patch-url "s3://example-bucket/p1.patch"
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p2 inbound http://localhost:9898 --patch-url s3://example-bucket/p1.patch
   (200, u'{\n  "request_id": 1\n}')
+  $ ottoland autoland-job-status $AUTOLAND_URL 1 --poll
+  (200, u'{\n  "destination": "inbound", \n  "error_msg": "importing patches from s3 not implemented", \n  "landed": false, \n  "ldap_username": "autolanduser@example.com", \n  "patch_urls": [\n    "s3://example-bucket/p1.patch"\n  ], \n  "result": "", \n  "rev": "p2", \n  "tree": "test-repo"\n}')
 
-Post a job from localhost
-
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p1 inbound http://localhost:9898 --patch-url "http://localhost/path/to/p0.patch"
-  (200, u'{\n  "request_id": 2\n}')
+Post a job from private ip
 
-Post a job from http url
-
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p2 inbound http://localhost:9898 --patch-url "http://example.com/p2.patch"
-  (400, u'{\n  "error": "Bad request: bad patch_url"\n}')
-
-Post a job with try syntax
-
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p3 try http://localhost:9898 --trysyntax "stuff" --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
-  (400, u'{\n  "error": "Bad request: trysyntax is not supported with patch_urls"\n}')
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p3 inbound http://localhost:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  (200, u'{\n  "request_id": 2\n}')
+  $ ottoland autoland-job-status $AUTOLAND_URL 2 --poll
+  (200, u'{\n  "destination": "inbound", \n  "error_msg": "", \n  "landed": true, \n  "ldap_username": "autolanduser@example.com", \n  "patch_urls": [\n    "http://$DOCKER_HOSTNAME:$HGPORT/test-repo/raw-rev/4b444b4e2552"\n  ], \n  "result": null, \n  "rev": "p3", \n  "tree": "test-repo"\n}')
+  $ mozreview exec autoland hg log /repos/test-repo/ --template '{rev}:{desc\|firstline}:{phase}\\n'
+  0:Bug 1 - some stuff; r?cthulhu:public
 
 Post a job using a bookmark
 
   $ echo foo2 > foo
   $ hg commit -m 'Bug 1 - more goodness; r?cthulhu'
-  $ hg push --config reviewboard.autopublish=false
+  $ hg push --config reviewboard.autopublish=false -c .
   pushing to ssh://$DOCKER_HOSTNAME:$HGPORT6/test-repo
   searching for changes
   remote: adding changesets
   remote: adding manifests
   remote: adding file changes
   remote: added 1 changesets with 1 changes to 1 files
   remote: recorded push in pushlog
-  submitting 2 changesets for review
+  submitting 1 changesets for review
   
-  changeset:  1:e2507be7827c
-  summary:    Bug 1 - some stuff; r?cthulhu
+  changeset:  1:d06c12941c46
+  summary:    Bug 1 - more goodness; r?cthulhu
   review:     http://$DOCKER_HOSTNAME:$HGPORT1/r/2 (draft)
   
-  changeset:  2:373b6ff60965
-  summary:    Bug 1 - more goodness; r?cthulhu
-  review:     http://$DOCKER_HOSTNAME:$HGPORT1/r/3 (draft)
-  
   review id:  bz://1/mynick
   review url: http://$DOCKER_HOSTNAME:$HGPORT1/r/1 (draft)
   (visit review url to publish these review requests so others can see them)
   $ REV=`hg log -r . --template "{node|short}"`
 
   $ ottoland post-autoland-job $AUTOLAND_URL test-repo p4 inbound http://localhost:9898 --push-bookmark "bookmark" --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (200, u'{\n  "request_id": 3\n}')
   $ ottoland autoland-job-status $AUTOLAND_URL 3 --poll
-  (200, u'{\n  "destination": "inbound", \n  "error_msg": "patch based landings not implemented", \n  "landed": false, \n  "ldap_username": "autolanduser@example.com", \n  "patch_urls": [\n    "http://$DOCKER_HOSTNAME:$HGPORT/test-repo/raw-rev/373b6ff60965"\n  ], \n  "push_bookmark": "bookmark", \n  "result": "", \n  "rev": "p4", \n  "tree": "test-repo"\n}')
+  (200, u'{\n  "destination": "inbound", \n  "error_msg": "", \n  "landed": true, \n  "ldap_username": "autolanduser@example.com", \n  "patch_urls": [\n    "http://$DOCKER_HOSTNAME:$HGPORT/test-repo/raw-rev/d06c12941c46"\n  ], \n  "push_bookmark": "bookmark", \n  "result": null, \n  "rev": "p4", \n  "tree": "test-repo"\n}')
   $ mozreview exec autoland hg log /repos/inbound-test-repo/ --template '{rev}:{desc\|firstline}:{phase}\\n'
+  1:Bug 1 - more goodness; r?cthulhu:public
+  0:Bug 1 - some stuff; r?cthulhu:public
+
+Post a job with unicode
+
+  $ echo foo3 > foo
+  $ hg commit --encoding utf-8 -m 'Bug 1 - こんにちは; r?cthulhu'
+  $ hg push --config reviewboard.autopublish=false -c .
+  pushing to ssh://$DOCKER_HOSTNAME:$HGPORT6/test-repo
+  searching for changes
+  remote: adding changesets
+  remote: adding manifests
+  remote: adding file changes
+  remote: added 1 changesets with 1 changes to 1 files
+  remote: recorded push in pushlog
+  submitting 1 changesets for review
+  
+  changeset:  2:cd45da3f63e8
+  summary:    Bug 1 - ?????; r?cthulhu
+  review:     http://$DOCKER_HOSTNAME:$HGPORT1/r/2 (draft)
+  
+  review id:  bz://1/mynick
+  review url: http://$DOCKER_HOSTNAME:$HGPORT1/r/1 (draft)
+  (visit review url to publish these review requests so others can see them)
+  $ REV=`hg log -r . --template "{node|short}"`
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p5 inbound http://localhost:9898 --push-bookmark "bookmark" --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  (200, u'{\n  "request_id": 4\n}')
+  $ ottoland autoland-job-status $AUTOLAND_URL 4 --poll
+  (200, u'{\n  "destination": "inbound", \n  "error_msg": "", \n  "landed": true, \n  "ldap_username": "autolanduser@example.com", \n  "patch_urls": [\n    "http://$DOCKER_HOSTNAME:$HGPORT/test-repo/raw-rev/cd45da3f63e8"\n  ], \n  "push_bookmark": "bookmark", \n  "result": null, \n  "rev": "p5", \n  "tree": "test-repo"\n}')
+  $ mozreview exec autoland hg log --encoding=utf-8 /repos/test-repo/ --template '{rev}:{desc\|firstline}:{phase}\\n'
+  2:Bug 1 - \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf; r?cthulhu:public (esc)
+  1:Bug 1 - more goodness; r?cthulhu:public
+  0:Bug 1 - some stuff; r?cthulhu:public
+
+Create a commit to test on Try
+
+  $ echo try > foo
+  $ hg commit -m 'Bug 1 - some stuff; r?cthulhu'
+  $ hg push --config reviewboard.autopublish=false
+  pushing to ssh://$DOCKER_HOSTNAME:$HGPORT6/test-repo
+  searching for changes
+  remote: adding changesets
+  remote: adding manifests
+  remote: adding file changes
+  remote: added 1 changesets with 1 changes to 1 files
+  remote: recorded push in pushlog
+  submitting 4 changesets for review
+  
+  changeset:  0:4b444b4e2552
+  summary:    Bug 1 - some stuff; r?cthulhu
+  review:     http://$DOCKER_HOSTNAME:$HGPORT1/r/3 (draft)
+  
+  changeset:  1:d06c12941c46
+  summary:    Bug 1 - more goodness; r?cthulhu
+  review:     http://$DOCKER_HOSTNAME:$HGPORT1/r/4 (draft)
+  
+  changeset:  2:cd45da3f63e8
+  summary:    Bug 1 - ?????; r?cthulhu
+  review:     http://$DOCKER_HOSTNAME:$HGPORT1/r/2 (draft)
+  
+  changeset:  3:e525ec140d61
+  summary:    Bug 1 - some stuff; r?cthulhu
+  review:     http://$DOCKER_HOSTNAME:$HGPORT1/r/5 (draft)
+  
+  review id:  bz://1/mynick
+  review url: http://$DOCKER_HOSTNAME:$HGPORT1/r/1 (draft)
+  (visit review url to publish these review requests so others can see them)
+  $ REV=`hg log -r . --template "{node|short}"`
+
+Post a job with try syntax
+
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p6 try http://localhost:9898 --trysyntax "stuff" --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  (400, u'{\n  "error": "Bad request: trysyntax is not supported with patch_urls"\n}')
 
 Getting status for an unknown job should return a 404
 
   $ ottoland autoland-job-status $AUTOLAND_URL 42
   (404, u'{\n  "error": "Not found"\n}')
 
-  $ mozreview exec autoland hg log --encoding=utf-8 /repos/test-repo/ --template '{rev}:{desc\|firstline}:{phase}\\n'
-
-  $ mozreview exec autoland hg log /repos/try/ --template '{rev}:{desc\|firstline}:{phase}\\n'
-
-  $ mozreview exec autoland hg log --encoding=utf-8 /repos/inbound-test-repo/ --template '{rev}:{desc\|firstline}:{phase}\\n'
-
 Test pingback url whitelist.  localhost, private IPs, and example.com are in
 the whitelist. example.org is not.
 
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p5 inbound1 http://example.com:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
-  (200, u'{\n  "request_id": 4\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p6 inbound2 http://localhost --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p7 inbound1 http://example.com:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (200, u'{\n  "request_id": 5\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p7 inbound3 http://127.0.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p8 inbound2 http://localhost --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (200, u'{\n  "request_id": 6\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p8 inbound4 http://192.168.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p9 inbound2 http://localhost --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (200, u'{\n  "request_id": 7\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p9 inbound5 http://172.16.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p10 inbound3 http://127.0.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (200, u'{\n  "request_id": 8\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p10 inbound6 http://10.0.0.1:443 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p11 inbound4 http://192.168.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (200, u'{\n  "request_id": 9\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p11 inbound7 http://8.8.8.8:443 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p12 inbound5 http://172.16.0.1 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  (200, u'{\n  "request_id": 10\n}')
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p13 inbound6 http://10.0.0.1:443 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  (200, u'{\n  "request_id": 11\n}')
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p14 inbound7 http://8.8.8.8:443 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (400, u'{\n  "error": "Bad request: bad pingback_url"\n}')
-  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p12 inbound8 http://example.org:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
+  $ ottoland post-autoland-job $AUTOLAND_URL test-repo p15 inbound8 http://example.org:9898 --patch-url ${MERCURIAL_URL}test-repo/raw-rev/$REV
   (400, u'{\n  "error": "Bad request: bad pingback_url"\n}')
 
 Post the same job twice.  Start with stopping the autoland service to
 guarentee the first request is still in the queue when the second is submitted.
 
   $ PID=`mozreview exec autoland ps x | grep autoland.py | grep -v grep | awk '{ print $1 }'`
   $ mozreview exec autoland kill $PID
+  [1] (?)
   $ ottoland post-autoland-job $AUTOLAND_URL test-repo $REV try http://localhost:9898 --trysyntax "stuff"
-  (200, u'{\n  "request_id": 10\n}')
+  (200, u'{\n  "request_id": 12\n}')
   $ ottoland post-autoland-job $AUTOLAND_URL test-repo $REV try http://localhost:9898 --trysyntax "stuff"
-  (400, u'{\n  "error": "Bad Request: a request to land revision 373b6ff60965 to try is already in progress"\n}')
+  (400, u'{\n  "error": "Bad Request: a request to land revision e525ec140d61 to try is already in progress"\n}')
 
   $ mozreview stop
   stopped 9 containers