autoland: add bucket configuration and whitelisting to patch based requests (bug 1368516) r?smacleod draft
authorbyron jones <glob@mozilla.com>
Tue, 25 Jul 2017 15:57:10 +0800
changeset 11682 602f2d837a3b5052304b475eea62a2de7d6cf976
parent 11681 567825c58a84c178b728bff0922968408085f4e0
child 11683 0ac9ae8c1e3f0b9ea58589504b6c4baeea8baeb2
push id1790
push userbjones@mozilla.com
push dateTue, 19 Sep 2017 04:17:41 +0000
reviewerssmacleod
bugs1368516
autoland: add bucket configuration and whitelisting to patch based requests (bug 1368516) r?smacleod Add the valid bucket name(s) and aws credentials to our config file. The request's patch_url must be a whitelisted s3 bucket url, or a http[s] url that is either loopback or a private IP address; the later is for development/testing only. MozReview-Commit-ID: GIjckwnjV82
ansible/roles/autoland/templates/config.json.j2
autoland/autoland/autoland_rest.py
testing/docker/builder-autoland/config.json
--- a/ansible/roles/autoland/templates/config.json.j2
+++ b/ansible/roles/autoland/templates/config.json.j2
@@ -5,10 +5,16 @@
   "bugzilla": {
     "user": "{{ secrets.bugzilla_user }}",
     "passwd": "{{ secrets.bugzilla_password  }}"
   },
   "database": "dbname={{ secrets.db_name }} user={{ secrets.db_user }} password={{ secrets.db_password }} host={{ secrets.db_host }}",
   "repos" : {{ repos | to_nice_json }},
   "pingback_allow": [
     "reviewboard.mozilla.org"
-  ]
+  ],
+  "patch_url_buckets": {
+    "to-be-determined": {
+      "aws_access_key_id": "{{ secrets.lando_aws_access_key_id }}",
+      "aws_secret_access_key": "{{ secrets.lando_aws_secret_access_key }}"
+    }
+  }
 }
--- a/autoland/autoland/autoland_rest.py
+++ b/autoland/autoland/autoland_rest.py
@@ -67,16 +67,50 @@ def check_pingback_url(pingback_url):
     # Allow pingbacks to whitelisted hosts from config.json
     for allowed_host in config.get('pingback_allow', []):
         if url.hostname == allowed_host:
             return True
 
     return False
 
 
+def check_patch_url(patch_url):
+    try:
+        url = urlparse.urlparse(patch_url)
+    except ValueError:
+        logging.error('invalid patch_url "%s": malformed url' % patch_url)
+        return False
+
+    # http is only supported when using loopback and private IPs (for dev/test)
+    if url.scheme in ('http', 'https'):
+        if url.hostname == 'localhost':
+            return True
+        try:
+            ip = ipaddress.ip_address(url.hostname)
+            if ip.is_loopback or ip.is_private:
+                return True
+        except ValueError:
+            # Ignore hostnames and invalid addresses.
+            pass
+        logging.error('invalid patch_url "%s": public http url' % patch_url)
+
+    # Deployed environments must use the s3 scheme.  s3://bucket/path/to/file
+    if url.scheme != 's3':
+        logging.error('invalid patch_url "%s": not a s3:// url' % patch_url)
+        return False
+
+    # Allow patches only from buckets configured in config.json.
+    if url.hostname not in config.get('patch_url_buckets', {}):
+        logging.error('invalid patch_url "%s": not whitelisted by config'
+                      % patch_url)
+        return False
+
+    return True
+
+
 def validate_request(request):
     if request.json is None:
         raise ValueError('missing json')
     request_json = request.json
 
     required = {'ldap_username', 'tree', 'rev', 'pingback_url', 'destination'}
     optional = set()
 
@@ -118,16 +152,21 @@ def validate_request(request):
     if extra:
         raise ValueError('unexpected field%s: %s' % (
             '' if len(extra) == 1 else 's',
             ', '.join(sorted(extra))))
 
     if not check_pingback_url(request_json['pingback_url']):
         raise ValueError('bad pingback_url')
 
+    if is_patch:
+        for patch_url in request_json['patch_urls']:
+            if not check_patch_url(patch_url):
+                raise ValueError('bad patch_url')
+
 
 @app.route('/autoland', methods=['POST'])
 def autoland():
     """
     Autoland a patch from one tree to another.
 
     Example repository based landing request:
     (All fields are required except for push_bookmark)
--- a/testing/docker/builder-autoland/config.json
+++ b/testing/docker/builder-autoland/config.json
@@ -10,10 +10,16 @@
   "database": "dbname=autoland user=postgres host=autolanddb",
   "repos" : {
     "test-repo": {
       "tree": "test"
     }
   },
   "pingback_allow": [
     "example.com"
-  ]
+  ],
+  "patch_url_buckets": {
+    "example-bucket": {
+      "aws_access_key_id": "DEADBEEFF00D",
+      "aws_secret_access_key": "secret"
+    }
+  }
 }