autoland: rework pingback config and code (bug 1368516) r?smacleod draft
authorbyron jones <glob@mozilla.com>
Thu, 17 Aug 2017 12:19:26 +0800
changeset 11695 03db54b2ad8d7ab1733504633daf7a134bebcb9f
parent 11694 2044b6101b8b22b6ab3f7572ea52ed4a0b56b377
child 11696 b4d8dbb82ac228322c75951192cf615184139ae1
push id1791
push userbjones@mozilla.com
push dateTue, 19 Sep 2017 04:21:13 +0000
reviewerssmacleod
bugs1368516
autoland: rework pingback config and code (bug 1368516) r?smacleod Rework pingback configuration and code to support multiple pingback schemes, and move the authentication for pingbacks from config-global to per-pingback host. MozReview-Commit-ID: 2eHyD0pTZAs
ansible/roles/autoland/templates/config.json.j2
autoland/autoland/autoland.py
autoland/autoland/autoland_rest.py
autoland/autoland/mozreview.py
testing/docker/builder-autoland/config.json
--- a/ansible/roles/autoland/templates/config.json.j2
+++ b/ansible/roles/autoland/templates/config.json.j2
@@ -1,20 +1,20 @@
 {
   "auth": {
     "mozreview": "{{ secrets.mozreview_token }}"
   },
-  "bugzilla": {
-    "user": "{{ secrets.bugzilla_user }}",
-    "passwd": "{{ secrets.bugzilla_password  }}"
-  },
   "database": "dbname={{ secrets.db_name }} user={{ secrets.db_user }} password={{ secrets.db_password }} host={{ secrets.db_host }}",
   "repos" : {{ repos | to_nice_json }},
-  "pingback_allow": [
-    "reviewboard.mozilla.org"
-  ],
+  "pingback": {
+    "reviewboard.mozilla.org": {
+      "type": "mozreview",
+      "user": "{{ secrets.bugzilla_user }}",
+      "password": "{{ secrets.bugzilla_password  }}"
+    }
+  },
   "patch_url_buckets": {
     "to-be-determined": {
       "aws_access_key_id": "{{ secrets.lando_aws_access_key_id }}",
       "aws_secret_access_key": "{{ secrets.lando_aws_secret_access_key }}"
     }
   }
 }
--- a/autoland/autoland/autoland.py
+++ b/autoland/autoland/autoland.py
@@ -5,16 +5,17 @@ import datetime
 import json
 import logging
 import mozreview
 import os
 import psycopg2
 import sys
 import time
 import traceback
+import urlparse
 
 sys.path.insert(0, os.path.normpath(os.path.join(os.path.normpath(
                 os.path.abspath(os.path.dirname(__file__))), '..',
                                                              '..',
                                                              'pylib',
                                                              'mozautomation')))
 
 from transplant import (RepoTransplant, PatchTransplant)
@@ -252,38 +253,63 @@ def handle_pending_mozreview_updates(dbc
     query = """
         select MozreviewUpdate.id,transplant_id,request,data
         from MozreviewUpdate inner join Transplant
         on (Transplant.id = MozreviewUpdate.transplant_id)
         limit %(limit)s
     """
     cursor.execute(query, {'limit': MOZREVIEW_COMMENT_LIMIT})
 
-    bugzilla_auth = mozreview.instantiate_authentication()
+    mozreview_pingback = mozreview.MozReviewPingback()
 
     updated = []
     all_posted = True
     for row in cursor.fetchall():
         update_id, transplant_id, request, data = row
+
+        # Validate the pingback hostname is still present in config.json.
         pingback_url = request.get('pingback_url')
+        hostname = urlparse.urlparse(pingback_url).hostname
 
-        logger.info('trying to post mozreview update to: %s for request: %s' %
-                    (pingback_url, transplant_id))
+        if hostname == 'localhost':
+            # localhost pingbacks are always a NO-OP; used during development
+            # and testing.
+            pingback_url = None
+
+        else:
+            if hostname not in config.get('pingback', {}):
+                logging.error('ignoring pingback to %s: unconfigured'
+                              % hostname)
+                pingback_url = None
 
-        # We allow empty pingback_urls as they make testing easier. We can
-        # always check the logs for misconfigured pingback_urls.
+        # Use the appropriate handler for this pingback.
         if pingback_url:
-            status_code, text = mozreview.update_review(bugzilla_auth,
-                                                        pingback_url, data)
+            pingback_config = config.get('pingback').get(hostname)
+
+            if pingback_config['type'] == 'mozreview':
+                pingback = mozreview_pingback
+
+            else:
+                logging.warning('ignoring pinback to %s: not supported'
+                                % hostname)
+                pingback_url = None
+
+        # Update the requester if required.
+        if pingback_url:
+            logger.info('trying to post %s update to: %s for request: %s' %
+                        (pingback.name, pingback_url, transplant_id))
+
+            status_code, text = pingback.update(pingback_url, data)
             if status_code == 200:
                 updated.append([update_id])
             else:
                 logger.info('failed: %s - %s' % (status_code, text))
                 all_posted = False
                 break
+
         else:
             updated.append([update_id])
 
     if updated:
         query = """
             delete from MozreviewUpdate
             where id=%s
         """
--- a/autoland/autoland/autoland_rest.py
+++ b/autoland/autoland/autoland_rest.py
@@ -60,21 +60,17 @@ def check_pingback_url(pingback_url):
         ip = ipaddress.ip_address(url.hostname)
         if ip.is_loopback or ip.is_private:
             return True
     except ValueError:
         # Ignore hostnames and invalid addresses.
         pass
 
     # Allow pingbacks to whitelisted hosts from config.json
-    for allowed_host in config.get('pingback_allow', []):
-        if url.hostname == allowed_host:
-            return True
-
-    return False
+    return url.hostname in config.get('pingback', {})
 
 
 def check_patch_url(patch_url):
     try:
         url = urlparse.urlparse(patch_url)
     except ValueError:
         logging.error('invalid patch_url "%s": malformed url' % patch_url)
         return False
--- a/autoland/autoland/mozreview.py
+++ b/autoland/autoland/mozreview.py
@@ -14,16 +14,17 @@ API_KEY_LOGIN_PATH = ('/api/extensions/m
 #   }
 #
 # For user/password authentication:
 #   "bugzilla": {
 #       "user": "level1@example.com",
 #       "passwd": "password",
 #   }
 
+
 class BugzillaAuthException(Exception):
     pass
 
 
 class BugzillaAuth(object):
     """Base class for authentication."""
 
     _config = None
@@ -39,17 +40,17 @@ class BugzillaAuth(object):
         """Basic HTTP auth credentials, as user/pass tuple."""
         return None
 
 
 class BugzillaAuthPassword(BugzillaAuth):
     """Username/password authentication.  Used in dev and test."""
 
     def http_auth(self):
-        return self._config['user'], self._config['passwd']
+        return self._config['user'], self._config['password']
 
 
 class BugzillaAuthApiKey(BugzillaAuth):
     """Username/API-Key authentication."""
 
     _cookies = {}
 
     def headers(self, pingback_url):
@@ -71,33 +72,44 @@ class BugzillaAuthApiKey(BugzillaAuth):
                 raise BugzillaAuthException('API-Key authentication failed')
             self._cookies[host] = 'rbsessionid=%s' % res.cookies['rbsessionid']
 
         headers = super(BugzillaAuthApiKey, self).headers(pingback_url)
         headers['Cookie'] = self._cookies[host]
         return headers
 
 
-def instantiate_authentication():
-    """Return the appropriate BugzillaAuth object."""
-    bugzilla_config = config.get('bugzilla')
-    if 'api-key' in bugzilla_config:
-        return BugzillaAuthApiKey(bugzilla_config)
-    else:
-        return BugzillaAuthPassword(bugzilla_config)
+class MozReviewPingback(object):
+    """Handle updating MozReview/RB requests."""
+
+    def __init__(self):
+        self.name = 'mozreview'
+        self.auth = {}
 
+    def _auth_for(self, pingback_url):
+        hostname = urlparse.urlparse(pingback_url).hostname
+
+        if hostname not in self.auth:
+            auth_config = config.get('pingback').get(hostname)
+            if 'api-key' in auth_config:
+                self.auth[hostname] = BugzillaAuthApiKey(auth_config)
+            else:
+                self.auth[hostname] = BugzillaAuthPassword(auth_config)
 
-def update_review(bugzilla_auth, pingback_url, data):
-    """Sends the 'data' to the 'pingback_url', handing auth and errors"""
-    try:
-        res = requests.post(pingback_url,
-                            data=data,
-                            headers=bugzilla_auth.headers(pingback_url),
-                            auth=bugzilla_auth.http_auth())
-        if res.status_code == 401:
-            raise BugzillaAuthException('Login failure')
-        return res.status_code, res.text
-    except BugzillaAuthException as e:
-        return None, 'Failed to connect authenticate with MozReview: %s' % e
-    except requests.exceptions.ConnectionError as e:
-        return None, 'Failed to connect to MozReview: %s' % e
-    except requests.exceptions.RequestException as e:
-        return None, 'Failed to update MozReview: %s' % e
+        return self.auth[hostname]
+
+    def update(self, pingback_url, data):
+        """Sends the 'data' to the 'pingback_url', handing auth and errors"""
+        try:
+            auth = self._auth_for(pingback_url)
+            res = requests.post(pingback_url,
+                                data=data,
+                                headers=auth.headers(pingback_url),
+                                auth=auth.http_auth())
+            if res.status_code == 401:
+                raise BugzillaAuthException('Login failure')
+            return res.status_code, res.text
+        except BugzillaAuthException as e:
+            return None, 'Failed to connect authenticate with MozReview: %s' % e
+        except requests.exceptions.ConnectionError as e:
+            return None, 'Failed to connect to MozReview: %s' % e
+        except requests.exceptions.RequestException as e:
+            return None, 'Failed to update MozReview: %s' % e
--- a/testing/docker/builder-autoland/config.json
+++ b/testing/docker/builder-autoland/config.json
@@ -1,25 +1,25 @@
 {
   "testing": true,
-  "bugzilla": {
-    "user": "admin@example.com",
-    "passwd": "password"
-  },
   "auth": {
     "autoland": "autoland"
   },
   "database": "dbname=autoland user=postgres host=autolanddb",
   "repos" : {
     "test-repo": {
       "tree": "test"
     }
   },
-  "pingback_allow": [
-    "example.com"
-  ],
+  "pingback": {
+    "example.com": {
+      "type": "mozreview",
+      "user": "admin@example.com",
+      "password": "password"
+    }
+  },
   "patch_url_buckets": {
     "example-bucket": {
       "aws_access_key_id": "DEADBEEFF00D",
       "aws_secret_access_key": "secret"
     }
   }
 }