autoland: rework pingback config and code (
bug 1368516) r?smacleod
Rework pingback configuration and code to support multiple pingback schemes,
and move the authentication for pingbacks from config-global to per-pingback
host.
MozReview-Commit-ID: 2eHyD0pTZAs
--- a/ansible/roles/autoland/templates/config.json.j2
+++ b/ansible/roles/autoland/templates/config.json.j2
@@ -1,20 +1,20 @@
{
"auth": {
"mozreview": "{{ secrets.mozreview_token }}"
},
- "bugzilla": {
- "user": "{{ secrets.bugzilla_user }}",
- "passwd": "{{ secrets.bugzilla_password }}"
- },
"database": "dbname={{ secrets.db_name }} user={{ secrets.db_user }} password={{ secrets.db_password }} host={{ secrets.db_host }}",
"repos" : {{ repos | to_nice_json }},
- "pingback_allow": [
- "reviewboard.mozilla.org"
- ],
+ "pingback": {
+ "reviewboard.mozilla.org": {
+ "type": "mozreview",
+ "user": "{{ secrets.bugzilla_user }}",
+ "password": "{{ secrets.bugzilla_password }}"
+ }
+ },
"patch_url_buckets": {
"to-be-determined": {
"aws_access_key_id": "{{ secrets.lando_aws_access_key_id }}",
"aws_secret_access_key": "{{ secrets.lando_aws_secret_access_key }}"
}
}
}
--- a/autoland/autoland/autoland.py
+++ b/autoland/autoland/autoland.py
@@ -5,16 +5,17 @@ import datetime
import json
import logging
import mozreview
import os
import psycopg2
import sys
import time
import traceback
+import urlparse
sys.path.insert(0, os.path.normpath(os.path.join(os.path.normpath(
os.path.abspath(os.path.dirname(__file__))), '..',
'..',
'pylib',
'mozautomation')))
from transplant import (RepoTransplant, PatchTransplant)
@@ -252,38 +253,63 @@ def handle_pending_mozreview_updates(dbc
query = """
select MozreviewUpdate.id,transplant_id,request,data
from MozreviewUpdate inner join Transplant
on (Transplant.id = MozreviewUpdate.transplant_id)
limit %(limit)s
"""
cursor.execute(query, {'limit': MOZREVIEW_COMMENT_LIMIT})
- bugzilla_auth = mozreview.instantiate_authentication()
+ mozreview_pingback = mozreview.MozReviewPingback()
updated = []
all_posted = True
for row in cursor.fetchall():
update_id, transplant_id, request, data = row
+
+ # Validate the pingback hostname is still present in config.json.
pingback_url = request.get('pingback_url')
+ hostname = urlparse.urlparse(pingback_url).hostname
- logger.info('trying to post mozreview update to: %s for request: %s' %
- (pingback_url, transplant_id))
+ if hostname == 'localhost':
+ # localhost pingbacks are always a NO-OP; used during development
+ # and testing.
+ pingback_url = None
+
+ else:
+ if hostname not in config.get('pingback', {}):
+ logging.error('ignoring pingback to %s: unconfigured'
+ % hostname)
+ pingback_url = None
- # We allow empty pingback_urls as they make testing easier. We can
- # always check the logs for misconfigured pingback_urls.
+ # Use the appropriate handler for this pingback.
if pingback_url:
- status_code, text = mozreview.update_review(bugzilla_auth,
- pingback_url, data)
+ pingback_config = config.get('pingback').get(hostname)
+
+ if pingback_config['type'] == 'mozreview':
+ pingback = mozreview_pingback
+
+ else:
+ logging.warning('ignoring pinback to %s: not supported'
+ % hostname)
+ pingback_url = None
+
+ # Update the requester if required.
+ if pingback_url:
+ logger.info('trying to post %s update to: %s for request: %s' %
+ (pingback.name, pingback_url, transplant_id))
+
+ status_code, text = pingback.update(pingback_url, data)
if status_code == 200:
updated.append([update_id])
else:
logger.info('failed: %s - %s' % (status_code, text))
all_posted = False
break
+
else:
updated.append([update_id])
if updated:
query = """
delete from MozreviewUpdate
where id=%s
"""
--- a/autoland/autoland/autoland_rest.py
+++ b/autoland/autoland/autoland_rest.py
@@ -60,21 +60,17 @@ def check_pingback_url(pingback_url):
ip = ipaddress.ip_address(url.hostname)
if ip.is_loopback or ip.is_private:
return True
except ValueError:
# Ignore hostnames and invalid addresses.
pass
# Allow pingbacks to whitelisted hosts from config.json
- for allowed_host in config.get('pingback_allow', []):
- if url.hostname == allowed_host:
- return True
-
- return False
+ return url.hostname in config.get('pingback', {})
def check_patch_url(patch_url):
try:
url = urlparse.urlparse(patch_url)
except ValueError:
logging.error('invalid patch_url "%s": malformed url' % patch_url)
return False
--- a/autoland/autoland/mozreview.py
+++ b/autoland/autoland/mozreview.py
@@ -14,16 +14,17 @@ API_KEY_LOGIN_PATH = ('/api/extensions/m
# }
#
# For user/password authentication:
# "bugzilla": {
# "user": "level1@example.com",
# "passwd": "password",
# }
+
class BugzillaAuthException(Exception):
pass
class BugzillaAuth(object):
"""Base class for authentication."""
_config = None
@@ -39,17 +40,17 @@ class BugzillaAuth(object):
"""Basic HTTP auth credentials, as user/pass tuple."""
return None
class BugzillaAuthPassword(BugzillaAuth):
"""Username/password authentication. Used in dev and test."""
def http_auth(self):
- return self._config['user'], self._config['passwd']
+ return self._config['user'], self._config['password']
class BugzillaAuthApiKey(BugzillaAuth):
"""Username/API-Key authentication."""
_cookies = {}
def headers(self, pingback_url):
@@ -71,33 +72,44 @@ class BugzillaAuthApiKey(BugzillaAuth):
raise BugzillaAuthException('API-Key authentication failed')
self._cookies[host] = 'rbsessionid=%s' % res.cookies['rbsessionid']
headers = super(BugzillaAuthApiKey, self).headers(pingback_url)
headers['Cookie'] = self._cookies[host]
return headers
-def instantiate_authentication():
- """Return the appropriate BugzillaAuth object."""
- bugzilla_config = config.get('bugzilla')
- if 'api-key' in bugzilla_config:
- return BugzillaAuthApiKey(bugzilla_config)
- else:
- return BugzillaAuthPassword(bugzilla_config)
+class MozReviewPingback(object):
+ """Handle updating MozReview/RB requests."""
+
+ def __init__(self):
+ self.name = 'mozreview'
+ self.auth = {}
+ def _auth_for(self, pingback_url):
+ hostname = urlparse.urlparse(pingback_url).hostname
+
+ if hostname not in self.auth:
+ auth_config = config.get('pingback').get(hostname)
+ if 'api-key' in auth_config:
+ self.auth[hostname] = BugzillaAuthApiKey(auth_config)
+ else:
+ self.auth[hostname] = BugzillaAuthPassword(auth_config)
-def update_review(bugzilla_auth, pingback_url, data):
- """Sends the 'data' to the 'pingback_url', handing auth and errors"""
- try:
- res = requests.post(pingback_url,
- data=data,
- headers=bugzilla_auth.headers(pingback_url),
- auth=bugzilla_auth.http_auth())
- if res.status_code == 401:
- raise BugzillaAuthException('Login failure')
- return res.status_code, res.text
- except BugzillaAuthException as e:
- return None, 'Failed to connect authenticate with MozReview: %s' % e
- except requests.exceptions.ConnectionError as e:
- return None, 'Failed to connect to MozReview: %s' % e
- except requests.exceptions.RequestException as e:
- return None, 'Failed to update MozReview: %s' % e
+ return self.auth[hostname]
+
+ def update(self, pingback_url, data):
+ """Sends the 'data' to the 'pingback_url', handing auth and errors"""
+ try:
+ auth = self._auth_for(pingback_url)
+ res = requests.post(pingback_url,
+ data=data,
+ headers=auth.headers(pingback_url),
+ auth=auth.http_auth())
+ if res.status_code == 401:
+ raise BugzillaAuthException('Login failure')
+ return res.status_code, res.text
+ except BugzillaAuthException as e:
+ return None, 'Failed to connect authenticate with MozReview: %s' % e
+ except requests.exceptions.ConnectionError as e:
+ return None, 'Failed to connect to MozReview: %s' % e
+ except requests.exceptions.RequestException as e:
+ return None, 'Failed to update MozReview: %s' % e
--- a/testing/docker/builder-autoland/config.json
+++ b/testing/docker/builder-autoland/config.json
@@ -1,25 +1,25 @@
{
"testing": true,
- "bugzilla": {
- "user": "admin@example.com",
- "passwd": "password"
- },
"auth": {
"autoland": "autoland"
},
"database": "dbname=autoland user=postgres host=autolanddb",
"repos" : {
"test-repo": {
"tree": "test"
}
},
- "pingback_allow": [
- "example.com"
- ],
+ "pingback": {
+ "example.com": {
+ "type": "mozreview",
+ "user": "admin@example.com",
+ "password": "password"
+ }
+ },
"patch_url_buckets": {
"example-bucket": {
"aws_access_key_id": "DEADBEEFF00D",
"aws_secret_access_key": "secret"
}
}
}