Bug 1305804 - Resolve hostname before attempting to use pip link; r?ted
In automation, we try to use pypi.pvt.build.mozilla.org nearly
everywhere. This hostname doesn't resolve in TaskCluster and
outside of buildbot automation.
A consequence of work in
bug 1304176 and using a modern pip is
that we attempt to connect to all defined pip links. This was resulting
in pip attempting connections to pypi.pvt.build.mozilla.org. And
since pip was attempting retries, this resulted in a several
second delay for all pip operations if that host didn't resolve.
This commit adds a DNS lookup in mozharness before using a pip
--link. We spend a little bit of overhead in mozharness doing a
DNS lookup. In return, we guarantee we'll avoid a multiple second
pause if any links don't resolve. This is somewhat hacky.
But it seems like the easiest solution.
MozReview-Commit-ID: EecqBQSW75R
--- a/testing/mozharness/mozharness/base/python.py
+++ b/testing/mozharness/mozharness/base/python.py
@@ -8,16 +8,17 @@
'''
import distutils.version
import os
import subprocess
import sys
import time
import json
+import socket
import traceback
import urlparse
import mozharness
from mozharness.base.script import (
PostScriptAction,
PostScriptRun,
PreScriptAction,
@@ -261,18 +262,26 @@ class VirtualenvMixin(object):
self.fatal("install_module() doesn't understand an install_method of %s!" % install_method)
# Add --find-links pages to look at. Add --trusted-host automatically if
# the host isn't secure. This allows modern versions of pip to connect
# without requiring an override.
proxxy = Proxxy(self.config, self.log_obj)
trusted_hosts = set()
for link in proxxy.get_proxies_and_urls(c.get('find_links', [])):
+ parsed = urlparse.urlparse(link)
+
+ try:
+ socket.gethostbyname(parsed.hostname)
+ except socket.gaierror as e:
+ self.info('error resolving %s (ignoring): %s' %
+ (parsed.hostname, e.message))
+ continue
+
command.extend(["--find-links", link])
- parsed = urlparse.urlparse(link)
if parsed.scheme != 'https':
trusted_hosts.add(parsed.hostname)
if self.pip_version >= distutils.version.LooseVersion('6.0'):
for host in sorted(trusted_hosts):
command.extend(['--trusted-host', host])
# module_url can be None if only specifying requirements files