Bug 1341215 - Remove the dependency on the python taskcluster client module for `mach artifact`. r?chmanchester
This also removes the need for a virtualenv.
--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -56,17 +56,21 @@ import shutil
import stat
import subprocess
import tarfile
import tempfile
import urlparse
import zipfile
import pylru
-import taskcluster
+from taskgraph.util.taskcluster import (
+ find_task_id,
+ get_artifact_url,
+ list_artifacts,
+)
from mozbuild.action.test_archive import OBJDIR_TEST_FILES
from mozbuild.util import (
ensureParentDir,
FileAvoidWrite,
)
import mozinstall
from mozpack.files import (
@@ -629,18 +633,16 @@ class PushheadCache(CacheManager):
p['changesets'][-1] for p in result['pushes'].values()
]
class TaskCache(CacheManager):
'''Map candidate pushheads to Task Cluster task IDs and artifact URLs.'''
def __init__(self, cache_dir, log=None, skip_cache=False):
CacheManager.__init__(self, cache_dir, 'artifact_url', MAX_CACHED_TASKS, log=log, skip_cache=skip_cache)
- self._index = taskcluster.Index()
- self._queue = taskcluster.Queue()
@cachedmethod(operator.attrgetter('_cache'))
def artifact_urls(self, tree, job, rev, download_symbols):
try:
artifact_job = get_job_details(job, log=self._log, download_symbols=download_symbols)
except KeyError:
self.log(logging.INFO, 'artifact',
{'job': job},
@@ -657,32 +659,31 @@ class TaskCache(CacheManager):
tree=tree,
product=artifact_job.product,
job=job,
)
self.log(logging.DEBUG, 'artifact',
{'namespace': namespace},
'Searching Taskcluster index with namespace: {namespace}')
try:
- task = self._index.findTask(namespace)
+ taskId = find_task_id(namespace)
except Exception:
# Not all revisions correspond to pushes that produce the job we
# care about; and even those that do may not have completed yet.
raise ValueError('Task for {namespace} does not exist (yet)!'.format(namespace=namespace))
- taskId = task['taskId']
- artifacts = self._queue.listLatestArtifacts(taskId)['artifacts']
+ artifacts = list_artifacts(taskId)
urls = []
for artifact_name in artifact_job.find_candidate_artifacts(artifacts):
# We can easily extract the task ID from the URL. We can't easily
# extract the build ID; we use the .ini files embedded in the
# downloaded artifact for this. We could also use the uploaded
# public/build/buildprops.json for this purpose.
- url = self._queue.buildUrl('getLatestArtifact', taskId, artifact_name)
+ url = get_artifact_url(taskId, artifact_name)
urls.append(url)
if not urls:
raise ValueError('Task for {namespace} existed, but no artifacts found!'.format(namespace=namespace))
return urls
def print_last_item(self, args, sorted_kwargs, result):
tree, job, rev = args
self.log(logging.INFO, 'artifact',
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1472,41 +1472,17 @@ class PackageFrontend(MachCommandBase):
Never build libxul again!
'''
pass
def _set_log_level(self, verbose):
self.log_manager.terminal_handler.setLevel(logging.INFO if not verbose else logging.DEBUG)
- def _install_pip_package(self, package):
- if os.environ.get('MOZ_AUTOMATION'):
- self.virtualenv_manager._run_pip([
- 'install',
- package,
- '--no-index',
- '--find-links',
- 'http://pypi.pub.build.mozilla.org/pub',
- '--trusted-host',
- 'pypi.pub.build.mozilla.org',
- ])
- return
- self.virtualenv_manager.install_pip_package(package)
-
def _make_artifacts(self, tree=None, job=None, skip_cache=False):
- # Undo PATH munging that will be done by activating the virtualenv,
- # so that invoked subprocesses expecting to find system python
- # (git cinnabar, in particular), will not find virtualenv python.
- original_path = os.environ.get('PATH', '')
- self._activate_virtualenv()
- os.environ['PATH'] = original_path
-
- for package in ('taskcluster==0.0.32',):
- self._install_pip_package(package)
-
state_dir = self._mach_context.state_dir
cache_dir = os.path.join(state_dir, 'package-frontend')
try:
os.makedirs(cache_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
@@ -1525,17 +1501,16 @@ class PackageFrontend(MachCommandBase):
git = None
if conditions.is_git(build_obj):
if self._is_windows():
git = which.which('git.exe')
else:
git = which.which('git')
- # Absolutely must come after the virtualenv is populated!
from mozbuild.artifacts import Artifacts
artifacts = Artifacts(tree, self.substs, self.defines, job,
log=self.log, cache_dir=cache_dir,
skip_cache=skip_cache, hg=hg, git=git,
topsrcdir=self.topsrcdir)
return artifacts
@ArtifactSubCommand('artifact', 'install',