Bug 1341215 - Remove the dependency on the python taskcluster client module for `mach artifact`. r?chmanchester draft
authorMike Hommey <mh+mozilla@glandium.org>
Tue, 21 Feb 2017 15:32:28 +0900
changeset 488981 7ed6c3295e89b2ef31200b5b93ef636e933dd405
parent 488980 50e6ed1d6d50aad21ca7e60a4e50c1241fa80969
child 546899 d9a944961dc35bb2c218816c8832318777825177
push id46712
push userbmo:mh+mozilla@glandium.org
push dateFri, 24 Feb 2017 04:19:13 +0000
reviewerschmanchester
bugs1341215
milestone54.0a1
Bug 1341215 - Remove the dependency on the python taskcluster client module for `mach artifact`. r?chmanchester This also removes the need for a virtualenv.
python/mozbuild/mozbuild/artifacts.py
python/mozbuild/mozbuild/mach_commands.py
--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -56,17 +56,21 @@ import shutil
 import stat
 import subprocess
 import tarfile
 import tempfile
 import urlparse
 import zipfile
 
 import pylru
-import taskcluster
+from taskgraph.util.taskcluster import (
+    find_task_id,
+    get_artifact_url,
+    list_artifacts,
+)
 
 from mozbuild.action.test_archive import OBJDIR_TEST_FILES
 from mozbuild.util import (
     ensureParentDir,
     FileAvoidWrite,
 )
 import mozinstall
 from mozpack.files import (
@@ -629,18 +633,16 @@ class PushheadCache(CacheManager):
             p['changesets'][-1] for p in result['pushes'].values()
         ]
 
 class TaskCache(CacheManager):
     '''Map candidate pushheads to Task Cluster task IDs and artifact URLs.'''
 
     def __init__(self, cache_dir, log=None, skip_cache=False):
         CacheManager.__init__(self, cache_dir, 'artifact_url', MAX_CACHED_TASKS, log=log, skip_cache=skip_cache)
-        self._index = taskcluster.Index()
-        self._queue = taskcluster.Queue()
 
     @cachedmethod(operator.attrgetter('_cache'))
     def artifact_urls(self, tree, job, rev, download_symbols):
         try:
             artifact_job = get_job_details(job, log=self._log, download_symbols=download_symbols)
         except KeyError:
             self.log(logging.INFO, 'artifact',
                 {'job': job},
@@ -657,32 +659,31 @@ class TaskCache(CacheManager):
             tree=tree,
             product=artifact_job.product,
             job=job,
         )
         self.log(logging.DEBUG, 'artifact',
                  {'namespace': namespace},
                  'Searching Taskcluster index with namespace: {namespace}')
         try:
-            task = self._index.findTask(namespace)
+            taskId = find_task_id(namespace)
         except Exception:
             # Not all revisions correspond to pushes that produce the job we
             # care about; and even those that do may not have completed yet.
             raise ValueError('Task for {namespace} does not exist (yet)!'.format(namespace=namespace))
-        taskId = task['taskId']
 
-        artifacts = self._queue.listLatestArtifacts(taskId)['artifacts']
+        artifacts = list_artifacts(taskId)
 
         urls = []
         for artifact_name in artifact_job.find_candidate_artifacts(artifacts):
             # We can easily extract the task ID from the URL.  We can't easily
             # extract the build ID; we use the .ini files embedded in the
             # downloaded artifact for this.  We could also use the uploaded
             # public/build/buildprops.json for this purpose.
-            url = self._queue.buildUrl('getLatestArtifact', taskId, artifact_name)
+            url = get_artifact_url(taskId, artifact_name)
             urls.append(url)
         if not urls:
             raise ValueError('Task for {namespace} existed, but no artifacts found!'.format(namespace=namespace))
         return urls
 
     def print_last_item(self, args, sorted_kwargs, result):
         tree, job, rev = args
         self.log(logging.INFO, 'artifact',
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1472,41 +1472,17 @@ class PackageFrontend(MachCommandBase):
         Never build libxul again!
 
         '''
         pass
 
     def _set_log_level(self, verbose):
         self.log_manager.terminal_handler.setLevel(logging.INFO if not verbose else logging.DEBUG)
 
-    def _install_pip_package(self, package):
-        if os.environ.get('MOZ_AUTOMATION'):
-            self.virtualenv_manager._run_pip([
-                'install',
-                package,
-                '--no-index',
-                '--find-links',
-                'http://pypi.pub.build.mozilla.org/pub',
-                '--trusted-host',
-                'pypi.pub.build.mozilla.org',
-            ])
-            return
-        self.virtualenv_manager.install_pip_package(package)
-
     def _make_artifacts(self, tree=None, job=None, skip_cache=False):
-        # Undo PATH munging that will be done by activating the virtualenv,
-        # so that invoked subprocesses expecting to find system python
-        # (git cinnabar, in particular), will not find virtualenv python.
-        original_path = os.environ.get('PATH', '')
-        self._activate_virtualenv()
-        os.environ['PATH'] = original_path
-
-        for package in ('taskcluster==0.0.32',):
-            self._install_pip_package(package)
-
         state_dir = self._mach_context.state_dir
         cache_dir = os.path.join(state_dir, 'package-frontend')
 
         try:
             os.makedirs(cache_dir)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
@@ -1525,17 +1501,16 @@ class PackageFrontend(MachCommandBase):
 
         git = None
         if conditions.is_git(build_obj):
             if self._is_windows():
                 git = which.which('git.exe')
             else:
                 git = which.which('git')
 
-        # Absolutely must come after the virtualenv is populated!
         from mozbuild.artifacts import Artifacts
         artifacts = Artifacts(tree, self.substs, self.defines, job,
                               log=self.log, cache_dir=cache_dir,
                               skip_cache=skip_cache, hg=hg, git=git,
                               topsrcdir=self.topsrcdir)
         return artifacts
 
     @ArtifactSubCommand('artifact', 'install',