--- a/taskcluster/ci/source-test/jsshell.yml
+++ b/taskcluster/ci/source-test/jsshell.yml
@@ -23,24 +23,24 @@ job-defaults:
 
 bench-ares6:
     description: Ares6 JavaScript shell benchmark suite
     treeherder:
         symbol: js-bench(ares6)
     run:
         command: >
             cd $USE_ARTIFACT_PATH/build &&
-            unzip -q -d jsshell target.jsshell.zip &&
+            unzip -qo -d jsshell target.jsshell.zip &&
             export JSSHELL=$USE_ARTIFACT_PATH/build/jsshell/js &&
             cd $GECKO_PATH &&
             ./mach jsshell-bench --binary $JSSHELL --perfherder ares6
 
 bench-sixspeed:
     description: Six-Speed JavaScript shell benchmark suite
     treeherder:
         symbol: js-bench(6speed)
     run:
         command: >
             cd $USE_ARTIFACT_PATH/build &&
-            unzip -q -d jsshell target.jsshell.zip &&
+            unzip -qo -d jsshell target.jsshell.zip &&
             export JSSHELL=$USE_ARTIFACT_PATH/build/jsshell/js &&
             cd $GECKO_PATH &&
             ./mach jsshell-bench --binary $JSSHELL --perfherder six-speed

--- a/taskcluster/scripts/run-task
+++ b/taskcluster/scripts/run-task
@@ -23,16 +23,17 @@ if sys.version_info[0:2] < (3, 5):
 
 import argparse
 import datetime
 import errno
 import io
 import json
 import os
 import re
+import shutil
 import socket
 import stat
 import subprocess
 
 import urllib.error
 import urllib.request
 
 
@@ -702,19 +703,20 @@ def main(args):
             revision=os.environ.get('COMM_HEAD_REV'),
             branch=os.environ.get('COMM_HEAD_REF'))
 
     elif not os.environ.get('COMM_HEAD_REV') and \
             os.environ.get('COMM_HEAD_REF'):
         print('task should be defined in terms of non-symbolic revision')
         return 1
 
+    use_artifact_path = os.environ.get('USE_ARTIFACT_PATH')
+
     def prepare_use_artifact(key, url):
         print_line(b'setup', b'fetching artifact from %s\n' % url.encode('utf-8'))
-        use_artifact_path = os.environ['USE_ARTIFACT_PATH']
         path = os.path.join(use_artifact_path, key)
         if not os.path.isdir(path):
             os.makedirs(path)
 
         url = url.rstrip('/')
         path = os.path.join(path, os.path.basename(url))
         response = urllib.request.urlopen(url)
         with open(path, 'wb') as fh:
@@ -722,13 +724,17 @@ def main(args):
 
     use_artifacts = os.environ.get('USE_ARTIFACT_URLS')
     if use_artifacts:
         use_artifacts = json.loads(use_artifacts)
         for key, urls in use_artifacts.items():
             for url in urls:
                 prepare_use_artifact(key, url)
 
-    return run_and_prefix_output(b'task', task_args)
+    try:
+        return run_and_prefix_output(b'task', task_args)
+    finally:
+        if use_artifact_path and os.path.isdir(use_artifact_path):
+            shutil.rmtree(use_artifact_path)
 
 
 if __name__ == '__main__':
     sys.exit(main(sys.argv[1:]))