bug 1442793 - generate_taskcluster_artifact_prefix. r?bhearsum draft
authorAki Sasaki <asasaki@mozilla.com>
Wed, 21 Mar 2018 12:07:01 -0700
changeset 782031 030319c35cbebb91fa22059f36e191dff3314238
parent 782030 66a6070b2e4dd5fc56797d344ab4d6aee46d663f
child 782032 fcc8805ac6fd505a04a73960306c2eecd1b4bd86
push id106465
push userasasaki@mozilla.com
push dateSat, 14 Apr 2018 00:42:17 +0000
reviewersbhearsum
bugs1442793
milestone61.0a1
bug 1442793 - generate_taskcluster_artifact_prefix. r?bhearsum MozReview-Commit-ID: 2zRqm5C9ZZQ
taskcluster/taskgraph/transforms/final_verify.py
taskcluster/taskgraph/transforms/partials.py
taskcluster/taskgraph/transforms/repackage.py
taskcluster/taskgraph/transforms/update_verify.py
taskcluster/taskgraph/util/taskcluster.py
--- a/taskcluster/taskgraph/transforms/final_verify.py
+++ b/taskcluster/taskgraph/transforms/final_verify.py
@@ -20,17 +20,17 @@ def add_command(config, tasks):
         if not task["worker"].get("env"):
             task["worker"]["env"] = {}
 
         final_verify_configs = []
         for upstream in task.get("dependencies", {}).keys():
             if 'update-verify-config' in upstream:
                 final_verify_configs.append(
                     "{}update-verify.cfg".format(
-                        get_taskcluster_artifact_prefix("<{}>".format(upstream))
+                        get_taskcluster_artifact_prefix(task, "<{}>".format(upstream))
                     )
                 )
         task["worker"]["command"] = [
             "/bin/bash",
             "-c",
             {
                 "task-reference": "hg clone $BUILD_TOOLS_REPO tools && cd tools/release && " +
                                   "./final-verification.sh " +
--- a/taskcluster/taskgraph/transforms/partials.py
+++ b/taskcluster/taskgraph/transforms/partials.py
@@ -83,17 +83,17 @@ def make_task_description(config, jobs):
             if 'repackage-signing' in dependency:
                 signing_task = dependency
                 break
         signing_task_ref = '<{}>'.format(signing_task)
 
         extra = {'funsize': {'partials': list()}}
         update_number = 1
         artifact_path = "{}{}".format(
-            get_taskcluster_artifact_prefix(signing_task_ref, locale=locale),
+            get_taskcluster_artifact_prefix(dep_job, signing_task_ref, locale=locale),
             'target.complete.mar'
         )
         for build in sorted(builds):
             partial_info = {
                 'locale': build_locale,
                 'from_mar': builds[build]['mar_url'],
                 'to_mar': {'task-reference': artifact_path},
                 'platform': get_balrog_platform_name(dep_th_platform),
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -179,17 +179,17 @@ def make_job_description(config, jobs):
             'using': 'mozharness',
             'script': 'mozharness/scripts/repackage.py',
             'job-script': 'taskcluster/scripts/builder/repackage.sh',
             'actions': ['download_input', 'setup', 'repackage'],
             'extra-workspace-cache-key': 'repackage',
         })
 
         worker = {
-            'env': _generate_task_env(build_platform, build_task_ref,
+            'env': _generate_task_env(dep_job, build_platform, build_task_ref,
                                       signing_task_ref, locale=locale),
             'artifacts': _generate_task_output_files(build_platform, locale=locale),
             'chain-of-trust': True,
             'max-run-time': 7200 if build_platform.startswith('win') else 3600,
         }
 
         if locale:
             # Make sure we specify the locale-specific upload dir
@@ -237,19 +237,21 @@ def make_job_description(config, jobs):
         if build_platform.startswith('macosx'):
             task['toolchains'] = [
                 'linux64-libdmg',
                 'linux64-hfsplus',
             ]
         yield task
 
 
-def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=None):
-    mar_prefix = get_taskcluster_artifact_prefix(build_task_ref, postfix='host/bin/', locale=None)
-    signed_prefix = get_taskcluster_artifact_prefix(signing_task_ref, locale=locale)
+def _generate_task_env(task, build_platform, build_task_ref, signing_task_ref, locale=None):
+    mar_prefix = get_taskcluster_artifact_prefix(
+        task, build_task_ref, postfix='host/bin/', locale=None
+    )
+    signed_prefix = get_taskcluster_artifact_prefix(task, signing_task_ref, locale=locale)
 
     if build_platform.startswith('linux') or build_platform.startswith('macosx'):
         tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz'
         return {
             'SIGNED_INPUT': {'task-reference': '{}target.tar.{}'.format(
                 signed_prefix, tarball_extension
             )},
             'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)},
--- a/taskcluster/taskgraph/transforms/update_verify.py
+++ b/taskcluster/taskgraph/transforms/update_verify.py
@@ -42,17 +42,17 @@ def add_command(config, tasks):
             for thing in ("CHANNEL", "VERIFY_CONFIG", "BUILD_TOOLS_REPO"):
                 thing = "worker.env.{}".format(thing)
                 resolve_keyed_by(chunked, thing, thing, **config.params)
 
             update_verify_config = None
             for upstream in chunked.get("dependencies", {}).keys():
                 if 'update-verify-config' in upstream:
                     update_verify_config = "{}update-verify.cfg".format(
-                        get_taskcluster_artifact_prefix("<{}>".format(upstream))
+                        get_taskcluster_artifact_prefix(task, "<{}>".format(upstream))
                     )
             if not update_verify_config:
                 raise Exception("Couldn't find upate verify config")
 
             chunked["worker"]["env"]["TASKCLUSTER_VERIFY_CONFIG"] = {
                 "task-reference": update_verify_config
             }
 
--- a/taskcluster/taskgraph/util/taskcluster.py
+++ b/taskcluster/taskgraph/util/taskcluster.py
@@ -12,17 +12,17 @@ import yaml
 import requests
 import logging
 from mozbuild.util import memoize
 from requests.packages.urllib3.util.retry import Retry
 from requests.adapters import HTTPAdapter
 from taskgraph.task import Task
 
 _TC_ARTIFACT_LOCATION = \
-        'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/public/build/{postfix}'
+        'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/{artifact_prefix}/{postfix}'
 
 logger = logging.getLogger(__name__)
 
 # this is set to true for `mach taskgraph action-callback --test`
 testing = False
 
 
 @memoize
@@ -204,13 +204,17 @@ def purge_cache(provisioner_id, worker_t
     if testing:
         logger.info('Would have purged {}/{}/{}.'.format(provisioner_id, worker_type, cache_name))
     else:
         logger.info('Purging {}/{}/{}.'.format(provisioner_id, worker_type, cache_name))
         purge_cache_url = get_purge_cache_url(provisioner_id, worker_type, use_proxy)
         _do_request(purge_cache_url, json={'cacheName': cache_name})
 
 
-def get_taskcluster_artifact_prefix(task_id, postfix='', locale=None):
+def get_taskcluster_artifact_prefix(task, task_id, postfix='', locale=None):
     if locale:
         postfix = '{}/{}'.format(locale, postfix)
 
-    return _TC_ARTIFACT_LOCATION.format(task_id=task_id, postfix=postfix)
+    artifact_prefix = get_artifact_prefix(task)
+
+    return _TC_ARTIFACT_LOCATION.format(
+        task_id=task_id, postfix=postfix, artifact_prefix=artifact_prefix
+    )