bug 1442793 - beetmover artifact_prefix. r?bhearsum draft
authorAki Sasaki <asasaki@mozilla.com>
Wed, 07 Mar 2018 14:27:49 -0800
changeset 782027 d97c0517a40fa9825e84f1c9b0f939eb2999041d
parent 782026 b04655a6a5b965cc731236e38b973222ec801c4d
child 782028 8f7059fb60a633fa85364ba66a5e072d3662b80f
push id106465
push userasasaki@mozilla.com
push dateSat, 14 Apr 2018 00:42:17 +0000
reviewersbhearsum
bugs1442793
milestone61.0a1
bug 1442793 - beetmover artifact_prefix. r?bhearsum MozReview-Commit-ID: 6asTyW9T3OM
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_repackage.py
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -9,16 +9,17 @@ from __future__ import absolute_import, 
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import validate_schema, Schema
 from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
                                          get_beetmover_action_scope,
                                          get_phase,
                                          get_worker_type_for_scope)
+from taskgraph.util.taskcluster import get_artifact_prefix
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
 
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
@@ -388,24 +389,24 @@ def make_task_description(config, jobs):
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
             'shipping-phase': phase,
         }
 
         yield task
 
 
-def generate_upstream_artifacts(signing_task_ref, build_task_ref, platform,
+def generate_upstream_artifacts(job, signing_task_ref, build_task_ref, platform,
                                 locale=None):
     build_mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS
     signing_mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS
 
-    artifact_prefix = 'public/build'
+    artifact_prefix = get_artifact_prefix(job)
     if locale:
-        artifact_prefix = 'public/build/{}'.format(locale)
+        artifact_prefix = '{}/{}'.format(artifact_prefix, locale)
         platform = "{}-l10n".format(platform)
 
     upstream_artifacts = []
 
     # Some platforms (like android-api-16-nightly-l10n) may not depend on any unsigned artifact
     if build_mapping[platform]:
         upstream_artifacts.append({
             "taskId": {"task-reference": build_task_ref},
@@ -492,17 +493,17 @@ def make_task_worker(config, jobs):
 
         signing_task_ref = "<" + str(signing_task) + ">"
         build_task_ref = "<" + str(build_task) + ">"
 
         worker = {
             'implementation': 'beetmover',
             'release-properties': craft_release_properties(config, job),
             'upstream-artifacts': generate_upstream_artifacts(
-                signing_task_ref, build_task_ref, platform, locale
+                job, signing_task_ref, build_task_ref, platform, locale
             )
         }
 
         if locale:
             worker["locale"] = locale
         job["worker"] = worker
 
         yield job
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -13,16 +13,17 @@ from taskgraph.util.attributes import co
 from taskgraph.util.partials import (get_balrog_platform_name,
                                      get_partials_artifacts,
                                      get_partials_artifact_map)
 from taskgraph.util.schema import validate_schema, Schema
 from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
                                          get_beetmover_action_scope,
                                          get_phase,
                                          get_worker_type_for_scope)
+from taskgraph.util.taskcluster import get_artifact_prefix
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
 import logging
 import re
 
 logger = logging.getLogger(__name__)
 
@@ -254,28 +255,28 @@ def make_task_description(config, jobs):
             'treeherder': treeherder,
             'shipping-phase': job.get('shipping-phase', phase),
             'shipping-product': job.get('shipping-product'),
         }
 
         yield task
 
 
-def generate_upstream_artifacts(build_task_ref, build_signing_task_ref,
+def generate_upstream_artifacts(job, build_task_ref, build_signing_task_ref,
                                 repackage_task_ref, repackage_signing_task_ref,
                                 platform, locale=None):
 
     build_mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS
     build_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS
     repackage_mapping = UPSTREAM_ARTIFACT_REPACKAGE_PATHS
     repackage_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS
 
-    artifact_prefix = 'public/build'
+    artifact_prefix = get_artifact_prefix(job)
     if locale:
-        artifact_prefix = 'public/build/{}'.format(locale)
+        artifact_prefix = '{}/{}'.format(artifact_prefix, locale)
         platform = "{}-l10n".format(platform)
 
     upstream_artifacts = []
 
     task_refs = [
         build_task_ref,
         build_signing_task_ref,
         repackage_task_ref,
@@ -303,21 +304,20 @@ def generate_upstream_artifacts(build_ta
                         "paths": ["{}/{}".format(artifact_prefix, path) for path in paths],
                         "locale": locale or "en-US",
                     })
                 plarform_was_previously_matched_by_regex = platform_regex
 
     return upstream_artifacts
 
 
-def generate_partials_upstream_artifacts(artifacts, platform, locale=None):
-    if not locale or locale == 'en-US':
-        artifact_prefix = 'public/build'
-    else:
-        artifact_prefix = 'public/build/{}'.format(locale)
+def generate_partials_upstream_artifacts(job, artifacts, platform, locale=None):
+    artifact_prefix = get_artifact_prefix(job)
+    if locale and locale != 'en-US':
+        artifact_prefix = '{}/{}'.format(artifact_prefix, locale)
 
     upstream_artifacts = [{
         'taskId': {'task-reference': '<partials-signing>'},
         'taskType': 'signing',
         'paths': ["{}/{}".format(artifact_prefix, p)
                   for p in artifacts],
         'locale': locale or 'en-US',
     }]
@@ -381,17 +381,17 @@ def make_task_worker(config, jobs):
         build_signing_task_ref = "<" + str(build_signing_task) + ">"
         repackage_task_ref = "<" + str(repackage_task) + ">"
         repackage_signing_task_ref = "<" + str(repackage_signing_task) + ">"
 
         worker = {
             'implementation': 'beetmover',
             'release-properties': craft_release_properties(config, job),
             'upstream-artifacts': generate_upstream_artifacts(
-                build_task_ref, build_signing_task_ref, repackage_task_ref,
+                job, build_task_ref, build_signing_task_ref, repackage_task_ref,
                 repackage_signing_task_ref, platform, locale
             ),
         }
         if locale:
             worker["locale"] = locale
         job["worker"] = worker
 
         yield job
@@ -423,17 +423,17 @@ def make_partials_artifacts(config, jobs
             else:
                 yield job
                 continue
         else:
             if 'partials-signing' not in job['dependencies']:
                 continue
 
         upstream_artifacts = generate_partials_upstream_artifacts(
-            artifacts, balrog_platform, locale
+            job, artifacts, balrog_platform, locale
         )
 
         job['worker']['upstream-artifacts'].extend(upstream_artifacts)
 
         extra = list()
 
         artifact_map = get_partials_artifact_map(
             config.params.get('release_history'), balrog_platform, locale)