bug 1442793 - job public_artifacts -> artifacts. r?bhearsum draft
authorAki Sasaki <asasaki@mozilla.com>
Wed, 07 Mar 2018 14:07:58 -0800
changeset 782024 010b14e8b044e4e91f90f5815264435a4d3a723e
parent 782023 dd326fd78fed15a44863dd3483dd7600bfab1102
child 782025 b6e11b8bcda2054dab119726ef0b138b2ce499ba
push id106465
push userasasaki@mozilla.com
push dateSat, 14 Apr 2018 00:42:17 +0000
reviewersbhearsum
bugs1442793
milestone61.0a1
bug 1442793 - job public_artifacts -> artifacts. r?bhearsum MozReview-Commit-ID: F1byXqn5Yan
taskcluster/taskgraph/transforms/job/common.py
taskcluster/taskgraph/transforms/job/debian_package.py
taskcluster/taskgraph/transforms/job/hazard.py
taskcluster/taskgraph/transforms/job/mozharness.py
taskcluster/taskgraph/transforms/job/spidermonkey.py
taskcluster/taskgraph/transforms/job/toolchain.py
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Common support for various job types.  These functions are all named after the
 worker implementation they operate on, and take the same three parameters, for
 consistency.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.util.taskcluster import get_artifact_prefix
 
 SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
 
 
 def docker_worker_add_workspace_cache(config, job, taskdesc, extra=None):
     """Add the workspace cache.
 
     ``extra`` is an optional kwarg passed in that supports extending the cache
@@ -30,32 +31,35 @@ def docker_worker_add_workspace_cache(co
         'skip-untrusted': True,
     })
     if extra:
         taskdesc['worker']['caches'][-1]['name'] += '-{}'.format(
             extra
         )
 
 
-def add_public_artifacts(config, job, taskdesc, path):
+def add_artifacts(config, job, taskdesc, path):
     taskdesc['worker'].setdefault('artifacts', []).append({
-        'name': 'public/build',
+        'name': get_artifact_prefix(taskdesc),
         'path': path,
         'type': 'directory',
     })
 
 
-def docker_worker_add_public_artifacts(config, job, taskdesc):
-    """ Adds a public artifact directory to the task """
-    add_public_artifacts(config, job, taskdesc, path='/builds/worker/artifacts/')
+def docker_worker_add_artifacts(config, job, taskdesc):
+    """ Adds an artifact directory to the task """
+    add_artifacts(config, job, taskdesc, path='/builds/worker/artifacts/')
 
 
-def generic_worker_add_public_artifacts(config, job, taskdesc):
-    """ Adds a public artifact directory to the task """
-    add_public_artifacts(config, job, taskdesc, path=r'public/build')
+def generic_worker_add_artifacts(config, job, taskdesc):
+    """ Adds an artifact directory to the task """
+    # This ``public/build`` is the location on disk; it doesn't necessarily
+    # mean the artifacts will be public; that is set via the ``artifact_prefix``
+    # attribute.
+    add_artifacts(config, job, taskdesc, path=r'public/build')
 
 
 def docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc):
     """Add the GECKO_BASE_* and GECKO_HEAD_* env vars to the worker."""
     env = taskdesc['worker'].setdefault('env', {})
     env.update({
         'GECKO_BASE_REPOSITORY': config.params['base_repository'],
         'GECKO_HEAD_REF': config.params['head_rev'],
--- a/taskcluster/taskgraph/transforms/job/debian_package.py
+++ b/taskcluster/taskgraph/transforms/job/debian_package.py
@@ -9,17 +9,17 @@ from __future__ import absolute_import, 
 
 import os
 import re
 
 from taskgraph.util.schema import Schema
 from voluptuous import Any, Optional, Required
 
 from taskgraph.transforms.job import run_job_using
-from taskgraph.transforms.job.common import add_public_artifacts
+from taskgraph.transforms.job.common import add_artifacts
 
 from taskgraph.util.hash import hash_path
 from taskgraph import GECKO
 from taskgraph.util.cached_tasks import add_optimization
 
 DSC_PACKAGE_RE = re.compile('.*(?=_)')
 SOURCE_PACKAGE_RE = re.compile('.*(?=[-_]\d)')
 
@@ -66,17 +66,17 @@ def docker_worker_debian_package(config,
     name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
 
     worker = taskdesc['worker']
     worker['artifacts'] = []
     worker['docker-image'] = 'debian:{dist}-{date}'.format(
         dist=run['dist'],
         date=run['snapshot'][:8])
 
-    add_public_artifacts(config, job, taskdesc, path='/tmp/artifacts')
+    add_artifacts(config, job, taskdesc, path='/tmp/artifacts')
 
     env = worker.setdefault('env', {})
     env['DEBFULLNAME'] = 'Mozilla build team'
     env['DEBEMAIL'] = 'dev-builds@lists.mozilla.org'
 
     if 'dsc' in run:
         src = run['dsc']
         unpack = 'dpkg-source -x {src_file} {package}'
--- a/taskcluster/taskgraph/transforms/job/hazard.py
+++ b/taskcluster/taskgraph/transforms/job/hazard.py
@@ -9,17 +9,17 @@ from __future__ import absolute_import, 
 
 from taskgraph.util.schema import Schema
 from voluptuous import Required, Optional, Any
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
     docker_worker_add_workspace_cache,
     docker_worker_setup_secrets,
-    docker_worker_add_public_artifacts,
+    docker_worker_add_artifacts,
     docker_worker_add_tooltool,
     support_vcs_checkout,
 )
 
 haz_run_schema = Schema({
     Required('using'): 'hazard',
 
     # The command to run within the task image (passed through to the worker)
@@ -39,17 +39,17 @@ haz_run_schema = Schema({
 
 @run_job_using("docker-worker", "hazard", schema=haz_run_schema)
 def docker_worker_hazard(config, job, taskdesc):
     run = job['run']
 
     worker = taskdesc['worker']
     worker['artifacts'] = []
 
-    docker_worker_add_public_artifacts(config, job, taskdesc)
+    docker_worker_add_artifacts(config, job, taskdesc)
     docker_worker_add_workspace_cache(config, job, taskdesc)
     docker_worker_add_tooltool(config, job, taskdesc)
     docker_worker_setup_secrets(config, job, taskdesc)
     support_vcs_checkout(config, job, taskdesc)
 
     env = worker['env']
     env.update({
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -16,19 +16,19 @@ from textwrap import dedent
 from taskgraph.util.schema import Schema
 from voluptuous import Required, Optional, Any
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
     docker_worker_add_workspace_cache,
     docker_worker_add_gecko_vcs_env_vars,
     docker_worker_setup_secrets,
-    docker_worker_add_public_artifacts,
+    docker_worker_add_artifacts,
     docker_worker_add_tooltool,
-    generic_worker_add_public_artifacts,
+    generic_worker_add_artifacts,
     support_vcs_checkout,
 )
 
 mozharness_run_schema = Schema({
     Required('using'): 'mozharness',
 
     # the mozharness script used to run this task, relative to the testing/
     # directory and using forward slashes even on Windows
@@ -134,17 +134,17 @@ def mozharness_on_docker_worker_setup(co
 
     # Running via mozharness assumes an image that contains build.sh:
     # by default, debian7-amd64-build, but it could be another image (like
     # android-build).
     taskdesc['worker'].setdefault('docker-image', {'in-tree': 'debian7-amd64-build'})
 
     worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
 
-    docker_worker_add_public_artifacts(config, job, taskdesc)
+    docker_worker_add_artifacts(config, job, taskdesc)
     docker_worker_add_workspace_cache(config, job, taskdesc,
                                       extra=run.get('extra-workspace-cache-key'))
     support_vcs_checkout(config, job, taskdesc)
 
     env = worker.setdefault('env', {})
     env.update({
         'MOZHARNESS_CONFIG': ' '.join(run['config']),
         'MOZHARNESS_SCRIPT': run['script'],
@@ -231,17 +231,17 @@ def mozharness_on_generic_worker(config,
     if not run.get('keep-artifacts', True):
         invalid.append('keep-artifacts')
     if invalid:
         raise Exception("Jobs run using mozharness on Windows do not support properties " +
                         ', '.join(invalid))
 
     worker = taskdesc['worker']
 
-    generic_worker_add_public_artifacts(config, job, taskdesc)
+    generic_worker_add_artifacts(config, job, taskdesc)
 
     docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
 
     env = worker['env']
     env.update({
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
         'MOZ_AUTOMATION': '1',
--- a/taskcluster/taskgraph/transforms/job/spidermonkey.py
+++ b/taskcluster/taskgraph/transforms/job/spidermonkey.py
@@ -7,18 +7,18 @@ Support for running spidermonkey jobs vi
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.util.schema import Schema
 from voluptuous import Required, Any
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
-    docker_worker_add_public_artifacts,
-    generic_worker_add_public_artifacts,
+    docker_worker_add_artifacts,
+    generic_worker_add_artifacts,
     docker_worker_add_gecko_vcs_env_vars,
     docker_worker_add_tooltool,
     support_vcs_checkout,
 )
 
 sm_run_schema = Schema({
     Required('using'): Any('spidermonkey', 'spidermonkey-package', 'spidermonkey-mozjs-crate',
                            'spidermonkey-rust-bindings'),
@@ -42,17 +42,17 @@ def docker_worker_spidermonkey(config, j
     worker.setdefault('caches', []).append({
         'type': 'persistent',
         'name': 'level-{}-{}-build-spidermonkey-workspace'.format(
             config.params['level'], config.params['project']),
         'mount-point': "/builds/worker/workspace",
         'skip-untrusted': True,
     })
 
-    docker_worker_add_public_artifacts(config, job, taskdesc)
+    docker_worker_add_artifacts(config, job, taskdesc)
     docker_worker_add_tooltool(config, job, taskdesc)
 
     env = worker.setdefault('env', {})
     env.update({
         'MOZHARNESS_DISABLE': 'true',
         'SPIDERMONKEY_VARIANT': run['spidermonkey-variant'],
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
@@ -81,17 +81,17 @@ def docker_worker_spidermonkey(config, j
 @run_job_using("generic-worker", "spidermonkey", schema=sm_run_schema)
 def generic_worker_spidermonkey(config, job, taskdesc):
     assert job['worker']['os'] == 'windows', 'only supports windows right now'
 
     run = job['run']
 
     worker = taskdesc['worker']
 
-    generic_worker_add_public_artifacts(config, job, taskdesc)
+    generic_worker_add_artifacts(config, job, taskdesc)
     docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
 
     env = worker.setdefault('env', {})
     env.update({
         'MOZHARNESS_DISABLE': 'true',
         'SPIDERMONKEY_VARIANT': run['spidermonkey-variant'],
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -10,17 +10,17 @@ from __future__ import absolute_import, 
 from mozbuild.shellutil import quote as shell_quote
 
 from taskgraph.util.schema import Schema
 from voluptuous import Optional, Required, Any
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
     docker_worker_add_gecko_vcs_env_vars,
-    docker_worker_add_public_artifacts,
+    docker_worker_add_artifacts,
     docker_worker_add_tooltool,
     support_vcs_checkout,
 )
 from taskgraph.util.hash import hash_paths
 from taskgraph import GECKO
 from taskgraph.util.cached_tasks import add_optimization
 import taskgraph
 
@@ -119,17 +119,17 @@ def docker_worker_toolchain(config, job,
 
     worker = taskdesc['worker']
     worker['chain-of-trust'] = True
 
     # Allow the job to specify where artifacts come from, but add
     # public/build if it's not there already.
     artifacts = worker.setdefault('artifacts', [])
     if not any(artifact.get('name') == 'public/build' for artifact in artifacts):
-        docker_worker_add_public_artifacts(config, job, taskdesc)
+        docker_worker_add_artifacts(config, job, taskdesc)
 
     docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
     support_vcs_checkout(config, job, taskdesc, sparse=True)
 
     env = worker['env']
     env.update({
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],