Bug 1436037 - WIP Support generic-worker with run-task draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 02 Aug 2017 14:02:34 -0400
changeset 752596 5ca21325674297dfb9f46df031a7124a34e81679
parent 752595 73f34f5cb7847f581d42bf3c6df78cfe826995e3
push id98309
push userahalberstadt@mozilla.com
push dateThu, 08 Feb 2018 16:33:48 +0000
bugs1436037
milestone60.0a1
Bug 1436037 - WIP Support generic-worker with run-task This attempts to run the mozlint unittests on OSX as a proof of concept. MozReview-Commit-ID: C07FANaYzf7
python/mozlint/test/python.ini
taskcluster/ci/source-test/python.yml
taskcluster/docker/recipes/run-task
taskcluster/taskgraph/transforms/job/common.py
taskcluster/taskgraph/transforms/job/hazard.py
taskcluster/taskgraph/transforms/job/mach.py
taskcluster/taskgraph/transforms/job/mozharness.py
taskcluster/taskgraph/transforms/job/mozharness_test.py
taskcluster/taskgraph/transforms/job/run_task.py
taskcluster/taskgraph/transforms/job/spidermonkey.py
taskcluster/taskgraph/transforms/job/toolchain.py
taskcluster/taskgraph/transforms/task.py
tools/tryselect/tasks.py
--- a/python/mozlint/test/python.ini
+++ b/python/mozlint/test/python.ini
@@ -1,9 +1,9 @@
 [DEFAULT]
-subsuite = mozlint, os == "linux"
+subsuite = mozlint
 
 [test_cli.py]
 [test_filterpaths.py]
 [test_formatters.py]
 [test_parser.py]
 [test_roller.py]
 [test_types.py]
--- a/taskcluster/ci/source-test/python.yml
+++ b/taskcluster/ci/source-test/python.yml
@@ -95,18 +95,32 @@ mozharness:
             cd /builds/worker/checkouts/gecko/testing/mozharness &&
             /usr/local/bin/tox -e py27-hg4.3
     when:
         files-changed:
             - 'testing/mozharness/**'
 
 mozlint:
     description: python/mozlint unit tests
+    platform:
+        - linux64/opt
+        - macosx64/opt
     treeherder:
         symbol: py(ml)
+    worker-type:
+        by-platform:
+            linux64.*: aws-provisioner-v1/gecko-t-linux-xlarge
+            macosx64.*: releng-hardware/gecko-t-osx-1010
+    worker:
+        by-platform:
+            linux64.*:
+                docker-image: {in-tree: "lint"}
+                max-run-time: 3600
+            default:
+                max-run-time: 3600
     run:
         mach: python-test --subsuite mozlint
     when:
         files-changed:
             - 'python/mozlint/**'
 
 mozterm:
     description: python/mozterm unit tests
--- a/taskcluster/docker/recipes/run-task
+++ b/taskcluster/docker/recipes/run-task
@@ -304,41 +304,42 @@ def main(args):
     # effectively undoing the other's work. This would slow down task
     # execution in aggregate. Without monitoring for this, people may not notice
     # the problem and tasks would be slower than they could be. We follow the
     # principle of "fail fast" to ensure optimal task execution.
     #
     # We also write an audit log of who used the caches. This log is printed
     # during failures to help aid debugging.
 
+    our_requirements = {}
     if 'TASKCLUSTER_CACHES' in os.environ:
         caches = os.environ['TASKCLUSTER_CACHES'].split(';')
         del os.environ['TASKCLUSTER_CACHES']
+
+        our_requirements = {
+            # Include a version string that we can bump whenever to trigger
+            # fresh caches. The actual value is not relevant and doesn't need
+            # to follow any explicit order. Since taskgraph bakes this file's
+            # hash into cache names, any change to this file/version is sufficient
+            # to force the use of a new cache.
+            b'version=1',
+            # Include the UID and GID the task will run as to ensure that tasks
+            # with different UID and GID don't share the same cache.
+            b'uid=%d' % uid,
+            b'gid=%d' % gid,
+        }
     else:
         caches = []
 
     if 'TASKCLUSTER_UNTRUSTED_CACHES' in os.environ:
         untrusted_caches = True
         del os.environ['TASKCLUSTER_UNTRUSTED_CACHES']
     else:
         untrusted_caches = False
 
-    our_requirements = {
-        # Include a version string that we can bump whenever to trigger
-        # fresh caches. The actual value is not relevant and doesn't need
-        # to follow any explicit order. Since taskgraph bakes this file's
-        # hash into cache names, any change to this file/version is sufficient
-        # to force the use of a new cache.
-        b'version=1',
-        # Include the UID and GID the task will run as to ensure that tasks
-        # with different UID and GID don't share the same cache.
-        b'uid=%d' % uid,
-        b'gid=%d' % gid,
-    }
-
     def write_audit_entry(path, msg):
         now = datetime.datetime.utcnow().isoformat()
         with open(path, 'ab') as fh:
             fh.write(b'[%sZ %s] %s\n' % (
                      now, os.environ.get('TASK_ID', 'UNKNOWN'), msg))
 
     for cache in caches:
         if not os.path.isdir(cache):
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -67,50 +67,50 @@ def docker_worker_add_gecko_vcs_env_vars
         taskdesc['worker']['env'].update({
             'COMM_BASE_REPOSITORY': config.params['comm_base_repository'],
             'COMM_HEAD_REF': config.params['comm_head_rev'],
             'COMM_HEAD_REPOSITORY': config.params['comm_head_repository'],
             'COMM_HEAD_REV': config.params['comm_head_rev'],
         })
 
 
-def support_vcs_checkout(config, job, taskdesc, sparse=False):
+def support_vcs_checkout(config, job, taskdesc, checkoutdir, sparse=False):
     """Update a job/task with parameters to enable a VCS checkout.
 
     This can only be used with ``run-task`` tasks, as the cache name is
     reserved for ``run-task`` tasks.
     """
     level = config.params['level']
 
-    # native-engine does not support caches (yet), so we just do a full clone
-    # every time :(
+    # native-engine and generic-worker do not support caches (yet), so we just
+    # do a full clone every time :(
     if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
         name = 'level-%s-checkouts' % level
 
         # comm-central checkouts need their own cache, because clobber won't
         # remove the comm-central checkout
         if job['run'].get('comm-checkout', False):
             name += '-comm'
 
         # Sparse checkouts need their own cache because they can interfere
         # with clients that aren't sparse aware.
         if sparse:
             name += '-sparse'
 
         taskdesc['worker'].setdefault('caches', []).append({
             'type': 'persistent',
             'name': name,
-            'mount-point': '/builds/worker/checkouts',
+            'mount-point': checkoutdir,
         })
 
     taskdesc['worker'].setdefault('env', {}).update({
         'GECKO_BASE_REPOSITORY': config.params['base_repository'],
         'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
         'GECKO_HEAD_REV': config.params['head_rev'],
-        'HG_STORE_PATH': '/builds/worker/checkouts/hg-store',
+        'HG_STORE_PATH': '{}/hg-store'.format(checkoutdir),
     })
 
     if 'comm_base_repository' in config.params:
         taskdesc['worker']['env'].update({
             'COMM_BASE_REPOSITORY': config.params['comm_base_repository'],
             'COMM_HEAD_REPOSITORY': config.params['comm_head_repository'],
             'COMM_HEAD_REV': config.params['comm_head_rev'],
         })
--- a/taskcluster/taskgraph/transforms/job/hazard.py
+++ b/taskcluster/taskgraph/transforms/job/hazard.py
@@ -43,30 +43,32 @@ def docker_worker_hazard(config, job, ta
 
     worker = taskdesc['worker']
     worker['artifacts'] = []
 
     docker_worker_add_public_artifacts(config, job, taskdesc)
     docker_worker_add_workspace_cache(config, job, taskdesc)
     docker_worker_add_tooltool(config, job, taskdesc)
     docker_worker_setup_secrets(config, job, taskdesc)
-    support_vcs_checkout(config, job, taskdesc)
+
+    checkoutdir = '/builds/worker/checkouts'
+    support_vcs_checkout(config, job, taskdesc, checkoutdir)
 
     env = worker['env']
     env.update({
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
     })
 
     # script parameters
     if run.get('mozconfig'):
         env['MOZCONFIG'] = run['mozconfig']
 
     # build-haz-linux.sh needs this otherwise it assumes the checkout is in
     # the workspace.
-    env['GECKO_DIR'] = '/builds/worker/checkouts/gecko'
+    env['GECKO_DIR'] = '{}/gecko'.format(checkoutdir)
 
     worker['command'] = [
         '/builds/worker/bin/run-task',
-        '--vcs-checkout', '/builds/worker/checkouts/gecko',
+        '--vcs-checkout', '{}/gecko'.format(checkoutdir),
         '--',
         '/bin/bash', '-c', run['command']
     ]
--- a/taskcluster/taskgraph/transforms/job/mach.py
+++ b/taskcluster/taskgraph/transforms/job/mach.py
@@ -20,16 +20,17 @@ mach_schema = Schema({
     # if true, perform a checkout of a comm-central based branch inside the
     # gecko checkout
     Required('comm-checkout'): bool,
 })
 
 
 @run_job_using("docker-worker", "mach", schema=mach_schema, defaults={'comm-checkout': False})
 @run_job_using("native-engine", "mach", schema=mach_schema, defaults={'comm-checkout': False})
+@run_job_using("generic-worker", "mach", schema=mach_schema, defaults={'comm-checkout': False})
 def docker_worker_mach(config, job, taskdesc):
     run = job['run']
 
     # defer to the run_task implementation
     run['command'] = 'cd /builds/worker/checkouts/gecko && ./mach ' + run['mach']
     run['using'] = 'run-task'
     del run['mach']
     configure_taskdesc_for_run(config, job, taskdesc, job['worker']['implementation'])
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -137,17 +137,17 @@ def mozharness_on_docker_worker_setup(co
     # android-build).
     taskdesc['worker'].setdefault('docker-image', {'in-tree': 'debian7-amd64-build'})
 
     worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
 
     docker_worker_add_public_artifacts(config, job, taskdesc)
     docker_worker_add_workspace_cache(config, job, taskdesc,
                                       extra=run.get('extra-workspace-cache-key'))
-    support_vcs_checkout(config, job, taskdesc)
+    support_vcs_checkout(config, job, taskdesc, checkoutdir='/builds/worker/checkouts')
 
     env = worker.setdefault('env', {})
     env.update({
         'MOZHARNESS_CONFIG': ' '.join(run['config']),
         'MOZHARNESS_SCRIPT': run['script'],
         'MH_BRANCH': config.params['project'],
         'MH_BUILD_POOL': 'taskcluster',
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@@ -160,23 +160,24 @@ def mozharness_test_on_docker(config, jo
 
     # assemble the command line
     command = [
         '/builds/worker/bin/run-task',
     ]
 
     # Support vcs checkouts regardless of whether the task runs from
     # source or not in case it is needed on an interactive loaner.
-    support_vcs_checkout(config, job, taskdesc)
+    checkoutdir = '/builds/worker/checkouts'
+    support_vcs_checkout(config, job, taskdesc, checkoutdir)
 
     # If we have a source checkout, run mozharness from it instead of
     # downloading a zip file with the same content.
     if test['checkout']:
-        command.extend(['--vcs-checkout', '/builds/worker/checkouts/gecko'])
-        env['MOZHARNESS_PATH'] = '/builds/worker/checkouts/gecko/testing/mozharness'
+        command.extend(['--vcs-checkout', '{}/gecko'.format(checkoutdir)])
+        env['MOZHARNESS_PATH'] = '{}/gecko/testing/mozharness'.format(checkoutdir)
     else:
         env['MOZHARNESS_URL'] = {'task-reference': mozharness_url}
 
     command.extend([
         '--',
         '/builds/worker/bin/test-linux.sh',
     ])
 
--- a/taskcluster/taskgraph/transforms/job/run_task.py
+++ b/taskcluster/taskgraph/transforms/job/run_task.py
@@ -32,84 +32,105 @@ run_task_schema = Schema({
 
     # The command arguments to pass to the `run-task` script, after the
     # checkout arguments.  If a list, it will be passed directly; otherwise
     # it will be included in a single argument to `bash -cx`.
     Required('command'): Any([basestring], basestring),
 })
 
 
-def common_setup(config, job, taskdesc):
+def common_setup(config, job, taskdesc, command, checkoutdir='checkouts'):
     run = job['run']
     if run['checkout']:
-        support_vcs_checkout(config, job, taskdesc,
+        support_vcs_checkout(config, job, taskdesc, checkoutdir,
                              sparse=bool(run['sparse-profile']))
 
+        command.append('--vcs-checkout={}/gecko'.format(checkoutdir))
+
+        if run['sparse-profile']:
+            command.append('--sparse-profile=build/sparse-profiles/%s' %
+                           run['sparse-profile'])
+
     taskdesc['worker'].setdefault('env', {})['MOZ_SCM_LEVEL'] = config.params['level']
 
 
-def add_checkout_to_command(run, command):
-    if not run['checkout']:
-        return
-
-    command.append('--vcs-checkout=/builds/worker/checkouts/gecko')
-
-    if run['sparse-profile']:
-        command.append('--sparse-profile=build/sparse-profiles/%s' %
-                       run['sparse-profile'])
+def run_task_url(config):
+    return '{}/raw-file/{}/taskcluster/docker/recipes/run-task'.format(
+                config.params['head_repository'], config.params['head_rev'])
 
 
 docker_defaults = {
     'cache-dotcache': False,
     'checkout': True,
     'comm-checkout': False,
     'sparse-profile': None,
 }
 
 
 @run_job_using("docker-worker", "run-task", schema=run_task_schema, defaults=docker_defaults)
 def docker_worker_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
-    common_setup(config, job, taskdesc)
+    command = ['/builds/worker/bin/run-task']
+    common_setup(config, job, taskdesc, command, checkoutdir='/builds/worker/checkouts')
 
     if run.get('cache-dotcache'):
         worker['caches'].append({
             'type': 'persistent',
             'name': 'level-{level}-{project}-dotcache'.format(**config.params),
             'mount-point': '/builds/worker/.cache',
             'skip-untrusted': True,
         })
 
     run_command = run['command']
     if isinstance(run_command, basestring):
         run_command = ['bash', '-cx', run_command]
-    command = ['/builds/worker/bin/run-task']
-    add_checkout_to_command(run, command)
     if run['comm-checkout']:
         command.append('--comm-checkout=/builds/worker/checkouts/gecko/comm')
     command.append('--fetch-hgfingerprint')
     command.append('--')
     command.extend(run_command)
     worker['command'] = command
 
 
 @run_job_using("native-engine", "run-task", schema=run_task_schema)
 def native_engine_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
-    common_setup(config, job, taskdesc)
+    command = ['./run-task']
+    common_setup(config, job, taskdesc, command, checkoutdir='/builds/worker/checkouts')
 
-    worker['context'] = '{}/raw-file/{}/taskcluster/docker/recipes/run-task'.format(
-        config.params['head_repository'], config.params['head_rev']
-    )
-
+    worker['context'] = run_task_url(config)
     if run.get('cache-dotcache'):
-        raise Exception("No cache support on native-worker; can't use cache-dotcache")
+        raise Exception("No cache support on native-engine; can't use cache-dotcache")
 
     run_command = run['command']
     if isinstance(run_command, basestring):
         run_command = ['bash', '-cx', run_command]
-    command = ['./run-task']
-    add_checkout_to_command(run, command)
     command.append('--')
     command.extend(run_command)
     worker['command'] = command
+
+
+@run_job_using("generic-worker", "run-task", schema=run_task_schema, defaults=docker_defaults)
+def generic_worker_run_task(config, job, taskdesc):
+    run = job['run']
+    worker = taskdesc['worker'] = job['worker']
+    command = ['./run-task']
+    common_setup(config, job, taskdesc, command)
+
+    if run.get('cache-dotcache'):
+        raise Exception("No cache support on generic-worker; can't use cache-dotcache")
+
+    worker.setdefault('mounts', [])
+    worker['mounts'].append({
+        'content': {
+            'url': run_task_url(config),
+        },
+        'file': './run-task',
+    })
+
+    run_command = run['command']
+    if isinstance(run_command, basestring):
+        run_command = ['bash', '-cx', run_command]
+    command.append('--')
+    command.extend(run_command)
+    worker['command'] = [['chmod', '+x', 'run-task'], command]
--- a/taskcluster/taskgraph/transforms/job/spidermonkey.py
+++ b/taskcluster/taskgraph/transforms/job/spidermonkey.py
@@ -53,17 +53,17 @@ def docker_worker_spidermonkey(config, j
     env = worker.setdefault('env', {})
     env.update({
         'MOZHARNESS_DISABLE': 'true',
         'SPIDERMONKEY_VARIANT': run['spidermonkey-variant'],
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
     })
 
-    support_vcs_checkout(config, job, taskdesc)
+    support_vcs_checkout(config, job, taskdesc, checkoutdir='/builds/worker/checkouts')
 
     script = "build-sm.sh"
     if run['using'] == 'spidermonkey-package':
         script = "build-sm-package.sh"
     elif run['using'] == 'spidermonkey-mozjs-crate':
         script = "build-sm-mozjs-crate.sh"
     elif run['using'] == 'spidermonkey-rust-bindings':
         script = "build-sm-rust-bindings.sh"
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -122,17 +122,18 @@ def docker_worker_toolchain(config, job,
 
     # Allow the job to specify where artifacts come from, but add
     # public/build if it's not there already.
     artifacts = worker.setdefault('artifacts', [])
     if not any(artifact.get('name') == 'public/build' for artifact in artifacts):
         docker_worker_add_public_artifacts(config, job, taskdesc)
 
     docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
-    support_vcs_checkout(config, job, taskdesc, sparse=True)
+    support_vcs_checkout(config, job, taskdesc,
+                         checkoutdir='/builds/worker/checkouts', sparse=True)
 
     env = worker['env']
     env.update({
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
         'TOOLS_DISABLE': 'true',
         'MOZ_AUTOMATION': '1',
     })
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -926,17 +926,17 @@ def build_docker_worker_payload(config, 
 
 
 @payload_builder('generic-worker')
 def build_generic_worker_payload(config, task, task_def):
     worker = task['worker']
 
     artifacts = []
 
-    for artifact in worker['artifacts']:
+    for artifact in worker.get('artifacts', []):
         a = {
             'path': artifact['path'],
             'type': artifact['type'],
             'expires': task_def['expires'],  # always expire with the task
         }
         if 'name' in artifact:
             a['name'] = artifact['name']
         artifacts.append(a)
--- a/tools/tryselect/tasks.py
+++ b/tools/tryselect/tasks.py
@@ -64,17 +64,17 @@ def generate_tasks(params, full, root):
     print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
     try:
         params = load_parameters_file(params, strict=False)
         params.check()
     except ParameterMismatch as e:
         print(PARAMETER_MISMATCH.format(e.args[0]))
         sys.exit(1)
 
-    taskgraph.fast = True
+    taskgraph.fast = False
     cwd = os.getcwd()
     os.chdir(build.topsrcdir)
 
     root = os.path.join(root, 'taskcluster', 'ci')
     tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
     labels = [label for label in tg.graph.visit_postorder()]
 
     os.chdir(cwd)