deleted file mode 100644
--- a/testing/taskcluster/mach_commands.py
+++ /dev/null
@@ -1,633 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import
-
-from collections import defaultdict
-import os
-import json
-import copy
-import re
-import sys
-import time
-from collections import namedtuple
-
-from mach.decorators import (
- CommandArgument,
- CommandProvider,
- Command,
-)
-
-
-ROOT = os.path.dirname(os.path.realpath(__file__))
-GECKO = os.path.realpath(os.path.join(ROOT, '..', '..'))
-
-# XXX: If/when we have the taskcluster queue use construct url instead
-ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
-
-DEFINE_TASK = 'queue:define-task:aws-provisioner-v1/{}'
-
-DEFAULT_TRY = 'try: -b do -p all -u all'
-DEFAULT_JOB_PATH = os.path.join(
- ROOT, 'tasks', 'branches', 'base_jobs.yml'
-)
-
-# time after which a try build's results will expire
-TRY_EXPIRATION = "14 days"
-
-@CommandProvider
-class DecisionTask(object):
- @Command('taskcluster-decision', category="ci",
- description="Build a decision task")
- @CommandArgument('--project',
- required=True,
- help='Treeherder project name')
- @CommandArgument('--url',
- required=True,
- help='Gecko repository to use as head repository.')
- @CommandArgument('--revision',
- required=True,
- help='Revision for this project')
- @CommandArgument('--revision-hash',
- help='Treeherder revision hash')
- @CommandArgument('--comment',
- required=True,
- help='Commit message for this revision')
- @CommandArgument('--owner',
- required=True,
- help='email address of who owns this graph')
- @CommandArgument('task', help="Path to decision task to run.")
- def run_task(self, **params):
- from taskcluster_graph.mach_util import gaia_info
- from taskcluster_graph.slugidjar import SlugidJar
- from taskcluster_graph.from_now import (
- json_time_from_now,
- current_json_time,
- )
- from taskcluster_graph.templates import Templates
-
- templates = Templates(ROOT)
- # Template parameters used when expanding the graph
- parameters = dict(gaia_info().items() + {
- 'source': 'http://todo.com/soon',
- 'project': params['project'],
- 'comment': params['comment'],
- 'url': params['url'],
- 'revision': params['revision'],
- 'revision_hash': params.get('revision_hash', ''),
- 'owner': params['owner'],
- 'as_slugid': SlugidJar(),
- 'from_now': json_time_from_now,
- 'now': current_json_time()
- }.items())
- task = templates.load(params['task'], parameters)
- print(json.dumps(task, indent=4))
-
-@CommandProvider
-class LoadImage(object):
- @Command('taskcluster-load-image', category="ci",
- description="Load a pre-built Docker image")
- @CommandArgument('--task-id',
- help="Load the image at public/image.tar in this task, rather than "
- "searching the index")
- @CommandArgument('image_name', nargs='?',
- help="Load the image of this name based on the current contents of the tree "
- "(as built for mozilla-central or mozilla-inbound)")
- def load_image(self, image_name, task_id):
- from taskcluster_graph.image_builder import (
- task_id_for_image,
- docker_load_from_url
- )
-
- if not image_name and not task_id:
- print("Specify either IMAGE-NAME or TASK-ID")
- sys.exit(1)
-
- if not task_id:
- task_id = task_id_for_image({}, 'mozilla-inbound', image_name, create=False)
- if not task_id:
- print("No task found in the TaskCluster index for {}".format(image_name))
- sys.exit(1)
-
- print("Task ID: {}".format(task_id))
-
- ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
- image_name = docker_load_from_url(ARTIFACT_URL.format(task_id, 'public/image.tar'))
-
- print("Loaded image is named {}".format(image_name))
-
-
-@CommandProvider
-class Graph(object):
- @Command('taskcluster-graph', category="ci",
- description="Create taskcluster task graph")
- @CommandArgument('--base-repository',
- default=os.environ.get('GECKO_BASE_REPOSITORY'),
- help='URL for "base" repository to clone')
- @CommandArgument('--head-repository',
- default=os.environ.get('GECKO_HEAD_REPOSITORY'),
- help='URL for "head" repository to fetch revision from')
- @CommandArgument('--head-ref',
- default=os.environ.get('GECKO_HEAD_REF'),
- help='Reference (this is same as rev usually for hg)')
- @CommandArgument('--head-rev',
- default=os.environ.get('GECKO_HEAD_REV'),
- help='Commit revision to use from head repository')
- @CommandArgument('--message',
- help='Commit message to be parsed. Example: "try: -b do -p all -u all"')
- @CommandArgument('--revision-hash',
- required=False,
- help='Treeherder revision hash to attach results to')
- @CommandArgument('--project',
- required=True,
- help='Project to use for creating task graph. Example: --project=try')
- @CommandArgument('--pushlog-id',
- dest='pushlog_id',
- required=False,
- default=0)
- @CommandArgument('--owner',
- required=True,
- help='email address of who owns this graph')
- @CommandArgument('--level',
- default="1",
- help='SCM level of this repository')
- @CommandArgument('--extend-graph',
- action="store_true", dest="ci", help='Omit create graph arguments')
- @CommandArgument('--interactive',
- required=False,
- default=False,
- action="store_true",
- dest="interactive",
- help="Run the tasks with the interactive feature enabled")
- @CommandArgument('--print-names-only',
- action='store_true', default=False,
- help="Only print the names of each scheduled task, one per line.")
- @CommandArgument('--dry-run',
- action='store_true', default=False,
- help="Stub out taskIds and date fields from the task definitions.")
- @CommandArgument('--ignore-conditions',
- action='store_true',
- help='Run tasks even if their conditions are not met')
- def create_graph(self, **params):
- from functools import partial
-
- from mozpack.path import match as mozpackmatch
-
- from slugid import nice as slugid
-
- from taskcluster_graph.mach_util import (
- merge_dicts,
- gaia_info,
- configure_dependent_task,
- set_interactive_task,
- remove_caches_from_task,
- query_vcs_info
- )
- import taskcluster_graph.transform.routes as routes_transform
- import taskcluster_graph.transform.treeherder as treeherder_transform
- from taskcluster_graph.commit_parser import parse_commit
- from taskcluster_graph.image_builder import (
- docker_image,
- normalize_image_details,
- task_id_for_image
- )
- from taskcluster_graph.from_now import (
- json_time_from_now,
- current_json_time,
- )
- from taskcluster_graph.templates import Templates
- import taskcluster_graph.build_task
-
- if params['dry_run']:
- from taskcluster_graph.dry_run import (
- json_time_from_now,
- current_json_time,
- slugid,
- )
-
- project = params['project']
- message = params.get('message', '') if project == 'try' else DEFAULT_TRY
-
- templates = Templates(ROOT)
-
- job_path = os.path.join(ROOT, 'tasks', 'branches', project, 'job_flags.yml')
- job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH
-
- jobs = templates.load(job_path, {})
-
- job_graph, trigger_tests = parse_commit(message, jobs)
-
- cmdline_interactive = params.get('interactive', False)
-
- # Default to current time if querying the head rev fails
- pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
- vcs_info = query_vcs_info(params['head_repository'], params['head_rev'])
- changed_files = set()
- if vcs_info:
- pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(vcs_info.pushdate))
-
- sys.stderr.write('%d commits influencing task scheduling:\n' %
- len(vcs_info.changesets))
- for c in vcs_info.changesets:
- sys.stderr.write('%s %s\n' % (
- c['node'][0:12], c['desc'].splitlines()[0].encode('ascii', 'ignore')))
-
- changed_files |= set(c['files'])
-
- # Template parameters used when expanding the graph
- seen_images = {}
- parameters = dict(gaia_info().items() + {
- 'index': 'index',
- 'project': project,
- 'pushlog_id': params.get('pushlog_id', 0),
- 'docker_image': docker_image,
- 'task_id_for_image': partial(task_id_for_image, seen_images, project),
- 'base_repository': params['base_repository'] or \
- params['head_repository'],
- 'head_repository': params['head_repository'],
- 'head_ref': params['head_ref'] or params['head_rev'],
- 'head_rev': params['head_rev'],
- 'pushdate': pushdate,
- 'pushtime': pushdate[8:],
- 'year': pushdate[0:4],
- 'month': pushdate[4:6],
- 'day': pushdate[6:8],
- 'owner': params['owner'],
- 'level': params['level'],
- 'from_now': json_time_from_now,
- 'now': current_json_time(),
- 'revision_hash': params['revision_hash']
- }.items())
-
- treeherder_route = '{}.{}'.format(
- params['project'],
- params.get('revision_hash', '')
- )
-
- routes_file = os.path.join(ROOT, 'routes.json')
- with open(routes_file) as f:
- contents = json.load(f)
- json_routes = contents['routes']
- # TODO: Nightly and/or l10n routes
-
- # Task graph we are generating for taskcluster...
- graph = {
- 'tasks': [],
- 'scopes': set(),
- }
-
- if params['revision_hash']:
- for env in routes_transform.TREEHERDER_ROUTES:
- route = 'queue:route:{}.{}'.format(
- routes_transform.TREEHERDER_ROUTES[env],
- treeherder_route)
- graph['scopes'].add(route)
-
- graph['metadata'] = {
- 'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(repo=params['head_repository'], rev=params['head_rev']),
- 'owner': params['owner'],
- # TODO: Add full mach commands to this example?
- 'description': 'Task graph generated via ./mach taskcluster-graph',
- 'name': 'task graph local'
- }
-
- # Filter the job graph according to conditions met by this invocation run.
- def should_run(task):
- # Old style build or test task that doesn't define conditions. Always runs.
- if 'when' not in task:
- return True
-
- # Command line override to not filter.
- if params['ignore_conditions']:
- return True
-
- when = task['when']
-
- # If the task defines file patterns and we have a set of changed
- # files to compare against, only run if a file pattern matches one
- # of the changed files.
- file_patterns = when.get('file_patterns', None)
- if file_patterns and changed_files:
- # Always consider changes to the task definition itself
- file_patterns.append('testing/taskcluster/{task}'.format(task=task['task']))
- for pattern in file_patterns:
- for path in changed_files:
- if mozpackmatch(path, pattern):
- sys.stderr.write('scheduling %s because pattern %s '
- 'matches %s\n' % (task['task'],
- pattern,
- path))
- return True
-
- # No file patterns matched. Discard task.
- sys.stderr.write('discarding %s because no relevant files changed\n' %
- task['task'])
- return False
-
- return True
-
- job_graph = filter(should_run, job_graph)
-
- all_routes = {}
-
- for build in job_graph:
- interactive = cmdline_interactive or build["interactive"]
- build_parameters = merge_dicts(parameters, build['additional-parameters']);
- build_parameters['build_slugid'] = slugid()
- build_parameters['source'] = '{repo}file/{rev}/testing/taskcluster/{file}'.format(repo=params['head_repository'], rev=params['head_rev'], file=build['task'])
- build_task = templates.load(build['task'], build_parameters)
-
- # Copy build_* attributes to expose them to post-build tasks
- # as well as json routes and tests
- task_extra = build_task['task']['extra']
- build_parameters['build_name'] = task_extra['build_name']
- build_parameters['build_type'] = task_extra['build_type']
- build_parameters['build_product'] = task_extra['build_product']
-
- normalize_image_details(graph,
- build_task,
- seen_images,
- build_parameters,
- os.environ.get('TASK_ID', None))
- set_interactive_task(build_task, interactive)
-
- # try builds don't use cache
- if project == "try":
- remove_caches_from_task(build_task)
- set_expiration(build_task, json_time_from_now(TRY_EXPIRATION))
-
- if params['revision_hash']:
- treeherder_transform.add_treeherder_revision_info(build_task['task'],
- params['head_rev'],
- params['revision_hash'])
- routes_transform.decorate_task_treeherder_routes(build_task['task'],
- treeherder_route)
- routes_transform.decorate_task_json_routes(build_task['task'],
- json_routes,
- build_parameters)
-
- # Ensure each build graph is valid after construction.
- taskcluster_graph.build_task.validate(build_task)
- graph['tasks'].append(build_task)
-
- for location in build_task['task']['extra'].get('locations', {}):
- build_parameters['{}_url'.format(location)] = ARTIFACT_URL.format(
- build_parameters['build_slugid'],
- build_task['task']['extra']['locations'][location]
- )
-
- for url in build_task['task']['extra'].get('url', {}):
- build_parameters['{}_url'.format(url)] = \
- build_task['task']['extra']['url'][url]
-
- define_task = DEFINE_TASK.format(build_task['task']['workerType'])
-
- for route in build_task['task'].get('routes', []):
- if route.startswith('index.gecko.v2') and route in all_routes:
- raise Exception("Error: route '%s' is in use by multiple tasks: '%s' and '%s'" % (
- route,
- build_task['task']['metadata']['name'],
- all_routes[route],
- ))
- all_routes[route] = build_task['task']['metadata']['name']
-
- graph['scopes'].add(define_task)
- graph['scopes'] |= set(build_task['task'].get('scopes', []))
- route_scopes = map(lambda route: 'queue:route:' + route, build_task['task'].get('routes', []))
- graph['scopes'] |= set(route_scopes)
-
- # Treeherder symbol configuration for the graph required for each
- # build so tests know which platform they belong to.
- build_treeherder_config = build_task['task']['extra']['treeherder']
-
- if 'machine' not in build_treeherder_config:
- message = '({}), extra.treeherder.machine required for all builds'
- raise ValueError(message.format(build['task']))
-
- if 'build' not in build_treeherder_config:
- build_treeherder_config['build'] = \
- build_treeherder_config['machine']
-
- if 'collection' not in build_treeherder_config:
- build_treeherder_config['collection'] = { 'opt': True }
-
- if len(build_treeherder_config['collection'].keys()) != 1:
- message = '({}), extra.treeherder.collection must contain one type'
- raise ValueError(message.fomrat(build['task']))
-
- for post_build in build['post-build']:
- # copy over the old parameters to update the template
- # TODO additional-parameters is currently not an option, only
- # enabled for build tasks
- post_parameters = merge_dicts(build_parameters,
- post_build.get('additional-parameters', {}))
- post_task = configure_dependent_task(post_build['task'],
- post_parameters,
- slugid(),
- templates,
- build_treeherder_config)
- normalize_image_details(graph,
- post_task,
- seen_images,
- build_parameters,
- os.environ.get('TASK_ID', None))
- set_interactive_task(post_task, interactive)
- treeherder_transform.add_treeherder_revision_info(post_task['task'],
- params['head_rev'],
- params['revision_hash'])
- if project == "try":
- set_expiration(post_task, json_time_from_now(TRY_EXPIRATION))
- graph['tasks'].append(post_task)
-
- for test in build['dependents']:
- test = test['allowed_build_tasks'][build['task']]
- # TODO additional-parameters is currently not an option, only
- # enabled for build tasks
- test_parameters = merge_dicts(build_parameters,
- test.get('additional-parameters', {}))
- test_parameters = copy.copy(build_parameters)
-
- test_definition = templates.load(test['task'], {})['task']
- chunk_config = test_definition['extra'].get('chunks', {})
-
- # Allow branch configs to override task level chunking...
- if 'chunks' in test:
- chunk_config['total'] = test['chunks']
-
- chunked = 'total' in chunk_config
- if chunked:
- test_parameters['total_chunks'] = chunk_config['total']
-
- if 'suite' in test_definition['extra']:
- suite_config = test_definition['extra']['suite']
- test_parameters['suite'] = suite_config['name']
- test_parameters['flavor'] = suite_config.get('flavor', '')
-
- for chunk in range(1, chunk_config.get('total', 1) + 1):
- if 'only_chunks' in test and chunked and \
- chunk not in test['only_chunks']:
- continue
-
- if chunked:
- test_parameters['chunk'] = chunk
- test_task = configure_dependent_task(test['task'],
- test_parameters,
- slugid(),
- templates,
- build_treeherder_config)
- normalize_image_details(graph,
- test_task,
- seen_images,
- build_parameters,
- os.environ.get('TASK_ID', None))
- set_interactive_task(test_task, interactive)
-
- if params['revision_hash']:
- treeherder_transform.add_treeherder_revision_info(test_task['task'],
- params['head_rev'],
- params['revision_hash'])
- routes_transform.decorate_task_treeherder_routes(
- test_task['task'],
- treeherder_route
- )
-
- if project == "try":
- set_expiration(test_task, json_time_from_now(TRY_EXPIRATION))
-
- # This will schedule test jobs N times
- for i in range(0, trigger_tests):
- graph['tasks'].append(test_task)
- # If we're scheduling more tasks each have to be unique
- test_task = copy.deepcopy(test_task)
- test_task['taskId'] = slugid()
-
- define_task = DEFINE_TASK.format(
- test_task['task']['workerType']
- )
-
- graph['scopes'].add(define_task)
- graph['scopes'] |= set(test_task['task'].get('scopes', []))
-
- graph['scopes'] = sorted(graph['scopes'])
-
- if params['print_names_only']:
- tIDs = defaultdict(list)
-
- def print_task(task, indent=0):
- print('{}- {}'.format(' ' * indent, task['task']['metadata']['name']))
-
- for child in tIDs[task['taskId']]:
- print_task(child, indent=indent+2)
-
- # build a dependency map
- for task in graph['tasks']:
- if 'requires' in task:
- for tID in task['requires']:
- tIDs[tID].append(task)
-
- # recursively print root tasks
- for task in graph['tasks']:
- if 'requires' not in task:
- print_task(task)
- return
-
- # When we are extending the graph remove extra fields...
- if params['ci'] is True:
- graph.pop('scopes', None)
- graph.pop('metadata', None)
-
- print(json.dumps(graph, indent=4, sort_keys=True))
-
-@CommandProvider
-class CIBuild(object):
- @Command('taskcluster-build', category='ci',
- description="Create taskcluster try server build task")
- @CommandArgument('--base-repository',
- help='URL for "base" repository to clone')
- @CommandArgument('--head-repository',
- required=True,
- help='URL for "head" repository to fetch revision from')
- @CommandArgument('--head-ref',
- help='Reference (this is same as rev usually for hg)')
- @CommandArgument('--head-rev',
- required=True,
- help='Commit revision to use')
- @CommandArgument('--owner',
- default='foobar@mozilla.com',
- help='email address of who owns this graph')
- @CommandArgument('--level',
- default="1",
- help='SCM level of this repository')
- @CommandArgument('build_task',
- help='path to build task definition')
- @CommandArgument('--interactive',
- required=False,
- default=False,
- action="store_true",
- dest="interactive",
- help="Run the task with the interactive feature enabled")
- def create_ci_build(self, **params):
- from taskcluster_graph.mach_util import (
- gaia_info,
- set_interactive_task,
- query_vcs_info
- )
- from taskcluster_graph.templates import Templates
- from taskcluster_graph.image_builder import docker_image
- import taskcluster_graph.build_task
-
- templates = Templates(ROOT)
- # TODO handle git repos
- head_repository = params['head_repository']
- if not head_repository:
- head_repository = get_hg_url()
-
- head_rev = params['head_rev']
- if not head_rev:
- head_rev = get_latest_hg_revision(head_repository)
-
- head_ref = params['head_ref'] or head_rev
-
- # Default to current time if querying the head rev fails
- pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
- vcs_info = query_vcs_info(params['head_repository'], params['head_rev'])
- if vcs_info:
- pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(vcs_info.pushdate))
-
- from taskcluster_graph.from_now import (
- json_time_from_now,
- current_json_time,
- )
- build_parameters = dict(gaia_info().items() + {
- 'docker_image': docker_image,
- 'owner': params['owner'],
- 'level': params['level'],
- 'from_now': json_time_from_now,
- 'now': current_json_time(),
- 'base_repository': params['base_repository'] or head_repository,
- 'head_repository': head_repository,
- 'head_rev': head_rev,
- 'head_ref': head_ref,
- 'pushdate': pushdate,
- 'pushtime': pushdate[8:],
- 'year': pushdate[0:4],
- 'month': pushdate[4:6],
- 'day': pushdate[6:8],
- }.items())
-
- try:
- build_task = templates.load(params['build_task'], build_parameters)
- set_interactive_task(build_task, params.get('interactive', False))
- except IOError:
- sys.stderr.write(
- "Could not load build task file. Ensure path is a relative " \
- "path from testing/taskcluster"
- )
- sys.exit(1)
-
- taskcluster_graph.build_task.validate(build_task)
-
- print(json.dumps(build_task['task'], indent=4))