Bug 1465117 - Take two at updating backfill task. Use modifier this time.
MozReview-Commit-ID: CAs0WRb839r
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -267,53 +267,45 @@ class MachCommands(MachCommandBase):
help='Action input (.yml or .json)')
@CommandArgument('--task', default=None,
help='Task definition (.yml or .json; if omitted, the task will be'
'fetched from the queue)')
@CommandArgument('callback', default=None,
help='Action callback name (Python function name)')
def test_action_callback(self, **options):
import taskgraph.parameters
- from taskgraph.util.taskcluster import get_task_definition
import taskgraph.actions
import yaml
def load_data(filename):
with open(filename) as f:
if filename.endswith('.yml'):
return yaml.safe_load(f)
elif filename.endswith('.json'):
return json.load(f)
else:
raise Exception("unknown filename {}".format(filename))
try:
self.setup_logging()
task_id = options['task_id']
- if options['task']:
- task = load_data(options['task'])
- elif task_id:
- task = get_task_definition(task_id)
- else:
- task = None
if options['input']:
input = load_data(options['input'])
else:
input = None
parameters = taskgraph.parameters.load_parameters_file(options['parameters'])
parameters.check()
root = options['root']
return taskgraph.actions.trigger_action_callback(
task_group_id=options['task_group_id'],
task_id=task_id,
- task=task,
input=input,
callback=options['callback'],
parameters=parameters,
root=root,
test=True)
except Exception:
traceback.print_exc()
sys.exit(1)
--- a/taskcluster/taskgraph/actions/backfill.py
+++ b/taskcluster/taskgraph/actions/backfill.py
@@ -39,27 +39,45 @@ logger = logging.getLogger(__name__)
'depth': {
'type': 'integer',
'default': 5,
'minimum': 1,
'maximum': 10,
'title': 'Depth',
'description': ('The number of previous pushes before the current '
'push to attempt to trigger this task on.')
+ },
+ 'inclusive': {
+ 'type': 'boolean',
+ 'default': False,
+ 'title': 'Inclusive Range',
+ 'description': ('If true, the backfill will also retrigger the task '
+ 'on the selected push.')
+ },
+ 'addGeckoProfile': {
+ 'type': 'boolean',
+ 'default': False,
+ 'title': 'Add Gecko Profile',
+ 'description': 'If true, appends --geckoProfile to mozharness options.'
+ },
+ 'testPath': {
+ 'type': 'string',
+ 'title': 'Test Path',
}
},
'additionalProperties': False
},
available=lambda parameters: parameters.get('project', None) != 'try'
)
def backfill_action(parameters, graph_config, input, task_group_id, task_id, task):
label = task['metadata']['name']
pushes = []
- depth = input.get('depth', 5)
- end_id = int(parameters['pushlog_id']) - 1
+ inclusive_tweak = 1 if input.get('inclusive') else 0
+ depth = input.get('depth', 5) + inclusive_tweak
+ end_id = int(parameters['pushlog_id']) - (1 - inclusive_tweak)
while True:
start_id = max(end_id - depth, 0)
pushlog_url = PUSHLOG_TMPL.format(parameters['head_repository'], start_id, end_id)
r = requests.get(pushlog_url)
r.raise_for_status()
pushes = pushes + r.json()['pushes'].keys()
if len(pushes) >= depth:
@@ -85,13 +103,27 @@ def backfill_action(parameters, graph_co
INDEX_TMPL.format(parameters['project'], push),
'public/parameters.yml')
push_decision_task_id = find_decision_task(push_params, graph_config)
except HTTPError as e:
logger.info('Skipping {} due to missing index artifacts! Error: {}'.format(push, e))
continue
if label in full_task_graph.tasks.keys():
- create_tasks(
- [label], full_task_graph, label_to_taskid,
- push_params, push_decision_task_id, push)
+ def modifier(task):
+ if task.label != label:
+ return task
+ if input.get('addGeckoProfile'):
+ mh = task.task['payload'].setdefault('env', {}) \
+ .get('MOZHARNESS_OPTIONS', '')
+ task.task['payload']['env']['MOZHARNESS_OPTIONS'] = mh + ' --geckoProfile'
+ task.task['extra']['treeherder']['symbol'] += '-p'
+
+ if input.get('testPath'):
+ env = task.task['payload'].setdefault('env', {})
+ env['MOZHARNESS_TEST_PATHS'] = input.get('testPath')
+ task.task['extra']['treeherder']['symbol'] += '-b'
+ return task
+
+ create_tasks([label], full_task_graph, label_to_taskid,
+ push_params, push_decision_task_id, push, modifier=modifier)
else:
logging.info('Could not find {} on {}. Skipping.'.format(label, push))
--- a/taskcluster/taskgraph/actions/util.py
+++ b/taskcluster/taskgraph/actions/util.py
@@ -106,35 +106,40 @@ def create_task_from_def(task_id, task_d
def update_parent(task, graph):
task.task.setdefault('extra', {})['parent'] = os.environ.get('TASK_ID', '')
return task
def create_tasks(to_run, full_task_graph, label_to_taskid,
- params, decision_task_id=None, suffix=''):
+ params, decision_task_id=None, suffix='', modifier=lambda t: t):
"""Create new tasks. The task definition will have {relative-datestamp':
'..'} rendered just like in a decision task. Action callbacks should use
this function to create new tasks,
allowing easy debugging with `mach taskgraph action-callback --test`.
This builds up all required tasks to run in order to run the tasks requested.
+ Optionally this function takes a `modifier` function that is passed in each
+ task before it is put into a new graph. It should return a valid task. Note
+ that this is passed _all_ tasks in the graph, not just the set in to_run. You
+ may want to skip modifying tasks not in your to_run list.
+
If you wish to create the tasks in a new group, leave out decision_task_id."""
if suffix != '':
suffix = '-{}'.format(suffix)
to_run = set(to_run)
# Copy to avoid side-effects later
full_task_graph = copy.deepcopy(full_task_graph)
label_to_taskid = label_to_taskid.copy()
target_graph = full_task_graph.graph.transitive_closure(to_run)
target_task_graph = TaskGraph(
- {l: full_task_graph[l] for l in target_graph.nodes},
+ {l: modifier(full_task_graph[l]) for l in target_graph.nodes},
target_graph)
target_task_graph.for_each_task(update_parent)
optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
params,
to_run,
label_to_taskid)
write_artifact('task-graph{}.json'.format(suffix), optimized_task_graph.to_json())
write_artifact('label-to-taskid{}.json'.format(suffix), label_to_taskid)
--- a/taskcluster/taskgraph/task.py
+++ b/taskcluster/taskgraph/task.py
@@ -16,17 +16,17 @@ class Task(object):
- optimization: optimization to apply to the task (see taskgraph.optimize)
- dependencies: tasks this one depends on, in the form {name: label}, for example
{'build': 'build-linux64/opt', 'docker-image': 'build-docker-image-desktop-test'}
And later, as the task-graph processing proceeds:
- task_id -- TaskCluster taskId under which this task will be created
- This class is just a convenience wraper for the data type and managing
+ This class is just a convenience wrapper for the data type and managing
display, comparison, serialization, etc. It has no functionality of its own.
"""
def __init__(self, kind, label, attributes, task,
optimization=None, dependencies=None):
self.kind = kind
self.label = label
self.attributes = attributes
self.task = task