Bug 1465117 - Take two at updating backfill task. Use modifier this time. draft
authorBrian Stack <bstack@mozilla.com>
Fri, 08 Jun 2018 12:08:34 -0700
changeset 808792 d8dd71a9e84b8fc63293ad4134153ffa76d620be
parent 805786 e0595117ff5bda3a63a72ad7b3b8754fec4fb4f0
push id113492
push userbstack@mozilla.com
push dateWed, 20 Jun 2018 18:59:01 +0000
bugs1465117
milestone62.0a1
Bug 1465117 - Take two at updating backfill task. Use modifier this time. MozReview-Commit-ID: CAs0WRb839r
taskcluster/mach_commands.py
taskcluster/taskgraph/actions/backfill.py
taskcluster/taskgraph/actions/util.py
taskcluster/taskgraph/task.py
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -267,53 +267,45 @@ class MachCommands(MachCommandBase):
                      help='Action input (.yml or .json)')
     @CommandArgument('--task', default=None,
                      help='Task definition (.yml or .json; if omitted, the task will be'
                           'fetched from the queue)')
     @CommandArgument('callback', default=None,
                      help='Action callback name (Python function name)')
     def test_action_callback(self, **options):
         import taskgraph.parameters
-        from taskgraph.util.taskcluster import get_task_definition
         import taskgraph.actions
         import yaml
 
         def load_data(filename):
             with open(filename) as f:
                 if filename.endswith('.yml'):
                     return yaml.safe_load(f)
                 elif filename.endswith('.json'):
                     return json.load(f)
                 else:
                     raise Exception("unknown filename {}".format(filename))
 
         try:
             self.setup_logging()
             task_id = options['task_id']
-            if options['task']:
-                task = load_data(options['task'])
-            elif task_id:
-                task = get_task_definition(task_id)
-            else:
-                task = None
 
             if options['input']:
                 input = load_data(options['input'])
             else:
                 input = None
 
             parameters = taskgraph.parameters.load_parameters_file(options['parameters'])
             parameters.check()
 
             root = options['root']
 
             return taskgraph.actions.trigger_action_callback(
                     task_group_id=options['task_group_id'],
                     task_id=task_id,
-                    task=task,
                     input=input,
                     callback=options['callback'],
                     parameters=parameters,
                     root=root,
                     test=True)
         except Exception:
             traceback.print_exc()
             sys.exit(1)
--- a/taskcluster/taskgraph/actions/backfill.py
+++ b/taskcluster/taskgraph/actions/backfill.py
@@ -39,27 +39,45 @@ logger = logging.getLogger(__name__)
             'depth': {
                 'type': 'integer',
                 'default': 5,
                 'minimum': 1,
                 'maximum': 10,
                 'title': 'Depth',
                 'description': ('The number of previous pushes before the current '
                                 'push to attempt to trigger this task on.')
+            },
+            'inclusive': {
+                'type': 'boolean',
+                'default': False,
+                'title': 'Inclusive Range',
+                'description': ('If true, the backfill will also retrigger the task '
+                                'on the selected push.')
+            },
+            'addGeckoProfile': {
+                'type': 'boolean',
+                'default': False,
+                'title': 'Add Gecko Profile',
+                'description': 'If true, appends --geckoProfile to mozharness options.'
+            },
+            'testPath': {
+                'type': 'string',
+                'title': 'Test Path',
             }
         },
         'additionalProperties': False
     },
     available=lambda parameters: parameters.get('project', None) != 'try'
 )
 def backfill_action(parameters, graph_config, input, task_group_id, task_id, task):
     label = task['metadata']['name']
     pushes = []
-    depth = input.get('depth', 5)
-    end_id = int(parameters['pushlog_id']) - 1
+    inclusive_tweak = 1 if input.get('inclusive') else 0
+    depth = input.get('depth', 5) + inclusive_tweak
+    end_id = int(parameters['pushlog_id']) - (1 - inclusive_tweak)
 
     while True:
         start_id = max(end_id - depth, 0)
         pushlog_url = PUSHLOG_TMPL.format(parameters['head_repository'], start_id, end_id)
         r = requests.get(pushlog_url)
         r.raise_for_status()
         pushes = pushes + r.json()['pushes'].keys()
         if len(pushes) >= depth:
@@ -85,13 +103,27 @@ def backfill_action(parameters, graph_co
                     INDEX_TMPL.format(parameters['project'], push),
                     'public/parameters.yml')
             push_decision_task_id = find_decision_task(push_params, graph_config)
         except HTTPError as e:
             logger.info('Skipping {} due to missing index artifacts! Error: {}'.format(push, e))
             continue
 
         if label in full_task_graph.tasks.keys():
-            create_tasks(
-                    [label], full_task_graph, label_to_taskid,
-                    push_params, push_decision_task_id, push)
+            def modifier(task):
+                if task.label != label:
+                    return task
+                if input.get('addGeckoProfile'):
+                    mh = task.task['payload'].setdefault('env', {}) \
+                                                    .get('MOZHARNESS_OPTIONS', '')
+                    task.task['payload']['env']['MOZHARNESS_OPTIONS'] = mh + ' --geckoProfile'
+                    task.task['extra']['treeherder']['symbol'] += '-p'
+
+                if input.get('testPath'):
+                    env = task.task['payload'].setdefault('env', {})
+                    env['MOZHARNESS_TEST_PATHS'] = input.get('testPath')
+                    task.task['extra']['treeherder']['symbol'] += '-b'
+                return task
+
+            create_tasks([label], full_task_graph, label_to_taskid,
+                         push_params, push_decision_task_id, push, modifier=modifier)
         else:
             logging.info('Could not find {} on {}. Skipping.'.format(label, push))
--- a/taskcluster/taskgraph/actions/util.py
+++ b/taskcluster/taskgraph/actions/util.py
@@ -106,35 +106,40 @@ def create_task_from_def(task_id, task_d
 
 
 def update_parent(task, graph):
     task.task.setdefault('extra', {})['parent'] = os.environ.get('TASK_ID', '')
     return task
 
 
 def create_tasks(to_run, full_task_graph, label_to_taskid,
-                 params, decision_task_id=None, suffix=''):
+                 params, decision_task_id=None, suffix='', modifier=lambda t: t):
     """Create new tasks.  The task definition will have {relative-datestamp':
     '..'} rendered just like in a decision task.  Action callbacks should use
     this function to create new tasks,
     allowing easy debugging with `mach taskgraph action-callback --test`.
     This builds up all required tasks to run in order to run the tasks requested.
 
+    Optionally this function takes a `modifier` function that is passed in each
+    task before it is put into a new graph. It should return a valid task. Note
+    that this is passed _all_ tasks in the graph, not just the set in to_run. You
+    may want to skip modifying tasks not in your to_run list.
+
     If you wish to create the tasks in a new group, leave out decision_task_id."""
     if suffix != '':
         suffix = '-{}'.format(suffix)
     to_run = set(to_run)
 
     #  Copy to avoid side-effects later
     full_task_graph = copy.deepcopy(full_task_graph)
     label_to_taskid = label_to_taskid.copy()
 
     target_graph = full_task_graph.graph.transitive_closure(to_run)
     target_task_graph = TaskGraph(
-        {l: full_task_graph[l] for l in target_graph.nodes},
+        {l: modifier(full_task_graph[l]) for l in target_graph.nodes},
         target_graph)
     target_task_graph.for_each_task(update_parent)
     optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
                                                                 params,
                                                                 to_run,
                                                                 label_to_taskid)
     write_artifact('task-graph{}.json'.format(suffix), optimized_task_graph.to_json())
     write_artifact('label-to-taskid{}.json'.format(suffix), label_to_taskid)
--- a/taskcluster/taskgraph/task.py
+++ b/taskcluster/taskgraph/task.py
@@ -16,17 +16,17 @@ class Task(object):
     - optimization: optimization to apply to the task (see taskgraph.optimize)
     - dependencies: tasks this one depends on, in the form {name: label}, for example
       {'build': 'build-linux64/opt', 'docker-image': 'build-docker-image-desktop-test'}
 
     And later, as the task-graph processing proceeds:
 
     - task_id -- TaskCluster taskId under which this task will be created
 
-    This class is just a convenience wraper for the data type and managing
+    This class is just a convenience wrapper for the data type and managing
     display, comparison, serialization, etc. It has no functionality of its own.
     """
     def __init__(self, kind, label, attributes, task,
                  optimization=None, dependencies=None):
         self.kind = kind
         self.label = label
         self.attributes = attributes
         self.task = task