Bug 1420449: Add mach taskgraph -F/--fast r=ahal draft
authorChris AtLee <catlee@mozilla.com>
Thu, 30 Nov 2017 12:07:02 -0500
changeset 708608 c41e4d838e8920b865cd62bb8de38e64b85b2d84
parent 708607 7cbb88c525ac15a9309523a217af5fe02668f7a7
child 743199 5fe54a359e2cf2ed5029384191762f08e3e2c077
push id92392
push usercatlee@mozilla.com
push dateWed, 06 Dec 2017 21:29:57 +0000
reviewersahal
bugs1420449
milestone59.0a1
Bug 1420449: Add mach taskgraph -F/--fast r=ahal Initially this will skip toolchain task optimizations, which avoids hashing local directory contents and speeds up taskgraph generation by about 25%. MozReview-Commit-ID: B4LB5BV86nw
taskcluster/mach_commands.py
taskcluster/taskgraph/__init__.py
taskcluster/taskgraph/transforms/job/toolchain.py
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -46,17 +46,19 @@ class ShowTaskGraphSubCommand(SubCommand
                             help="parameters file (.yml or .json; see "
                                  "`taskcluster/docs/parameters.rst`)`"),
             CommandArgument('--no-optimize', dest="optimize", action="store_false",
                             default="true",
                             help="do not remove tasks from the graph that are found in the "
                             "index (a.k.a. optimize the graph)"),
             CommandArgument('--tasks-regex', '--tasks', default=None,
                             help="only return tasks with labels matching this regular "
-                            "expression.")
+                            "expression."),
+            CommandArgument('-F', '--fast', dest='fast', default=False, action='store_true',
+                            help="enable fast task generation for local debugging.")
 
         ]
         for arg in args:
             after = arg(after)
         return after
 
 
 @CommandProvider
@@ -311,16 +313,19 @@ class MachCommands(MachCommandBase):
 
         # all of the taskgraph logging is unstructured logging
         self.log_manager.enable_unstructured()
 
     def show_taskgraph(self, graph_attr, options):
         import taskgraph.parameters
         import taskgraph.target_tasks
         import taskgraph.generator
+        import taskgraph
+        if options['fast']:
+            taskgraph.fast = True
 
         try:
             self.setup_logging(quiet=options['quiet'], verbose=options['verbose'])
             parameters = taskgraph.parameters.load_parameters_file(options['parameters'])
             parameters.check()
 
             tgg = taskgraph.generator.TaskGraphGenerator(
                 root_dir=options.get('root'),
--- a/taskcluster/taskgraph/__init__.py
+++ b/taskcluster/taskgraph/__init__.py
@@ -2,8 +2,13 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
 
 GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
+
+# Enable fast task generation for local debugging
+# This is normally switched on via the --fast/-F flag to `mach taskgraph`
+# Currently this skips toolchain task optimizations
+fast = False
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -17,16 +17,17 @@ from taskgraph.transforms.job.common imp
     docker_worker_add_gecko_vcs_env_vars,
     docker_worker_add_public_artifacts,
     docker_worker_add_tooltool,
     support_vcs_checkout,
 )
 from taskgraph.util.hash import hash_paths
 from taskgraph import GECKO
 from taskgraph.util.cached_tasks import add_optimization
+import taskgraph
 
 
 CACHE_TYPE = 'toolchains.v2'
 
 toolchain_run_schema = Schema({
     Required('using'): 'toolchain-script',
 
     # The script (in taskcluster/scripts/misc) to run.
@@ -150,23 +151,24 @@ def docker_worker_toolchain(config, job,
             wrapper, run['script'], args)
     ]
 
     attributes = taskdesc.setdefault('attributes', {})
     attributes['toolchain-artifact'] = run['toolchain-artifact']
     if 'toolchain-alias' in run:
         attributes['toolchain-alias'] = run['toolchain-alias']
 
-    name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
-    add_optimization(
-        config, taskdesc,
-        cache_type=CACHE_TYPE,
-        cache_name=name,
-        digest_data=get_digest_data(config, run, taskdesc),
-    )
+    if not taskgraph.fast:
+        name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
+        add_optimization(
+            config, taskdesc,
+            cache_type=CACHE_TYPE,
+            cache_name=name,
+            digest_data=get_digest_data(config, run, taskdesc),
+        )
 
 
 @run_job_using("generic-worker", "toolchain-script", schema=toolchain_run_schema)
 def windows_toolchain(config, job, taskdesc):
     run = job['run']
     taskdesc['run-on-projects'] = ['trunk', 'try']
 
     worker = taskdesc['worker']
@@ -212,15 +214,16 @@ def windows_toolchain(config, job, taskd
             bash, run['script'], args)
     ]
 
     attributes = taskdesc.setdefault('attributes', {})
     attributes['toolchain-artifact'] = run['toolchain-artifact']
     if 'toolchain-alias' in run:
         attributes['toolchain-alias'] = run['toolchain-alias']
 
-    name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
-    add_optimization(
-        config, taskdesc,
-        cache_type=CACHE_TYPE,
-        cache_name=name,
-        digest_data=get_digest_data(config, run, taskdesc),
-    )
+    if not taskgraph.fast:
+        name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
+        add_optimization(
+            config, taskdesc,
+            cache_type=CACHE_TYPE,
+            cache_name=name,
+            digest_data=get_digest_data(config, run, taskdesc),
+        )