Bug 1286075: scripts for testing (DO NOT LAND) draft
authorDustin J. Mitchell <dustin@mozilla.com>
Thu, 08 Sep 2016 00:55:07 +0000
changeset 411348 5758708a3ca4b13a8a42ec776b92970a14e320b6
parent 411347 2255558ad1d209061c86752d7ca12a76b9012bb9
child 411349 ca467a39f99ad208722b4062a53b339841080958
push id28889
push userdmitchell@mozilla.com
push dateThu, 08 Sep 2016 01:22:23 +0000
bugs1286075
milestone51.0a1
Bug 1286075: scripts for testing (DO NOT LAND) These scripts are used to compare "old" (legacy-generated) and "new" task graphs. For each of the pushes listed in paramsets, compare.py will compare the old and new taskgraphs, ignoring known inconsequential changes and highlighting any important differences. To use: * run scripts/gather_old.py at this commit * run scripts/compare.py at the end of the revset (you can use its -f option to re-generate cached taskgraphs) MozReview-Commit-ID: 6DNctgAz5KJ
inputs/parameters.yml
inputs/paramsets.yml
scripts/compare.py
scripts/gather_old.py
scripts/get_paramsets.py
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/task/legacy.py
new file mode 100644
--- /dev/null
+++ b/inputs/parameters.yml
@@ -0,0 +1,4 @@
+{base_repository: 'https://hg.mozilla.org/mozilla-central', head_ref: b83235ed0ea5a46e58ee9e6f402171bf908ac25f,
+  head_repository: 'https://hg.mozilla.org/mozilla-central/', head_rev: b83235ed0ea5a46e58ee9e6f402171bf908ac25f,
+  level: '3', message: ' ', optimize_target_tasks: true, owner: cbook@mozilla.com,
+  project: mozilla-central, pushdate: 1470836899, pushlog_id: '30548', target_tasks_method: all_builds_and_tests}
new file mode 100644
--- /dev/null
+++ b/inputs/paramsets.yml
@@ -0,0 +1,326 @@
+full-graph:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: f97a056ae6235de7855fd8aaa04fb1c8d183bd06
+  head_repository: https://hg.mozilla.org/projects/ash/
+  head_rev: f97a056ae6235de7855fd8aaa04fb1c8d183bd06
+  level: '1'
+  message: '-b do -p all -u all -t all -j all'
+  optimize_target_tasks: false
+  owner: ryanvm@gmail.com
+  project: try
+  pushdate: 1471830480
+  pushlog_id: '122'
+  target_tasks_method: all
+  legacy_optimize: false
+admiring-ash:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: f97a056ae6235de7855fd8aaa04fb1c8d183bd06
+  head_repository: https://hg.mozilla.org/projects/ash/
+  head_rev: f97a056ae6235de7855fd8aaa04fb1c8d183bd06
+  level: '2'
+  message: ' '
+  optimize_target_tasks: true
+  owner: ryanvm@gmail.com
+  project: ash
+  pushdate: 1471830480
+  pushlog_id: '122'
+  target_tasks_method: ash_tasks
+admiring-mozilla-central:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 30fa88c82366a358f412ca2cc3268142c8991f0d
+  head_repository: https://hg.mozilla.org/mozilla-central/
+  head_rev: 30fa88c82366a358f412ca2cc3268142c8991f0d
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: kwierso@gmail.com
+  project: mozilla-central
+  pushdate: 1471653459
+  pushlog_id: '30580'
+  target_tasks_method: default
+awesome-autoland:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 4bfd89efb0c1da0f94b1d580fa7c7f6ce031fcdc
+  head_repository: https://hg.mozilla.org/integration/autoland/
+  head_rev: 4bfd89efb0c1da0f94b1d580fa7c7f6ce031fcdc
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: cpearce@mozilla.com
+  project: autoland
+  pushdate: 1471682286
+  pushlog_id: '31555'
+  target_tasks_method: default
+awesome-mozilla-inbound:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 66d066d5f608c969d486c310d297d2c7fac79aaa
+  head_repository: https://hg.mozilla.org/integration/mozilla-inbound/
+  head_rev: 66d066d5f608c969d486c310d297d2c7fac79aaa
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: spohl@mozilla.com
+  project: mozilla-inbound
+  pushdate: 1471663102
+  pushlog_id: '80846'
+  target_tasks_method: default
+backstabbing-autoland:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: fb9dbacb7f8b52dd47b1ff87f8678179bcf68f4e
+  head_repository: https://hg.mozilla.org/integration/autoland/
+  head_rev: fb9dbacb7f8b52dd47b1ff87f8678179bcf68f4e
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: bchiou@mozilla.com
+  project: autoland
+  pushdate: 1471662396
+  pushlog_id: '31553'
+  target_tasks_method: default
+cocky-autoland:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 258059b7acd8f2ea1448623962a7ca990a2272b6
+  head_repository: https://hg.mozilla.org/integration/autoland/
+  head_rev: 258059b7acd8f2ea1448623962a7ca990a2272b6
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: kwierso@gmail.com
+  project: autoland
+  pushdate: 1471657246
+  pushlog_id: '31552'
+  target_tasks_method: default
+dreamy-mozilla-aurora:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 3443b3c753233c7ea260b09366b4b28ac2c4525a
+  head_repository: https://hg.mozilla.org/releases/mozilla-aurora/
+  head_rev: 3443b3c753233c7ea260b09366b4b28ac2c4525a
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: kwierso@gmail.com
+  project: mozilla-aurora
+  pushdate: 1470869736
+  pushlog_id: '9928'
+  target_tasks_method: default
+drunk-try:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 8035c65525854735f5a5880182a2978414669ce7
+  head_repository: https://hg.mozilla.org/try/
+  head_rev: 8035c65525854735f5a5880182a2978414669ce7
+  level: '1'
+  message: 'try: -b d -p linux -u all -t none'
+  optimize_target_tasks: true
+  owner: leo@gaspard.io
+  project: try
+  pushdate: 1471653704
+  pushlog_id: '140097'
+  target_tasks_method: try_option_syntax
+eager-mozilla-aurora:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 2a0981531355dc77614e72c72fda7498bb074ffe
+  head_repository: https://hg.mozilla.org/releases/mozilla-aurora/
+  head_rev: 2a0981531355dc77614e72c72fda7498bb074ffe
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: kwierso@gmail.com
+  project: mozilla-aurora
+  pushdate: 1470869736
+  pushlog_id: '9927'
+  target_tasks_method: default
+ecstatic-autoland:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 3f3459b7c54ac0225037edc6665113603b85e15c
+  head_repository: https://hg.mozilla.org/integration/autoland/
+  head_rev: 3f3459b7c54ac0225037edc6665113603b85e15c
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: ttromey@mozilla.com
+  project: autoland
+  pushdate: 1471720815
+  pushlog_id: '31556'
+  target_tasks_method: default
+elastic-ash:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 7963ebdd52b93f96b812eff2eab8d94097147b9c
+  head_repository: https://hg.mozilla.org/projects/ash/
+  head_rev: 7963ebdd52b93f96b812eff2eab8d94097147b9c
+  level: '2'
+  message: ' '
+  optimize_target_tasks: true
+  owner: ryanvm@gmail.com
+  project: ash
+  pushdate: 1472145383
+  pushlog_id: '126'
+  target_tasks_method: ash_tasks
+elegant-mozilla-central:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 5f3d66a9ea5ddcf59e0b83a81579773aa7d68ae4
+  head_repository: https://hg.mozilla.org/mozilla-central/
+  head_rev: 5f3d66a9ea5ddcf59e0b83a81579773aa7d68ae4
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: ryanvm@gmail.com
+  project: mozilla-central
+  pushdate: 1471873097
+  pushlog_id: '30584'
+  target_tasks_method: default
+furious-ash:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 01748a2b1a463f24efd9cd8abad9ccfd76b037b8
+  head_repository: https://hg.mozilla.org/projects/ash/
+  head_rev: 01748a2b1a463f24efd9cd8abad9ccfd76b037b8
+  level: '2'
+  message: ' '
+  optimize_target_tasks: true
+  owner: ryanvm@gmail.com
+  project: ash
+  pushdate: 1472048616
+  pushlog_id: '125'
+  target_tasks_method: ash_tasks
+gigantic-mozilla-central:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: f97a056ae6235de7855fd8aaa04fb1c8d183bd06
+  head_repository: https://hg.mozilla.org/mozilla-central/
+  head_rev: f97a056ae6235de7855fd8aaa04fb1c8d183bd06
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: kwierso@gmail.com
+  project: mozilla-central
+  pushdate: 1471655785
+  pushlog_id: '30582'
+  target_tasks_method: default
+goofy-mozilla-inbound:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: afd27635d16ebe223b9186f0033b77cd2bbb48d5
+  head_repository: https://hg.mozilla.org/integration/mozilla-inbound/
+  head_rev: afd27635d16ebe223b9186f0033b77cd2bbb48d5
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: kwierso@gmail.com
+  project: mozilla-inbound
+  pushdate: 1471657199
+  pushlog_id: '80844'
+  target_tasks_method: default
+grave-mozilla-inbound:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 7a3c14d1770f8e3f3811642455e144741e9ff67a
+  head_repository: https://hg.mozilla.org/integration/mozilla-inbound/
+  head_rev: 7a3c14d1770f8e3f3811642455e144741e9ff67a
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: jvarga@mozilla.com
+  project: mozilla-inbound
+  pushdate: 1471659416
+  pushlog_id: '80845'
+  target_tasks_method: default
+happy-autoland:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 9de2153ec067c283e5c7fa9732f8742092eddd58
+  head_repository: https://hg.mozilla.org/integration/autoland/
+  head_rev: 9de2153ec067c283e5c7fa9732f8742092eddd58
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: sledru@mozilla.com
+  project: autoland
+  pushdate: 1471681069
+  pushlog_id: '31554'
+  target_tasks_method: default
+loving-mozilla-central:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 662270014cb72f82af415385f2f0bef5b135ec04
+  head_repository: https://hg.mozilla.org/mozilla-central/
+  head_rev: 662270014cb72f82af415385f2f0bef5b135ec04
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: ryanvm@gmail.com
+  project: mozilla-central
+  pushdate: 1471872992
+  pushlog_id: '30583'
+  target_tasks_method: default
+peaceful-mozilla-central:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: f0b406c1b36b6b5c881fbb24bc9669d9ef8b9d8a
+  head_repository: https://hg.mozilla.org/mozilla-central/
+  head_rev: f0b406c1b36b6b5c881fbb24bc9669d9ef8b9d8a
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: kwierso@gmail.com
+  project: mozilla-central
+  pushdate: 1471654505
+  pushlog_id: '30581'
+  target_tasks_method: default
+sleepy-ash:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 194fe275b4e60ded2af6b25173eec421f0dba8ad
+  head_repository: https://hg.mozilla.org/projects/ash/
+  head_rev: 194fe275b4e60ded2af6b25173eec421f0dba8ad
+  level: '2'
+  message: ' '
+  optimize_target_tasks: true
+  owner: ryanvm@gmail.com
+  project: ash
+  pushdate: 1471873287
+  pushlog_id: '123'
+  target_tasks_method: ash_tasks
+sleepy-mozilla-inbound:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 59287ede98169ffe2603753658c922b834c2d08a
+  head_repository: https://hg.mozilla.org/integration/mozilla-inbound/
+  head_rev: 59287ede98169ffe2603753658c922b834c2d08a
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: maglione.k@gmail.com
+  project: mozilla-inbound
+  pushdate: 1471652467
+  pushlog_id: '80842'
+  target_tasks_method: default
+stoic-ash:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 052656fc513c05da969590ac5934abd67271a897
+  head_repository: https://hg.mozilla.org/projects/ash/
+  head_rev: 052656fc513c05da969590ac5934abd67271a897
+  level: '2'
+  message: ' '
+  optimize_target_tasks: true
+  owner: ryanvm@gmail.com
+  project: ash
+  pushdate: 1471961311
+  pushlog_id: '124'
+  target_tasks_method: ash_tasks
+tiny-mozilla-aurora:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 7b44f5b3b0bb503b79166ac861b1f3f9c1f8abb9
+  head_repository: https://hg.mozilla.org/releases/mozilla-aurora/
+  head_rev: 7b44f5b3b0bb503b79166ac861b1f3f9c1f8abb9
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: kwierso@gmail.com
+  project: mozilla-aurora
+  pushdate: 1470869736
+  pushlog_id: '9929'
+  target_tasks_method: default
+tiny-mozilla-inbound:
+  base_repository: https://hg.mozilla.org/mozilla-central
+  head_ref: 14733a383b4cf29ce6ed70a5d22dd3cb060c0eb1
+  head_repository: https://hg.mozilla.org/integration/mozilla-inbound/
+  head_rev: 14733a383b4cf29ce6ed70a5d22dd3cb060c0eb1
+  level: '3'
+  message: ' '
+  optimize_target_tasks: true
+  owner: ecoal95@gmail.com
+  project: mozilla-inbound
+  pushdate: 1471656409
+  pushlog_id: '80843'
+  target_tasks_method: default
new file mode 100644
--- /dev/null
+++ b/scripts/compare.py
@@ -0,0 +1,226 @@
+#! /usr/bin/python
+
+"""
+Compare new taskgraphs to old taskgraphs; this uses the data generated by
+gather_old.py (representing the "old" taskgraphs, generated by the legacy kind)
+and calls `mach taskgraph` to create a new taskgraph, then compares the two,
+ignoring unimportant differences and showing any remaining differences, either in the
+tasks created or in the definitions of those tasks.
+"""
+
+import difflib
+import json
+import os
+import re
+import subprocess
+import sys
+
+sys.path.insert(0, 'python/pyyaml/lib')
+import yaml
+
+def relabel(graph):
+    rv = {}
+    for task_id, t in graph.iteritems():
+        l = {}
+        if t['attributes']['kind'] == 'upload-symbols' or t['attributes'].get('post_build') == 'upload-symbols':
+            # no treeherder info, so we make it up
+            l = "US {build_platform}/{build_type}".format(**t['attributes'])
+        elif t['attributes']['kind'] == 'docker-image':
+            # no treeherder info, so we make it up
+            l = "I {image_name}".format(**t['attributes'])
+        else:
+            for name, path, default in [
+                ('gs', 'task.extra.treeherder.groupSymbol', '?'),
+                ('s', 'task.extra.treeherder.symbol', '?'),
+                ('pl', 'task.extra.treeherder.machine.platform', '?'),
+            ]:
+                try:
+                    x = t
+                    for elt in path.split('.'):
+                        x = x[elt]
+                    l[name] = str(x)
+                except KeyError:
+                    l[name] = default
+            try:
+                l['ty'] = '/'.join(sorted(t['task']['extra']['treeherder']['collection'].keys()))
+            except KeyError:
+                l['ty'] = '?'
+            l = '{gs}({s})-{pl}/{ty}'.format(**l)
+        if l in rv:
+            raise Exception("duplicate label %s for %s / %s" % (l, t['label'], rv[l]['label']))
+        rv[l] = t
+    return rv
+
+def replace_labels(graph, str):
+    inverse = {t['task_id']: l for l, t in graph.iteritems()}
+    if not inverse:
+        return str
+    return re.sub("|".join(inverse), lambda mo: inverse[mo.group(0)], str)
+
+def remove(t, path):
+    path = path.split('.')
+    try:
+        for elt in path[:-1]:
+            t = t[elt]
+        del t[path[-1]]
+    except KeyError:
+        pass
+
+def annotate_images(graph):
+    for t in graph.itervalues():
+        # search dependencies for a docker image
+        if 'docker-image' in t['dependencies']:
+            img_task = graph[t['dependencies']['docker-image']]
+            image_name = img_task['attributes']['image_name']
+            t['task']['payload']['docker-image'] = image_name
+    return graph
+
+def drop_stuff(graph):
+    """
+    Drop unimportant aspects of the tasks.
+    """
+    tid2label = {t['task_id']: label for label, t in graph.iteritems()}
+    for t in graph.itervalues():
+        remove(t, 'kind_implementation')
+        remove(t, 'task.created')
+        remove(t, 'task.metadata')
+        remove(t, 'task.deadline')
+        remove(t, 'task.expires')
+        remove(t, 'label')
+        remove(t, 'task.schedulerId')
+        remove(t, 'attributes.kind')
+        remove(t, 'attributes.post_build')
+        remove(t, 'attributes.legacy_kind')
+        remove(t, 'attributes.job')
+        remove(t, 'attributes.e10s')
+        remove(t, 'attributes.run_on_projects')
+
+        # t['dependencies'] is irrelevant in an optimized taskgraph
+        remove(t, 'dependencies')
+
+        # convert dependency taskIds to the target task's label, and sort
+        t['task']['dependencies'] = sorted([tid2label.get(d, 'no-' + d) for d in t['task']['dependencies']])
+
+        if 'env' in t['task']['payload']:
+            env = t['task']['payload']['env']
+            # strip trailing space from configs
+            if 'MOZHARNESS_CONFIG' in env:
+                env['MOZHARNESS_CONFIG'] = env['MOZHARNESS_CONFIG'].rstrip()
+            # treat 'true' and True as the same for env values, fold strings
+            t['task']['payload']['env'] = {v: 'true' if e is True else str(e) for v, e in env.iteritems()}
+
+        # groupName was inconsistent for the same group; fixed now, but ignore the differences
+        remove(t, 'task.extra.treeherder.groupName')
+
+        # ignore groupSymbol == '?'
+        try:
+            if t['task']['extra']['treeherder']['groupSymbol'] == '?':
+                del t['task']['extra']['treeherder']['groupSymbol']
+        except KeyError:
+            pass
+
+        remove(t, 'task.extra.treeherderEnv')
+
+        # this is ignored per
+        # https://schemas.taskcluster.net/taskcluster-treeherder/v1/task-treeherder-config.json#
+        remove(t, 'task.extra.treeherder.build')
+
+        # remove some unused fields; these were leftover variables that snuck
+        # into extra
+        remove(t, 'task.extra.build_name')
+        remove(t, 'task.extra.build_platform')
+        remove(t, 'task.extra.build_product')
+        remove(t, 'task.extra.build_type')
+
+        # unnecessary in new taskgraphs
+        remove(t, 'task.extra.locations')
+
+        # symbols should be strings, even if they are just a chunk number
+        try:
+            t['task']['extra']['treeherder']['symbol'] = str(t['task']['extra']['treeherder']['symbol'])
+        except KeyError:
+            pass
+
+        # sort command options for docker-worker runs, since option order is not consistent
+        if isinstance(t['task']['payload'].get('artifacts'), (dict, type(None))) and 'command' in t['task']['payload']:
+            command = t['task']['payload']['command']
+            command = [s.strip() if isinstance(s, basestring) else s for s in command]
+            t['task']['payload']['command'] = command[:2] + sorted(command[2:])
+
+
+        # make sure that these fields exist, even if empty
+        t['task'].setdefault('scopes', []).sort()
+        t['task'].setdefault('routes', []).sort()
+    return graph
+
+def compare(old, new):
+    graph1 = annotate_images(json.load(open(old)))
+    graph2 = annotate_images(json.load(open(new)))
+
+    graph1 = drop_stuff(relabel(graph1))
+    graph2 = drop_stuff(relabel(graph2))
+
+    print("{} tasks -> {} tasks".format(len(graph1), len(graph2)))
+
+    diff = False
+    show = True
+    ignored = []
+    combined = set(graph1) | set(graph2)
+    for l in sorted(combined):
+        if l in graph1:
+            if l not in graph2:
+                diff = True
+                if show: print("-" + l)
+        else:
+            if l in graph2:
+                # don't count added I tasks as a difference, nor changes to where SM tasks run
+                if l.startswith("I") or 'sm-' in l:
+                    ignored.append(l)
+                else:
+                    diff = True
+                if show: print("+" + l)
+    if diff:
+        return False
+
+    # remove ignored tasks
+    for l in ignored:
+        del graph2[l]
+
+    print "replacing labels"
+    graph1 = replace_labels(graph1, json.dumps(graph1, indent=4, sort_keys=True)).split('\n')
+    graph2 = replace_labels(graph2, json.dumps(graph2, indent=4, sort_keys=True)).split('\n')
+
+    print "calculating diff"
+    for line in difflib.unified_diff(graph1, graph2, fromfile="tasks-old.json", tofile="tasks-new.json", lineterm='', n=20):
+        diff = True
+        print line
+
+    if not diff:
+        print("++++ no diff")
+
+    return not diff
+
+
+def main():
+    force = False
+    verbose = ''
+    if '-f' in sys.argv:
+        sys.argv.remove('-f')
+        force = True
+    if '-v' in sys.argv:
+        sys.argv.remove('-v')
+        verbose = '-v'
+    to_show = set(sys.argv[1:])
+    paramsets = yaml.load(open("inputs/paramsets.yml"))
+    for name, params in paramsets.iteritems():
+        if to_show and name not in to_show:
+            continue
+        print("====== {}".format(name))
+        if force or not os.path.exists("outputs/tasks-new-{}.json".format(name)):
+            with open("parameters.yml", "w") as f:
+                yaml.dump(params, f)
+            subprocess.check_call('./mach taskgraph optimized {} -p parameters.yml -J > outputs/tasks-new-{}.json'.format(verbose, name), shell=True)
+        if not compare("outputs/tasks-old-{}.json".format(name), "outputs/tasks-new-{}.json".format(name)):
+            break
+
+main()
new file mode 100755
--- /dev/null
+++ b/scripts/gather_old.py
@@ -0,0 +1,25 @@
+#! /usr/bin/python
+
+import os
+import sys
+sys.path.insert(0, 'python/pyyaml/lib')
+import subprocess
+
+import yaml
+
+def main():
+    todo = set(sys.argv[1:])
+
+    paramsets = yaml.load(open("inputs/paramsets.yml"))
+    for name, params in paramsets.iteritems():
+        if todo and name not in todo:
+            continue
+        yaml.safe_dump(params, sys.stdout, default_flow_style=False)
+        try:
+            with open("parameters.yml", "w") as f:
+                yaml.dump(params, f)
+            subprocess.check_call('./mach taskgraph optimized -p parameters.yml -J > outputs/tasks-old-{}.json'.format(name), shell=True)
+        finally:
+            os.unlink("parameters.yml")
+
+main()
new file mode 100644
--- /dev/null
+++ b/scripts/get_paramsets.py
@@ -0,0 +1,154 @@
+import sys
+import random
+
+sys.path.insert(0, "python/requests")
+import requests
+sys.path.insert(0, "python/pyyaml/lib")
+import yaml
+
+ADJ = [
+    "admiring",
+    "adoring",
+    "affectionate",
+    "agitated",
+    "amazing",
+    "angry",
+    "awesome",
+    "backstabbing",
+    "berserk",
+    "big",
+    "boring",
+    "clever",
+    "cocky",
+    "compassionate",
+    "condescending",
+    "cranky",
+    "desperate",
+    "determined",
+    "distracted",
+    "dreamy",
+    "drunk",
+    "eager",
+    "ecstatic",
+    "elastic",
+    "elated",
+    "elegant",
+    "evil",
+    "fervent",
+    "focused",
+    "furious",
+    "gigantic",
+    "gloomy",
+    "goofy",
+    "grave",
+    "happy",
+    "high",
+    "hopeful",
+    "hungry",
+    "infallible",
+    "jolly",
+    "jovial",
+    "kickass",
+    "lonely",
+    "loving",
+    "mad",
+    "modest",
+    "naughty",
+    "nauseous",
+    "nostalgic",
+    "peaceful",
+    "pedantic",
+    "pensive",
+    "prickly",
+    "reverent",
+    "romantic",
+    "sad",
+    "serene",
+    "sharp",
+    "sick",
+    "silly",
+    "sleepy",
+    "small",
+    "stoic",
+    "stupefied",
+    "suspicious",
+    "tender",
+    "thirsty",
+    "tiny",
+    "trusting",
+    "zen",
+]
+
+def adjectives():
+    adj = ADJ[:]
+    random.shuffle(adj)
+    for a in adj:
+        yield a
+    for root in adjectives():
+        for a in adj:
+            yield "{}-{}".format(root, a)
+
+## generate a list of decision tasks
+
+headers = {'User-agent': 'Mozilla/5.0'}
+
+paramsets = {}
+
+# create a paramset that will generate the full task graph
+paramsets["full-graph"] = yaml.load("""
+base_repository: https://hg.mozilla.org/mozilla-central
+head_ref: f97a056ae6235de7855fd8aaa04fb1c8d183bd06
+head_repository: https://hg.mozilla.org/projects/ash/
+head_rev: f97a056ae6235de7855fd8aaa04fb1c8d183bd06
+level: '1'
+message: '-b do -p all -u all -t all -j all'
+optimize_target_tasks: false
+owner: ryanvm@gmail.com
+project: try
+pushdate: 1471830480
+pushlog_id: '122'
+target_tasks_method: all
+legacy_optimize: false
+""")
+
+continuationToken = None
+for tree, N in [
+        ("try", 1),
+        ("mozilla-aurora", 5),
+        ("mozilla-central", 5),
+        ("ash", 5),
+        ("autoland", 5),
+        ("mozilla-inbound", 5),
+        #('stylo', 5),
+        #('stylo-try', 5),
+    ]:
+    adj_seq = adjectives()
+    def mkname():
+        return "{}-{}".format(adj_seq.next(), tree)
+    url = "https://treeherder.mozilla.org/api/project/{}/jobs/?count={}&job_type_symbol=D&last_modified__gt=2016-08-20T00:00:00".format(tree, N)
+    jobs = requests.get(url, headers=headers).json()['results']
+    for job in jobs:
+        job_guid = job['job_guid']
+        print >>sys.stderr, job_guid, job['last_modified']
+        url = "https://treeherder.mozilla.org/api/jobdetail/?job_guid={}".format(job_guid)
+        things = requests.get(url, headers=headers).json()['results']
+        parameters_url = None
+        for thing in things:
+            if thing['value'] == 'parameters.yml':
+                parameters_url = thing['url']
+                break
+        if parameters_url:
+            parameters = requests.get(parameters_url).text
+            new_param = yaml.load(parameters)
+            if 'level' not in new_param:
+                continue # not real
+            if 'pushdate' not in new_param:
+                new_param['pushdate'] = 1470869736
+            if new_param['target_tasks_method'] == 'all_builds_and_tests':
+                new_param['target_tasks_method'] = 'default'
+            name = mkname()
+            print " ", name
+            paramsets[name] = new_param
+
+with open("inputs/paramsets.yml", "wb") as f:
+    yaml.safe_dump(paramsets, f, default_flow_style=False)
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -24,16 +24,23 @@ def _target_task(name):
     return wrap
 
 
 def get_method(method):
     """Get a target_task_method to pass to a TaskGraphGenerator."""
     return _target_task_methods[method]
 
 
+@_target_task('all')
+def target_tasks_all(full_task_graph, parameters):
+    """Target all tasks.  This can be useful when testing, but is probably
+    not the taskgraph you actually want to run!"""
+    return full_task_graph.tasks.keys()
+
+
 @_target_task('from_parameters')
 def target_tasks_from_parameters(full_task_graph, parameters):
     """Get the target task set from parameters['target_tasks'].  This is
     useful for re-running a decision task with the same target set as in an
     earlier run, by copying `target_tasks.json` into `parameters.yml`."""
     return parameters['target_tasks']
 
 
--- a/taskcluster/taskgraph/task/legacy.py
+++ b/taskcluster/taskgraph/task/legacy.py
@@ -432,16 +432,18 @@ class LegacyTask(base.Task):
             'owner': params['owner'],
             # TODO: Add full mach commands to this example?
             'description': 'Task graph generated via ./mach taskcluster-graph',
             'name': 'task graph local'
         }
 
         # Filter the job graph according to conditions met by this invocation run.
         def should_run(task):
+            if not params.get('legacy_optimize', True):
+                return True
             # Old style build or test task that doesn't define conditions. Always runs.
             if 'when' not in task:
                 return True
 
             when = task['when']
 
             # If the task defines file patterns and we have a set of changed
             # files to compare against, only run if a file pattern matches one