new file mode 100644
--- /dev/null
+++ b/extract.py
@@ -0,0 +1,99 @@
+import json
+import os
+
+def summarize_dict(d):
+ if not d:
+ return 'empty'
+
+ by_value = {}
+ for k, v in d.iteritems():
+ by_value.setdefault(v, []).append(k)
+
+ if len(by_value) == 1:
+ return v
+
+ by_len = [(len(ks), v) for v, ks in by_value.iteritems()]
+ default = by_len[-1][1]
+ lines = []
+ dfl_line = []
+ for v, ks in by_value.iteritems():
+ if v == default:
+ dfl_line.append("<em>{}</em>: {}".format('default', v))
+ else:
+ ks = ', '.join(sorted(ks))
+ lines.append("<em>{}</em>: {}".format(ks, v))
+ return '<br />\n'.join(sorted(lines) + dfl_line)
+
+def main():
+ build_rops = {} # [p][]build-plat/type]: rop
+ test_rops = set() # (p, test-plat/type, build-plat/type, try_name, rop)
+ projects = [p[:-5] for p in os.listdir('after')]
+ try_names = set()
+ test_plats = set()
+ for p in projects:
+ graph = json.load(open("after/{}.json".format(p)))
+ for task in graph.itervalues():
+ attr = task['attributes'].get
+ rop = ' '.join(sorted(attr('run_on_projects')))
+ if attr('kind') == 'build':
+ build_plat_type = "{}/{}".format(attr('build_platform'), attr('build_type'))
+ build_rops.setdefault(p, {})[build_plat_type] = rop
+ elif attr('kind') == 'test':
+ test_plat_type = attr('test_platform')
+ test_plats.add(test_plat_type)
+ build_plat_type = "{}/{}".format(attr('build_platform'), attr('build_type'))
+ try_name = attr('unittest_try_name') or attr('talos_try_name')
+ try_names.add(try_name)
+ test_rops.add((p, test_plat_type, build_plat_type, try_name, rop))
+
+ matrices = {}
+ for p, test_plat_type, build_plat_type, try_name, test_rop in test_rops:
+ matrix = matrices.setdefault(try_name, {p: {} for p in projects})
+ build_plat_row = matrix.setdefault('@build-plat', {})
+ build_rop_row = matrix.setdefault('@build-rop', {})
+ build_plat_row.setdefault(test_plat_type, {})[p] = build_plat_type
+ build_rop_row.setdefault(test_plat_type, {})[p] = build_rops[p][build_plat_type]
+ row = matrix.setdefault(p, {})
+ assert test_plat_type not in row
+ row[test_plat_type] = test_rop
+
+ # summarize build plats and rop
+ for try_name in try_names:
+ matrix = matrices.setdefault(try_name, {})
+ for row in '@build-plat', '@build-rop':
+ row = matrix[row]
+ for col, val in row.iteritems():
+ row[col] = summarize_dict(val)
+
+ print("""\
+<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap-theme.min.css" integrity="sha384-rHyoN1iRsVXV4nD0JutlnGaslCJuC7uwjduW9SVrLvRYooPp2bWYgmgJQIXwl/Sp" crossorigin="anonymous">
+ <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
+ </head>
+ <body>
+""")
+
+ for try_name in sorted(try_names):
+ print("<table class=\"table table-striped table-hover table-condensed\">")
+ print("<tr>")
+ print("<th><h2>{}</h2></th>".format(try_name))
+ for col in sorted(test_plats):
+ print("<th>{}</th>".format(col))
+ print("</tr>")
+ for rowname, row in sorted(matrices[try_name].iteritems()):
+ print("<tr>")
+ print("<th>{}</th>".format(rowname))
+ for col in sorted(test_plats):
+ print("<td>{}</td>".format(row.get(col, '')))
+ print("</tr>")
+ print("</table>")
+ print("</body></html>")
+
+
+main()
new file mode 100644
--- /dev/null
+++ b/gather.sh
@@ -0,0 +1,14 @@
+time=$1
+if [ -z "$time" ]; then
+ echo no time;
+ exit 1
+fi
+
+set -e
+
+projects="autoland mozilla-release stylo mozilla-beta mozilla-aurora try graphics mozilla-esr45 mozilla-central mozilla-inbound"
+[ -n "$2" ] && projects=$2
+for p in $projects; do
+ sed -i -e "s/project:.*/project: $p/" parameters.yml
+ ./mach taskgraph target-graph -p parameters.yml -J > $time/$p.json
+done
new file mode 100644
--- /dev/null
+++ b/graphdiff.py
@@ -0,0 +1,77 @@
+import json
+import difflib
+import sys
+import re
+import os
+
+def relabel(graph):
+ 'restore the original labels, based on task.metadata.name'
+ newgraph = {}
+ changes = {}
+ for old, t in graph.iteritems():
+ new = t['task']['metadata']['name']
+ if new in newgraph:
+ raise Exception("duplicate label %s for %s / %s" % (new, t['label'], newgraph[new]['label']))
+ changes[old] = new
+ newgraph[new] = t
+ return newgraph, changes
+
+def remove_stuff(graph):
+ for t in graph.itervalues():
+ del t['attributes']
+ return graph
+
+def compare(project):
+ print("project {}".format(project))
+ graph1 = json.load(open("before/{}.json".format(project)))
+ graph2 = json.load(open("after/{}.json".format(project)))
+
+ graph1 = remove_stuff(graph1)
+ graph2 = remove_stuff(graph2)
+
+ print("comparing {} tasks".format(len(graph1)))
+
+ diff = False
+
+ only1 = set(graph1) - set(graph2)
+ only2 = set(graph2) - set(graph1)
+ for task in sorted(only1 | only2):
+ if task in only1:
+ print "-" + task
+ del graph1[task]
+ diff = True
+ else:
+ print "+" + task
+ del graph2[task]
+ diff = True
+
+ graph1 = json.dumps(graph1, indent=4, sort_keys=True)
+ graph2 = json.dumps(graph2, indent=4, sort_keys=True)
+
+ graph1 = graph1.split('\n')
+ graph2 = graph2.split('\n')
+
+ if graph1 != graph2:
+ difflines = 0
+ for line in difflib.unified_diff(graph1, graph2, fromfile="before", tofile="after", lineterm='', n=4):
+ diff = True
+ difflines += 1
+ print line
+ # give up after a while..
+ if difflines > 100:
+ break
+
+ if not diff:
+ print("no diff")
+ else:
+ sys.exit(1)
+
+def main():
+ projects = sys.argv[1:]
+ if not projects:
+ projects = [p[:-5] for p in os.listdir('after')]
+ for p in projects:
+ compare(p)
+
+main()
+