Bug 1174313 - Mach target for building job list for all/specific projects draft
authorGregory Arndt <garndt@mozilla.com>
Mon, 15 Jun 2015 14:47:06 -0500
changeset 272381 a538f1251be794abcd4824e99c779a982c96baf6
parent 272380 f4d2f27b10d29d7c2cbabb82ec598f6836f603af
child 506948 f0906fad14941b0dc654449715f51f295c0387f8
push id2879
push usergarndt@mozilla.com
push dateMon, 15 Jun 2015 19:47:24 +0000
bugs1174313
milestone41.0a1
Bug 1174313 - Mach target for building job list for all/specific projects
testing/taskcluster/mach_commands.py
--- a/testing/taskcluster/mach_commands.py
+++ b/testing/taskcluster/mach_commands.py
@@ -454,8 +454,75 @@ class CIBuild(object):
                 "Could not load build task file.  Ensure path is a relative " \
                 "path from testing/taskcluster"
             )
             sys.exit(1)
 
         taskcluster_graph.build_task.validate(build_task)
 
         print(json.dumps(build_task['task'], indent=4))
+
+
+@CommandProvider
+class Jobs(object):
+    @Command('taskcluster-jobs', category='ci',
+        description="Create list of all tasks for a given branch")
+    @CommandArgument('--project',
+        help='Project to use for creating task graph. Example: --project=try')
+    def create_job_list(self, **params):
+        project = params['project']
+
+        job_list = []
+        if project:
+            job_list = self.build_job_list_for_project(project)
+        else:
+            job_flags_path = os.path.join(ROOT, 'tasks', 'branches')
+            dirs = []
+            for d in os.listdir(job_flags_path):
+                if os.path.isdir(os.path.join(job_flags_path, d)):
+                    dirs.append(d)
+
+            for project in dirs:
+                job_list = job_list + self.build_job_list_for_project(project)
+
+        print json.dumps(job_list, indent=4)
+
+    def build_job_list_for_project(self, project):
+        templates = Templates(ROOT)
+
+        job_path = os.path.join(ROOT, 'tasks', 'branches', project, 'job_flags.yml')
+        job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH
+
+        jobs = templates.load(job_path, {})
+        job_list = []
+
+        tests = jobs['tests']
+        builds = jobs['builds']
+        seen_tests = {}
+
+        # build_flag === linux_64_gecko
+        for build_flag in builds:
+            build_types = builds[build_flag]['types']
+            for build_type in build_types:
+                # debug or opt
+                build_task_file = builds[build_flag]['types'][build_type]['task']
+                builder_name = ' '.join([build_flag, project, build_type])
+                job_list.append("{} build".format(builder_name))
+                for test, definition in tests.iteritems():
+                    allowed_build_tasks = definition['allowed_build_tasks']
+                    if build_task_file in allowed_build_tasks.keys():
+                        job_list.append("{} test {}".format(builder_name, test))
+
+                        test_task = allowed_build_tasks[build_task_file]['task']
+                        chunks = 0
+                        if not test_task in seen_tests:
+                            test_task_file = templates.load(test_task, {})
+                            chunks = test_task_file['task'] \
+                                        .get('extra', {})   \
+                                        .get('chunks', {})  \
+                                        .get('total', 0)
+                            seen_tests[test_task] = chunks
+                        if chunks > 1:
+                            for chunk in range(1, chunks+1):
+                                job_list.append(
+                                    "{} test {}-{}".format(builder_name, test, chunk)
+                                )
+        return job_list