Bug 1275409: move taskcluster_graph.commit_parser; r?wcosta draft
authorDustin J. Mitchell <dustin@mozilla.com>
Sat, 04 Jun 2016 21:09:18 +0000
changeset 376704 6a369097dbdff0fbd40a14ca6bf508f81487b8f4
parent 376703 f38a5f406619d7356124d9e75a8ef5b94ca310ea
child 376705 73235567967c8eb835261862c3d364539c26f3c5
push id20643
push userdmitchell@mozilla.com
push dateWed, 08 Jun 2016 13:31:04 +0000
reviewerswcosta
bugs1275409
milestone50.0a1
Bug 1275409: move taskcluster_graph.commit_parser; r?wcosta MozReview-Commit-ID: JWCzOWVUEP6
taskcluster/taskgraph/kind/legacy.py
taskcluster/taskgraph/test/test_util_legacy_commit_parser.py
taskcluster/taskgraph/util/legacy_commit_parser.py
testing/taskcluster/taskcluster_graph/commit_parser.py
testing/taskcluster/taskcluster_graph/try_test_parser.py
testing/taskcluster/tests/test_commit_parser.py
testing/taskcluster/tests/test_try_test_parser.py
--- a/taskcluster/taskgraph/kind/legacy.py
+++ b/taskcluster/taskgraph/kind/legacy.py
@@ -13,17 +13,17 @@ import sys
 import time
 from collections import defaultdict, namedtuple
 
 from . import base
 from ..types import Task
 from functools import partial
 from mozpack.path import match as mozpackmatch
 from slugid import nice as slugid
-from taskcluster_graph.commit_parser import parse_commit
+from taskgraph.util.legacy_commit_parser import parse_commit
 from taskgraph.util.time import (
     json_time_from_now,
     current_json_time,
 )
 from taskgraph.util.templates import Templates
 import taskcluster_graph.build_task
 from taskgraph.util.docker import docker_image
 
rename from testing/taskcluster/tests/test_commit_parser.py
rename to taskcluster/taskgraph/test/test_util_legacy_commit_parser.py
--- a/testing/taskcluster/tests/test_commit_parser.py
+++ b/taskcluster/taskgraph/test/test_util_legacy_commit_parser.py
@@ -1,19 +1,20 @@
 #!/usr/bin/env python
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import unittest
 import mozunit
-from taskcluster_graph.commit_parser import (
+from taskgraph.util.legacy_commit_parser import (
     parse_commit,
-    normalize_test_list
+    normalize_test_list,
+    parse_test_opts
 )
 
 class TestCommitParser(unittest.TestCase):
 
     def test_normalize_test_list_none(self):
         self.assertEqual(
             normalize_test_list({}, ['woot'], 'none'), []
         )
@@ -109,17 +110,18 @@ class TestCommitParser(unittest.TestCase
         expected = [
             {
                 'task': 'task/linux',
                 'dependents': [],
                 'additional-parameters': {},
                 'build_name': 'linux',
                 'build_type': 'opt',
                 'post-build': [],
-                'interactive': False
+                'interactive': False,
+                'when': {}
             }
         ]
 
         result, triggers = parse_commit(commit, jobs)
         self.assertEqual(expected, result)
 
     def test_flag_aliasing(self):
         commit = 'try: -b o -p magic-alias -u none -t none'
@@ -149,17 +151,18 @@ class TestCommitParser(unittest.TestCase
         expected = [
             {
                 'task': 'task/linux',
                 'dependents': [],
                 'additional-parameters': {},
                 'build_name': 'linux',
                 'build_type': 'opt',
                 'interactive': False,
-                'post-build': []
+                'post-build': [],
+                'when': {},
             }
         ]
 
         result, triggers = parse_commit(commit, jobs)
         self.assertEqual(expected, result)
 
     def test_try_flag_in_middle_of_commit(self):
         '''
@@ -190,17 +193,18 @@ class TestCommitParser(unittest.TestCase
         expected = [
             {
                 'task': 'task/linux',
                 'dependents': [],
                 'additional-parameters': {},
                 'build_name': 'linux',
                 'build_type': 'opt',
                 'interactive': False,
-                'post-build': []
+                'post-build': [],
+                'when': {}
             }
         ]
 
         result, triggers = parse_commit(commit, jobs)
         self.assertEqual(expected, result)
 
     def test_try_flags_not_specified(self):
         '''
@@ -261,17 +265,18 @@ class TestCommitParser(unittest.TestCase
         expected = [
             {
                 'task': 'task/linux',
                 'dependents': [],
                 'post-build': [],
                 'build_name': 'linux',
                 'build_type': 'opt',
                 'interactive': False,
-                'additional-parameters': {}
+                'additional-parameters': {},
+                'when': {}
             }
         ]
 
         result, triggers = parse_commit(commit, jobs)
         self.assertEqual(expected, result)
 
     def test_some_test_tasks_restricted(self):
         '''
@@ -310,31 +315,34 @@ class TestCommitParser(unittest.TestCase
             {
                 'task': 'task/linux-debug',
                 'dependents': [],
                 'additional-parameters': {},
                 'post-build': [],
                 'build_name': 'linux',
                 'build_type': 'debug',
                 'interactive': False,
+                'when': {},
             },
             {
                 'task': 'task/linux',
                 'dependents': [{
                     'allowed_build_tasks': {
                         'task/linux': {
-                            'task':'task/web-platform-tests'
+                            'task':'task/web-platform-tests',
+                            'unittest_try_name':'web-platform-tests'
                         }
                     }
                 }],
                 'additional-parameters': {},
                 'post-build': [],
                 'build_name': 'linux',
                 'build_type': 'opt',
                 'interactive': False,
+                'when': {},
             }
         ]
 
         result, triggers = parse_commit(commit, jobs)
         self.assertEqual(expected, result)
 
 
     def test_specific_test_platforms(self):
@@ -396,60 +404,67 @@ class TestCommitParser(unittest.TestCase
             {
                 'task': 'task/linux',
                 'dependents': [],
                 'additional-parameters': {},
                 'post-build': [],
                 'build_name': 'linux',
                 'build_type': 'opt',
                 'interactive': False,
+                'when': {},
             },
             {
                 'task': 'task/linux-debug',
                 'dependents': [],
                 'additional-parameters': {},
                 'post-build': [],
                 'build_name': 'linux',
                 'build_type': 'debug',
                 'interactive': False,
+                'when': {},
             },
             {
                 'task': 'task/win32',
                 'dependents': [
                     {
                         'allowed_build_tasks': {
                             'task/linux': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/win32': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             }
                         }
                     },
                     {
                         'allowed_build_tasks': {
                             'task/linux-debug': {
                                 'task': 'task/mochitest',
+                                'unittest_try_name': 'mochitest',
                             },
                             'task/win32': {
                                 'task': 'task/mochitest',
+                                'unittest_try_name': 'mochitest',
                             }
                         }
                     }
                 ],
                 'additional-parameters': {},
                 'post-build': [],
                 'build_name': 'win32',
                 'build_type': 'opt',
                 'interactive': False,
+                'when': {},
             }
         ]
 
         result, triggers = parse_commit(commit, jobs)
-        self.assertEqual(expected, result)
+        self.assertEqual(sorted(expected), sorted(result))
 
     def test_specific_test_platforms_with_specific_platform(self):
         '''
         This test cases covers the platform specific test exclusion options.
         '''
         commit = 'try: -b od -p win32 -u mochitest[windows] -t none'
         jobs = {
             'flags': {
@@ -503,28 +518,31 @@ class TestCommitParser(unittest.TestCase
         expected = [
             {
                 'task': 'task/win32',
                 'dependents': [
                     {
                         'allowed_build_tasks': {
                             'task/linux-debug': {
                                 'task': 'task/mochitest',
+                                'unittest_try_name': 'mochitest',
                             },
                             'task/win32': {
                                 'task': 'task/mochitest',
+                                'unittest_try_name': 'mochitest',
                             }
                         }
                     }
                 ],
                 'additional-parameters': {},
                 'post-build': [],
                 'build_name': 'win32',
                 'build_type': 'opt',
                 'interactive': False,
+                'when': {}
             }
         ]
 
         result, triggers = parse_commit(commit, jobs)
         self.assertEqual(expected, result)
 
     def test_specific_chunks(self):
         '''
@@ -564,26 +582,28 @@ class TestCommitParser(unittest.TestCase
             {
                 'task': 'task/linux',
                 'dependents': [
                     {
                         'allowed_build_tasks': {
                             'task/linux': {
                                 'task': 'task/mochitest',
                                 'chunks': 5,
-                                'only_chunks': set([1, 2])
+                                'only_chunks': set([1, 2]),
+                                'unittest_try_name': 'mochitest',
                             },
                         }
                     }
                 ],
                 'additional-parameters': {},
                 'post-build': [],
                 'build_name': 'linux',
                 'build_type': 'opt',
                 'interactive': False,
+                'when': {},
             }
         ]
         result, triggers = parse_commit(commit, jobs)
         self.assertEqual(expected, result)
 
     def test_commit_with_builds_and_tests(self):
         '''
         This test covers the broad case of a commit which has both builds and
@@ -640,91 +660,107 @@ class TestCommitParser(unittest.TestCase
         expected = [
             {
                 'task': 'task/linux',
                 'dependents': [
                     {
                         'allowed_build_tasks': {
                             'task/linux': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             }
                         }
                     }
                 ],
                 'additional-parameters': {}
             },
             {
                 'task': 'task/linux-debug',
                 'dependents': [
                     {
                         'allowed_build_tasks': {
                             'task/linux': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             }
                         }
                     }
                 ],
                 'additional-parameters': {}
             },
             {
                 'task': 'task/linux64',
                 'dependents': [
                     {
                         'allowed_build_tasks': {
                             'task/linux': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             }
                         }
                     }
                 ],
                 'additional-parameters': {}
             },
             {
                 'task': 'task/linux64-debug',
                 'dependents': [
                     {
                         'allowed_build_tasks': {
                             'task/linux': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             }
                         }
                     }
                 ],
                 'additional-parameters': {}
             }
         ]
 
@@ -785,113 +821,183 @@ class TestCommitParser(unittest.TestCase
         expected = [
             {
                 'task': 'task/linux',
                 'dependents': [
                     {
                         'allowed_build_tasks': {
                             'task/linux': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             }
                         }
                     }
                 ],
                 'additional-parameters': {},
                 'build_name': 'linux',
                 'build_type': 'opt',
                 'post-build': [],
-                'interactive': False
+                'interactive': False,
+                'when': {}
             },
             {
                 'task': 'task/linux-debug',
                 'dependents': [
                     {
                         'allowed_build_tasks': {
                             'task/linux': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             }
                         }
                     }
                 ],
                 'additional-parameters': {},
                 'build_name': 'linux',
                 'build_type': 'debug',
                 'post-build': [],
-                'interactive': False
+                'interactive': False,
+                'when': {}
             },
             {
                 'task': 'task/linux64',
                 'dependents': [
                     {
                         'allowed_build_tasks': {
                             'task/linux': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             }
                         }
                     }
                 ],
                 'additional-parameters': {},
                 'build_name': 'linux64',
                 'build_type': 'opt',
                 'post-build': [],
-                'interactive': False
+                'interactive': False,
+                'when': {}
             },
             {
                 'task': 'task/linux64-debug',
                 'dependents': [
                     {
                         'allowed_build_tasks': {
                             'task/linux': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             },
                             'task/linux64-debug': {
                                 'task': 'task/web-platform-tests',
+                                'unittest_try_name': 'web-platform-tests',
                             }
                         }
                     }
                 ],
                 'additional-parameters': {},
                 'build_name': 'linux64',
                 'build_type': 'debug',
                 'post-build': [],
-                'interactive': False
+                'interactive': False,
+                'when': {}
             }
         ]
 
         result, triggers = parse_commit(commit, jobs)
-        self.assertEqual(expected, result)
+        self.assertEqual(sorted(expected), sorted(result))
+
+
+class TryTestParserTest(unittest.TestCase):
+
+    def test_parse_opts_valid(self):
+        self.assertEquals(
+            parse_test_opts('all[Mulet Linux]'),
+            [{ 'test': 'all', 'platforms': ['Mulet Linux'] }]
+        )
+
+        self.assertEquals(
+            parse_test_opts('all[Amazing, Foobar woot,yeah]'),
+            [{ 'test': 'all', 'platforms': ['Amazing', 'Foobar woot', 'yeah'] }]
+        )
+
+        self.assertEquals(
+            parse_test_opts('a,b, c'),
+            [
+                { 'test': 'a' },
+                { 'test': 'b' },
+                { 'test': 'c' },
+            ]
+        )
+        self.assertEquals(
+            parse_test_opts('woot, bar[b], baz, qux[ z ],a'),
+            [
+                { 'test': 'woot' },
+                { 'test': 'bar', 'platforms': ['b'] },
+                { 'test': 'baz' },
+                { 'test': 'qux', 'platforms': ['z'] },
+                { 'test': 'a' }
+            ]
+        )
+
+        self.assertEquals(
+            parse_test_opts('mochitest-3[Ubuntu,10.6,10.8,Windows XP,Windows 7,Windows 8]'),
+            [
+                {
+                    'test': 'mochitest-3',
+                    'platforms': [
+                        'Ubuntu', '10.6', '10.8', 'Windows XP', 'Windows 7', 'Windows 8'
+                    ]
+                }
+            ]
+        )
+
+        self.assertEquals(
+            parse_test_opts(''),
+            []
+        )
 
 
 if __name__ == '__main__':
     mozunit.main()
rename from testing/taskcluster/taskcluster_graph/commit_parser.py
rename to taskcluster/taskgraph/util/legacy_commit_parser.py
--- a/testing/taskcluster/taskcluster_graph/commit_parser.py
+++ b/taskcluster/taskgraph/util/legacy_commit_parser.py
@@ -1,29 +1,86 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+# This commit parser is used by the legacy kind; once that is gone, it can be
+# removed.
 
 import argparse
 import copy
 import re
 import shlex
-from try_test_parser import parse_test_opts
 
 TRY_DELIMITER = 'try:'
 TEST_CHUNK_SUFFIX = re.compile('(.*)-([0-9]+)$')
 
 # The build type aliases are very cryptic and only used in try flags these are
 # mappings from the single char alias to a longer more recognizable form.
 BUILD_TYPE_ALIASES = {
     'o': 'opt',
     'd': 'debug'
 }
 
+def parse_test_opts(input_str):
+    '''Test argument parsing is surprisingly complicated with the "restrictions"
+    logic this function is responsible for parsing this out into a easier to
+    work with structure like { test: '..', platforms: ['..'] }'''
+
+    # Final results which we will return.
+    tests = []
+
+    cur_test = {}
+    token = ''
+    in_platforms = False
+
+    def add_test(value):
+        cur_test['test'] = value.strip()
+        tests.insert(0, cur_test)
+
+    def add_platform(value):
+        # Ensure platforms exists...
+        cur_test['platforms'] = cur_test.get('platforms', [])
+        cur_test['platforms'].insert(0, value.strip())
+
+    # This might be somewhat confusing but we parse the string _backwards_ so
+    # there is no ambiguity over what state we are in.
+    for char in reversed(input_str):
+        # , indicates exiting a state
+        if char == ',':
+            # Exit a particular platform.
+            if in_platforms:
+                add_platform(token)
+
+            # Exit a particular test.
+            else:
+                add_test(token)
+                cur_test = {}
+
+            # Token must always be reset after we exit a state
+            token = ''
+        elif char == '[':
+            # Exiting platform state entering test state.
+            add_platform(token)
+            token = ''
+            in_platforms = False
+        elif char == ']':
+            # Entering platform state.
+            in_platforms = True
+        else:
+            # Accumulator.
+            token = char + token
+
+    # Handle any left over tokens.
+    if token:
+        add_test(token)
+
+    return tests
+
+
 def escape_whitespace_in_brackets(input_str):
     '''
     In tests you may restrict them by platform [] inside of the brackets
     whitespace may occur this is typically invalid shell syntax so we escape it
     with backslash sequences    .
     '''
     result = ""
     in_brackets = False
deleted file mode 100644
--- a/testing/taskcluster/taskcluster_graph/try_test_parser.py
+++ /dev/null
@@ -1,58 +0,0 @@
-def parse_test_opts(input_str):
-    '''
-    Test argument parsing is surprisingly complicated with the "restrictions"
-    logic this function is responsible for parsing this out into a easier to
-    work with structure like { test: '..', platforms: ['..'] }
-    '''
-
-    # Final results which we will return.
-    tests = []
-
-    cur_test = {}
-    token = ''
-    in_platforms = False
-
-    def add_test(value):
-        cur_test['test'] = value.strip()
-        tests.insert(0, cur_test)
-
-    def add_platform(value):
-        # Ensure platforms exists...
-        cur_test['platforms'] = cur_test.get('platforms', [])
-        cur_test['platforms'].insert(0, value.strip())
-
-    # This might be somewhat confusing but we parse the string _backwards_ so
-    # there is no ambiguity over what state we are in.
-    for char in reversed(input_str):
-
-        # , indicates exiting a state
-        if char == ',':
-
-            # Exit a particular platform.
-            if in_platforms:
-                add_platform(token)
-
-            # Exit a particular test.
-            else:
-                add_test(token)
-                cur_test = {}
-
-            # Token must always be reset after we exit a state
-            token = ''
-        elif char == '[':
-            # Exiting platform state entering test state.
-            add_platform(token)
-            token = ''
-            in_platforms = False
-        elif char == ']':
-            # Entering platform state.
-            in_platforms = True
-        else:
-            # Accumulator.
-            token = char + token
-
-    # Handle any left over tokens.
-    if token:
-        add_test(token)
-
-    return tests
deleted file mode 100755
--- a/testing/taskcluster/tests/test_try_test_parser.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import unittest
-import mozunit
-from taskcluster_graph.try_test_parser import parse_test_opts
-
-class TryTestParserTest(unittest.TestCase):
-
-    def test_parse_opts_valid(self):
-        self.assertEquals(
-            parse_test_opts('all[Mulet Linux]'),
-            [{ 'test': 'all', 'platforms': ['Mulet Linux'] }]
-        )
-
-        self.assertEquals(
-            parse_test_opts('all[Amazing, Foobar woot,yeah]'),
-            [{ 'test': 'all', 'platforms': ['Amazing', 'Foobar woot', 'yeah'] }]
-        )
-
-        self.assertEquals(
-            parse_test_opts('a,b, c'),
-            [
-                { 'test': 'a' },
-                { 'test': 'b' },
-                { 'test': 'c' },
-            ]
-        )
-        self.assertEquals(
-            parse_test_opts('woot, bar[b], baz, qux[ z ],a'),
-            [
-                { 'test': 'woot' },
-                { 'test': 'bar', 'platforms': ['b'] },
-                { 'test': 'baz' },
-                { 'test': 'qux', 'platforms': ['z'] },
-                { 'test': 'a' }
-            ]
-        )
-
-        self.assertEquals(
-            parse_test_opts('mochitest-3[Ubuntu,10.6,10.8,Windows XP,Windows 7,Windows 8]'),
-            [
-                {
-                    'test': 'mochitest-3',
-                    'platforms': [
-                        'Ubuntu', '10.6', '10.8', 'Windows XP', 'Windows 7', 'Windows 8'
-                    ]
-                }
-            ]
-        )
-
-        self.assertEquals(
-            parse_test_opts(''),
-            []
-        )
-
-if __name__ == '__main__':
-    mozunit.main()