Bug 1356227 - Add support for excluding paths when doing a wpt sync, r=ato draft
authorJames Graham <james@hoppipolla.co.uk>
Tue, 04 Apr 2017 13:53:43 +0100
changeset 562048 24819110bc9aa5b0b4281063c7cf981f707ad23a
parent 561911 819a666afddc804b6099ee1b3cff3a0fdf35ec15
child 624166 68408419814310fe72b307034d0456ae90adddf3
push id53949
push userbmo:james@hoppipolla.co.uk
push dateThu, 13 Apr 2017 13:15:40 +0000
reviewersato
bugs1356227
milestone55.0a1
Bug 1356227 - Add support for excluding paths when doing a wpt sync, r=ato MozReview-Commit-ID: 6zadFuRDmVD
testing/web-platform/harness/wptrunner/update/sync.py
testing/web-platform/harness/wptrunner/update/tree.py
testing/web-platform/harness/wptrunner/update/update.py
testing/web-platform/harness/wptrunner/wptcommandline.py
testing/web-platform/mach_commands.py
--- a/testing/web-platform/harness/wptrunner/update/sync.py
+++ b/testing/web-platform/harness/wptrunner/update/sync.py
@@ -1,13 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import fnmatch
 import os
+import re
 import shutil
 import sys
 import uuid
 
 from .. import testloader
 
 from base import Step, StepRunner
 from tree import Commit
@@ -41,28 +43,43 @@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NO
 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGE.
 """
 
 
-def copy_wpt_tree(tree, dest):
+def copy_wpt_tree(tree, dest, excludes=None, includes=None):
     """Copy the working copy of a Tree to a destination directory.
 
     :param tree: The Tree to copy.
     :param dest: The destination directory"""
     if os.path.exists(dest):
         assert os.path.isdir(dest)
 
     shutil.rmtree(dest)
+
     os.mkdir(dest)
 
+    if excludes is None:
+        excludes = []
+
+    excludes = [re.compile(fnmatch.translate(item)) for item in excludes]
+
+    if includes is None:
+        includes = []
+
+    includes = [re.compile(fnmatch.translate(item)) for item in includes]
+
     for tree_path in tree.paths():
+        if (any(item.match(tree_path) for item in excludes) and
+            not any(item.match(tree_path) for item in includes)):
+            continue
+
         source_path = os.path.join(tree.root, tree_path)
         dest_path = os.path.join(dest, tree_path)
 
         dest_dir = os.path.split(dest_path)[0]
         if not os.path.isdir(source_path):
             if not os.path.exists(dest_dir):
                 os.makedirs(dest_dir)
             shutil.copy2(source_path, dest_path)
@@ -78,16 +95,17 @@ def copy_wpt_tree(tree, dest):
 
 def add_license(dest):
     """Write the bsd license string to a LICENSE file.
 
     :param dest: Directory in which to place the LICENSE file."""
     with open(os.path.join(dest, "LICENSE"), "w") as f:
         f.write(bsd_license)
 
+
 class UpdateCheckout(Step):
     """Pull changes from upstream into the local sync tree."""
 
     provides = ["local_branch"]
 
     def create(self, state):
         sync_tree = state.sync_tree
         state.local_branch = uuid.uuid4().hex
@@ -141,17 +159,19 @@ class UpdateManifest(Step):
         manifest.write(state.test_manifest, state.manifest_path)
 
 
 class CopyWorkTree(Step):
     """Copy the sync tree over to the destination in the local tree"""
 
     def create(self, state):
         copy_wpt_tree(state.sync_tree,
-                      state.tests_path)
+                      state.tests_path,
+                      excludes=state.path_excludes,
+                      includes=state.path_includes)
 
 
 class CreateSyncPatch(Step):
     """Add the updated test files to a commit/patch in the local tree."""
 
     def create(self, state):
         if state.no_patch:
             return
--- a/testing/web-platform/harness/wptrunner/update/tree.py
+++ b/testing/web-platform/harness/wptrunner/update/tree.py
@@ -311,19 +311,18 @@ class GitTree(object):
         """List paths in the tree"""
         repo_paths = [self.root] +  [os.path.join(self.root, path)
                                      for path in self.submodules()]
 
         rv = []
 
         for repo_path in repo_paths:
             paths = vcs.git("ls-tree", "-r", "--name-only", "HEAD", repo=repo_path).split("\n")
-            rel_path = os.path.relpath(repo_path, self.root)
-            rv.extend(os.path.join(rel_path, item.strip()) for item in paths if item.strip())
-
+            rv.extend(os.path.relpath(os.path.join(repo_path, item), self.root) for item in paths
+                      if item.strip())
         return rv
 
     def submodules(self):
         """List submodule directories"""
         output = self.git("submodule", "status", "--recursive")
         rv = []
         for line in output.split("\n"):
             line = line.strip()
--- a/testing/web-platform/harness/wptrunner/update/update.py
+++ b/testing/web-platform/harness/wptrunner/update/update.py
@@ -72,16 +72,18 @@ class SyncFromUpstream(Step):
             state.sync_tree = GitTree(root=state.sync["path"])
 
         kwargs = state.kwargs
         with state.push(["sync", "paths", "metadata_path", "tests_path", "local_tree",
                          "sync_tree"]):
             state.target_rev = kwargs["rev"]
             state.no_patch = kwargs["no_patch"]
             state.suite_name = kwargs["suite_name"]
+            state.path_excludes = kwargs["exclude"]
+            state.path_includes = kwargs["include"]
             runner = SyncFromUpstreamRunner(self.logger, state)
             runner.run()
 
 
 class UpdateMetadata(Step):
     """Update the expectation metadata from a set of run logs"""
 
     def create(self, state):
--- a/testing/web-platform/harness/wptrunner/wptcommandline.py
+++ b/testing/web-platform/harness/wptrunner/wptcommandline.py
@@ -398,16 +398,20 @@ def create_parser_update(product_choices
     parser.add_argument("--rev", action="store", help="Revision to sync to")
     parser.add_argument("--no-patch", action="store_true",
                         help="Don't create an mq patch or git commit containing the changes.")
     parser.add_argument("--sync", dest="sync", action="store_true", default=False,
                         help="Sync the tests with the latest from upstream")
     parser.add_argument("--ignore-existing", action="store_true", help="When updating test results only consider results from the logfiles provided, not existing expectations.")
     parser.add_argument("--continue", action="store_true", help="Continue a previously started run of the update script")
     parser.add_argument("--abort", action="store_true", help="Clear state from a previous incomplete run of the update script")
+    parser.add_argument("--exclude", action="store", nargs="*",
+                        help="List of glob-style paths to exclude when syncing tests")
+    parser.add_argument("--include", action="store", nargs="*",
+                        help="List of glob-style paths to include which would otherwise be excluded when syncing tests")
     # Should make this required iff run=logfile
     parser.add_argument("run_log", nargs="*", type=abs_path,
                         help="Log file from run of tests")
     commandline.add_logging_group(parser)
     return parser
 
 
 def create_parser_reduce(product_choices=None):
--- a/testing/web-platform/mach_commands.py
+++ b/testing/web-platform/mach_commands.py
@@ -90,16 +90,23 @@ class WebPlatformTestsUpdater(MozbuildOb
         import update
         from update import updatecommandline
 
         if kwargs["config"] is None:
             kwargs["config"] = os.path.join(self.topsrcdir, 'testing', 'web-platform', 'wptrunner.ini')
         if kwargs["product"] is None:
             kwargs["product"] = "firefox"
 
+        if kwargs["sync"]:
+            if not kwargs["exclude"]:
+                kwargs["exclude"] = ["css/*"]
+            if not kwargs["include"]:
+                kwargs["include"] = ["css/css-timing-1/*", "css/css-animations-1/*", "css/css-transitions-1/*"]
+
+
         updatecommandline.check_args(kwargs)
         logger = update.setup_logging(kwargs, {"mach": sys.stdout})
 
         try:
             update.run_update(logger, **kwargs)
         except Exception:
             import pdb
             import traceback
@@ -239,17 +246,16 @@ testing/web-platform/tests for tests tha
         if proc:
             proc.wait()
 
 
 class WPTManifestUpdater(MozbuildObject):
     def run_update(self, check_clean=False, **kwargs):
         import manifestupdate
         from wptrunner import wptlogging
-
         logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
         wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
         manifestupdate.update(logger, wpt_dir, check_clean)
 
 
 def create_parser_wpt():
     from wptrunner import wptcommandline
     return wptcommandline.create_parser(["firefox"])