Bug 1356227 - Add support for excluding paths when doing a wpt sync, r=ato
MozReview-Commit-ID: 6zadFuRDmVD
--- a/testing/web-platform/harness/wptrunner/update/sync.py
+++ b/testing/web-platform/harness/wptrunner/update/sync.py
@@ -1,13 +1,15 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import fnmatch
import os
+import re
import shutil
import sys
import uuid
from .. import testloader
from base import Step, StepRunner
from tree import Commit
@@ -41,28 +43,43 @@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NO
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
"""
-def copy_wpt_tree(tree, dest):
+def copy_wpt_tree(tree, dest, excludes=None, includes=None):
"""Copy the working copy of a Tree to a destination directory.
:param tree: The Tree to copy.
:param dest: The destination directory"""
if os.path.exists(dest):
assert os.path.isdir(dest)
shutil.rmtree(dest)
+
os.mkdir(dest)
+ if excludes is None:
+ excludes = []
+
+ excludes = [re.compile(fnmatch.translate(item)) for item in excludes]
+
+ if includes is None:
+ includes = []
+
+ includes = [re.compile(fnmatch.translate(item)) for item in includes]
+
for tree_path in tree.paths():
+ if (any(item.match(tree_path) for item in excludes) and
+ not any(item.match(tree_path) for item in includes)):
+ continue
+
source_path = os.path.join(tree.root, tree_path)
dest_path = os.path.join(dest, tree_path)
dest_dir = os.path.split(dest_path)[0]
if not os.path.isdir(source_path):
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
shutil.copy2(source_path, dest_path)
@@ -78,16 +95,17 @@ def copy_wpt_tree(tree, dest):
def add_license(dest):
"""Write the bsd license string to a LICENSE file.
:param dest: Directory in which to place the LICENSE file."""
with open(os.path.join(dest, "LICENSE"), "w") as f:
f.write(bsd_license)
+
class UpdateCheckout(Step):
"""Pull changes from upstream into the local sync tree."""
provides = ["local_branch"]
def create(self, state):
sync_tree = state.sync_tree
state.local_branch = uuid.uuid4().hex
@@ -141,17 +159,19 @@ class UpdateManifest(Step):
manifest.write(state.test_manifest, state.manifest_path)
class CopyWorkTree(Step):
"""Copy the sync tree over to the destination in the local tree"""
def create(self, state):
copy_wpt_tree(state.sync_tree,
- state.tests_path)
+ state.tests_path,
+ excludes=state.path_excludes,
+ includes=state.path_includes)
class CreateSyncPatch(Step):
"""Add the updated test files to a commit/patch in the local tree."""
def create(self, state):
if state.no_patch:
return
--- a/testing/web-platform/harness/wptrunner/update/tree.py
+++ b/testing/web-platform/harness/wptrunner/update/tree.py
@@ -311,19 +311,18 @@ class GitTree(object):
"""List paths in the tree"""
repo_paths = [self.root] + [os.path.join(self.root, path)
for path in self.submodules()]
rv = []
for repo_path in repo_paths:
paths = vcs.git("ls-tree", "-r", "--name-only", "HEAD", repo=repo_path).split("\n")
- rel_path = os.path.relpath(repo_path, self.root)
- rv.extend(os.path.join(rel_path, item.strip()) for item in paths if item.strip())
-
+ rv.extend(os.path.relpath(os.path.join(repo_path, item), self.root) for item in paths
+ if item.strip())
return rv
def submodules(self):
"""List submodule directories"""
output = self.git("submodule", "status", "--recursive")
rv = []
for line in output.split("\n"):
line = line.strip()
--- a/testing/web-platform/harness/wptrunner/update/update.py
+++ b/testing/web-platform/harness/wptrunner/update/update.py
@@ -72,16 +72,18 @@ class SyncFromUpstream(Step):
state.sync_tree = GitTree(root=state.sync["path"])
kwargs = state.kwargs
with state.push(["sync", "paths", "metadata_path", "tests_path", "local_tree",
"sync_tree"]):
state.target_rev = kwargs["rev"]
state.no_patch = kwargs["no_patch"]
state.suite_name = kwargs["suite_name"]
+ state.path_excludes = kwargs["exclude"]
+ state.path_includes = kwargs["include"]
runner = SyncFromUpstreamRunner(self.logger, state)
runner.run()
class UpdateMetadata(Step):
"""Update the expectation metadata from a set of run logs"""
def create(self, state):
--- a/testing/web-platform/harness/wptrunner/wptcommandline.py
+++ b/testing/web-platform/harness/wptrunner/wptcommandline.py
@@ -398,16 +398,20 @@ def create_parser_update(product_choices
parser.add_argument("--rev", action="store", help="Revision to sync to")
parser.add_argument("--no-patch", action="store_true",
help="Don't create an mq patch or git commit containing the changes.")
parser.add_argument("--sync", dest="sync", action="store_true", default=False,
help="Sync the tests with the latest from upstream")
parser.add_argument("--ignore-existing", action="store_true", help="When updating test results only consider results from the logfiles provided, not existing expectations.")
parser.add_argument("--continue", action="store_true", help="Continue a previously started run of the update script")
parser.add_argument("--abort", action="store_true", help="Clear state from a previous incomplete run of the update script")
+ parser.add_argument("--exclude", action="store", nargs="*",
+ help="List of glob-style paths to exclude when syncing tests")
+ parser.add_argument("--include", action="store", nargs="*",
+ help="List of glob-style paths to include which would otherwise be excluded when syncing tests")
# Should make this required iff run=logfile
parser.add_argument("run_log", nargs="*", type=abs_path,
help="Log file from run of tests")
commandline.add_logging_group(parser)
return parser
def create_parser_reduce(product_choices=None):
--- a/testing/web-platform/mach_commands.py
+++ b/testing/web-platform/mach_commands.py
@@ -90,16 +90,23 @@ class WebPlatformTestsUpdater(MozbuildOb
import update
from update import updatecommandline
if kwargs["config"] is None:
kwargs["config"] = os.path.join(self.topsrcdir, 'testing', 'web-platform', 'wptrunner.ini')
if kwargs["product"] is None:
kwargs["product"] = "firefox"
+ if kwargs["sync"]:
+ if not kwargs["exclude"]:
+ kwargs["exclude"] = ["css/*"]
+ if not kwargs["include"]:
+ kwargs["include"] = ["css/css-timing-1/*", "css/css-animations-1/*", "css/css-transitions-1/*"]
+
+
updatecommandline.check_args(kwargs)
logger = update.setup_logging(kwargs, {"mach": sys.stdout})
try:
update.run_update(logger, **kwargs)
except Exception:
import pdb
import traceback
@@ -239,17 +246,16 @@ testing/web-platform/tests for tests tha
if proc:
proc.wait()
class WPTManifestUpdater(MozbuildObject):
def run_update(self, check_clean=False, **kwargs):
import manifestupdate
from wptrunner import wptlogging
-
logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
manifestupdate.update(logger, wpt_dir, check_clean)
def create_parser_wpt():
from wptrunner import wptcommandline
return wptcommandline.create_parser(["firefox"])