--- a/dom/bindings/mach_commands.py
+++ b/dom/bindings/mach_commands.py
@@ -9,16 +9,17 @@ import sys
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
 from mozbuild.base import MachCommandBase
+from mozbuild.util import mkdir
 
 def get_test_parser():
     import runtests
     return runtests.get_parser
 
 @CommandProvider
 class WebIDLProvider(MachCommandBase):
     @Command('webidl-example', category='misc',
@@ -33,16 +34,20 @@ class WebIDLProvider(MachCommandBase):
             manager.generate_example_files(i)
 
     @Command('webidl-parser-test', category='testing', parser=get_test_parser,
              description='Run WebIDL tests (Interface Browser parser).')
     def webidl_test(self, **kwargs):
         sys.path.insert(0, os.path.join(self.topsrcdir, 'other-licenses',
                         'ply'))
 
+        # Ensure the topobjdir exists. On a Taskcluster test run there won't be
+        # an objdir yet.
+        mkdir(self.topobjdir)
+
         # Make sure we drop our cached grammar bits in the objdir, not
         # wherever we happen to be running from.
         os.chdir(self.topobjdir)
 
         if kwargs["verbose"] is None:
             kwargs["verbose"] = False
 
         # Now we're going to create the cached grammar file in the

--- a/dom/bindings/parser/runtests.py
+++ b/dom/bindings/parser/runtests.py
@@ -83,16 +83,17 @@ def run_tests(tests, verbose):
         print 'Result summary:'
         print 'Successful: %d' % all_passed
         print 'Unexpected: %d' % \
                 sum(len(failures) for _, failures in failed_tests)
         for test, failures in failed_tests:
             print '%s:' % test
             for failure in failures:
                 print 'TEST-UNEXPECTED-FAIL | %s' % failure
+    return 1 if failed_tests else 0
 
 def get_parser():
     usage = """%(prog)s [OPTIONS] [TESTS]
                Where TESTS are relative to the tests directory."""
     parser = argparse.ArgumentParser(usage=usage)
     parser.add_argument('-q', '--quiet', action='store_false', dest='verbose',
                         help="Don't print passing tests.", default=None)
     parser.add_argument('-v', '--verbose', action='store_true', dest='verbose',
@@ -100,9 +101,14 @@ def get_parser():
     parser.add_argument('tests', nargs="*", help="Tests to run")
     return parser
 
 if __name__ == '__main__':
     parser = get_parser()
     args = parser.parse_args()
     if args.verbose is None:
         args.verbose = True
-    run_tests(args.tests, verbose=args.verbose)
+
+    # Make sure the current directory is in the python path so we can cache the
+    # result of the webidlyacc.py generation.
+    sys.path.append('.')
+
+    sys.exit(run_tests(args.tests, verbose=args.verbose))

--- a/dom/bindings/test/Makefile.in
+++ b/dom/bindings/test/Makefile.in
@@ -10,12 +10,8 @@ include ../webidlsrcs.mk
 # TODO Update this variable in backend.mk.
 CPPSRCS += $(addprefix ../,$(test_sources))
 
 # Include rules.mk before any of our targets so our first target is coming from
 # rules.mk and running make with no target in this dir does the right thing.
 include $(topsrcdir)/config/rules.mk
 
 endif
-
-check::
-	PYTHONDONTWRITEBYTECODE=1 $(PYTHON) $(topsrcdir)/config/pythonpath.py \
-	  $(PLY_INCLUDE) $(srcdir)/../parser/runtests.py

--- a/taskcluster/ci/source-check/kind.yml
+++ b/taskcluster/ci/source-check/kind.yml
@@ -8,8 +8,9 @@ transforms:
    - taskgraph.transforms.build_attrs:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 jobs-from:
     - python-tests.yml
     - mozlint.yml
     - doc.yml
+    - webidl.yml

new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/source-check/webidl.yml
@@ -0,0 +1,24 @@
+webidl-test/opt:
+    description: WebIDL parser tests
+    treeherder:
+        symbol: Wp
+        kind: test
+        tier: 1
+        platform: lint/opt
+    worker-type: aws-provisioner-v1/b2gtest
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: mach
+        mach: webidl-parser-test --verbose
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            - 'dom/bindings/parser/runtests.py'
+            - 'dom/bindings/parser/WebIDL.py'
+            - 'dom/bindings/parser/tests/**'
+            - 'other-licenses/ply/**'