--- a/testing/mochitest/jetpack-package-harness.js
+++ b/testing/mochitest/jetpack-package-harness.js
@@ -49,19 +49,25 @@ function testModule(require, { url, expe
       let suiteModule;
       try {
         dump("TEST-INFO: " + path + " | Loading test module\n");
         suiteModule = loaderModule.main(loader, "tests/" + path.substring(0, path.length - 3));
       }
       catch (e) {
         // If `Unsupported Application` error thrown during test,
         // skip the test suite
-        suiteModule = {
-          'test suite skipped': assert => assert.pass(e.message)
-        };
+        if (/Unsupported Application/.test(e.message)) {
+          suiteModule = {
+            'test suite skipped': assert => assert.pass(e.message)
+          };
+        } else {
+          suiteModule = {
+            'test suite failed to load': assert => assert.fail(`${e.message}: ${e.stack}`)
+          };
+        }
       }
 
       for (let name of Object.keys(suiteModule).sort()) {
         if (NOT_TESTS.indexOf(name) != -1)
           continue;
 
         tests.push({
           setup: suiteModule.setup,