Bug 1273986 - Execute reftests in random order; r?dbaron
When tests are executed in the same order every time, undetected
cross-test dependencies can be introduced, leading to unexpected
passes or failures.
This commit randomizes the execution of reftests in automation.
This should hopefully lead to more test failures due to previously
undetected ordering dependencies and possibly even race conditions.
While this may upset people who care about seeing fewer test
failures, running tests in random order is more indicative of real
life browser behavior and thus could lead to higher browser quality
and stability.
MozReview-Commit-ID: 2VTBvK4Onh7
--- a/testing/mozharness/configs/unittests/linux_unittest.py
+++ b/testing/mozharness/configs/unittests/linux_unittest.py
@@ -154,16 +154,17 @@ config = {
"reftest": {
"options": [
"--appname=%(binary_path)s",
"--utility-path=tests/bin",
"--extra-profile-file=tests/bin/plugins",
"--symbols-path=%(symbols_path)s",
"--log-raw=%(raw_log_file)s",
"--log-errorsummary=%(error_summary_file)s",
+ "--shuffle",
],
"run_filename": "runreftest.py",
"testsdir": "reftest"
},
"xpcshell": {
"options": [
"--symbols-path=%(symbols_path)s",
"--test-plugin-path=%(test_plugin_path)s",
--- a/testing/mozharness/configs/unittests/mac_unittest.py
+++ b/testing/mozharness/configs/unittests/mac_unittest.py
@@ -105,17 +105,18 @@ config = {
"run_filename": "runtestlist.py",
"testsdir": "mozmill"
},
"reftest": {
"options": [
"--appname=%(binary_path)s",
"--utility-path=tests/bin",
"--extra-profile-file=tests/bin/plugins",
- "--symbols-path=%(symbols_path)s"
+ "--symbols-path=%(symbols_path)s",
+ "--shuffle",
],
"run_filename": "runreftest.py",
"testsdir": "reftest"
},
"xpcshell": {
"options": [
"--symbols-path=%(symbols_path)s",
"--test-plugin-path=%(test_plugin_path)s",
--- a/testing/mozharness/configs/unittests/win_unittest.py
+++ b/testing/mozharness/configs/unittests/win_unittest.py
@@ -119,16 +119,17 @@ config = {
"reftest": {
"options": [
"--appname=%(binary_path)s",
"--utility-path=tests/bin",
"--extra-profile-file=tests/bin/plugins",
"--symbols-path=%(symbols_path)s",
"--log-raw=%(raw_log_file)s",
"--log-errorsummary=%(error_summary_file)s",
+ "--shuffle",
],
"run_filename": "runreftest.py",
"testsdir": "reftest"
},
"xpcshell": {
"options": [
"--symbols-path=%(symbols_path)s",
"--test-plugin-path=%(test_plugin_path)s",