--- a/layout/tools/reftest/remotereftest.py
+++ b/layout/tools/reftest/remotereftest.py
@@ -138,22 +138,22 @@ class ReftestServer:
self.automation.log.info("Failed to shutdown server at %s" %
self.shutdownURL)
traceback.print_exc()
self._process.kill()
class RemoteReftest(RefTest):
use_marionette = False
- parse_manifest = False
remoteApp = ''
resolver_cls = RemoteReftestResolver
def __init__(self, automation, devicemanager, options, scriptDir):
- RefTest.__init__(self)
+ RefTest.__init__(self, options.suite)
+ self.run_by_manifest = False
self.automation = automation
self._devicemanager = devicemanager
self.scriptDir = scriptDir
self.remoteApp = options.app
self.remoteProfile = options.remoteProfile
self.remoteTestRoot = options.remoteTestRoot
self.remoteLogFile = options.remoteLogFile
self.remoteCache = os.path.join(options.remoteTestRoot, "cache/")
@@ -275,21 +275,16 @@ class RemoteReftest(RefTest):
# move necko cache to a location that can be cleaned up
prefs["browser.cache.disk.parent_directory"] = self.remoteCache
prefs["layout.css.devPixelsPerPx"] = "1.0"
# Because Fennec is a little wacky (see bug 1156817) we need to load the
# reftest pages at 1.0 zoom, rather than zooming to fit the CSS viewport.
prefs["apz.allow_zooming"] = False
- if options.totalChunks:
- prefs['reftest.totalChunks'] = options.totalChunks
- if options.thisChunk:
- prefs['reftest.thisChunk'] = options.thisChunk
-
# Set the extra prefs.
profile.set_preferences(prefs)
try:
self._devicemanager.pushDir(profileDir, options.remoteProfile)
self._devicemanager.chmodDir(options.remoteProfile)
except mozdevice.DMError:
print "Automation Error: Failed to copy profiledir to device"
@@ -356,17 +351,17 @@ class RemoteReftest(RefTest):
cmdargs,
utilityPath=options.utilityPath,
xrePath=options.xrePath,
debuggerInfo=debuggerInfo,
symbolsPath=symbolsPath,
timeout=timeout)
self.cleanup(profile.profile)
- return status, self.outputHandler.results
+ return status
def cleanup(self, profileDir):
# Pull results back from device
if self.remoteLogFile and \
self._devicemanager.fileExists(self.remoteLogFile):
self._devicemanager.getFile(self.remoteLogFile, self.localLogName)
else:
print "WARNING: Unable to retrieve log file (%s) from remote " \
--- a/layout/tools/reftest/runreftest.py
+++ b/layout/tools/reftest/runreftest.py
@@ -1,29 +1,29 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Runs the reftest test harness.
"""
-import collections
import copy
import json
import multiprocessing
import os
import platform
import re
import shutil
import signal
import subprocess
import sys
import tempfile
import threading
+from collections import defaultdict
from datetime import datetime, timedelta
SCRIPT_DIRECTORY = os.path.abspath(
os.path.realpath(os.path.dirname(__file__)))
if SCRIPT_DIRECTORY not in sys.path:
sys.path.insert(0, SCRIPT_DIRECTORY)
import mozcrash
@@ -222,28 +222,32 @@ class ReftestResolver(object):
manifests[key] = None
else:
manifests[key] = "|".join(list(manifests[key]))
return manifests
class RefTest(object):
oldcwd = os.getcwd()
- parse_manifest = True
resolver_cls = ReftestResolver
use_marionette = True
- def __init__(self):
+ def __init__(self, suite):
update_mozinfo()
self.lastTestSeen = None
self.haveDumpedScreen = False
self.resolver = self.resolver_cls()
self.log = None
+ self.outputHandler = None
self.testDumpFile = os.path.join(tempfile.gettempdir(), 'reftests.json')
+ self.run_by_manifest = True
+ if suite in ('crashtest', 'jstestbrowser'):
+ self.run_by_manifest = False
+
def _populate_logger(self, options):
if self.log:
return
self.log = getattr(options, 'log', None)
if self.log:
return
@@ -302,16 +306,22 @@ class RefTest(object):
prefs['reftest.suite'] = options.suite
# Unconditionally update the e10s pref.
if options.e10s:
prefs['browser.tabs.remote.autostart'] = True
else:
prefs['browser.tabs.remote.autostart'] = False
+ if not self.run_by_manifest:
+ if options.totalChunks:
+ prefs['reftest.totalChunks'] = options.totalChunks
+ if options.thisChunk:
+ prefs['reftest.thisChunk'] = options.thisChunk
+
# Bug 1262954: For winXP + e10s disable acceleration
if platform.system() in ("Windows", "Microsoft") and \
'5.1' in platform.version() and options.e10s:
prefs['layers.acceleration.disabled'] = True
sandbox_whitelist_paths = options.sandboxReadWhitelist
if (platform.system() == "Linux" or
platform.system() in ("Windows", "Microsoft")):
@@ -518,16 +528,17 @@ class RefTest(object):
self.log.info('::: Test verification %s' % finalResult)
self.log.info(':::')
return result
def runTests(self, tests, options, cmdargs=None):
cmdargs = cmdargs or []
self._populate_logger(options)
+ self.outputHandler = OutputHandler(self.log, options.utilityPath, options.symbolsPath)
if options.cleanupCrashes:
mozcrash.cleanup_pending_crash_reports()
manifests = self.resolver.resolveManifests(options, tests)
if options.filter:
manifests[""] = options.filter
@@ -589,17 +600,17 @@ class RefTest(object):
# have to worry about races between the needs-focus tests *actually*
# needing focus and the dummy windows in the non-needs-focus tests
# trying to focus themselves.
focusThread = ReftestThread(perProcessArgs[0])
focusThread.start()
focusThread.join()
# Output the summaries that the ReftestThread filters suppressed.
- summaryObjects = [collections.defaultdict(int) for s in summaryLines]
+ summaryObjects = [defaultdict(int) for s in summaryLines]
for t in threads:
for (summaryObj, (text, categories)) in zip(summaryObjects, summaryLines):
threadMatches = t.summaryMatches[text]
for (attribute, description) in categories:
amount = int(
threadMatches.group(attribute) if threadMatches else 0)
summaryObj[attribute] += amount
amount = int(
@@ -663,16 +674,17 @@ class RefTest(object):
process.kill()
def runApp(self, options, cmdargs=None, timeout=None, debuggerInfo=None,
symbolsPath=None, valgrindPath=None, valgrindArgs=None,
valgrindSuppFiles=None, **profileArgs):
if cmdargs is None:
cmdargs = []
+ cmdargs = cmdargs[:]
if self.use_marionette:
cmdargs.append('-marionette')
binary = options.app
profile = self.createReftestProfile(options, **profileArgs)
# browser environment
@@ -695,29 +707,27 @@ class RefTest(object):
if message['action'] == 'test_start':
if " " in message['test']:
self.lastTestSeen = message['test'].split(" ")[0]
else:
self.lastTestSeen = message['test']
self.log.add_handler(record_last_test)
- outputHandler = OutputHandler(self.log, options.utilityPath, symbolsPath=symbolsPath)
-
kp_kwargs = {
'kill_on_timeout': False,
'cwd': SCRIPT_DIRECTORY,
'onTimeout': [timeoutHandler],
- 'processOutputLine': [outputHandler],
+ 'processOutputLine': [self.outputHandler],
}
if mozinfo.isWin:
# Prevents log interleaving on Windows at the expense of losing
# true log order. See bug 798300 and bug 1324961 for more details.
- kp_kwargs['processStderrLine'] = [outputHandler]
+ kp_kwargs['processStderrLine'] = [self.outputHandler]
if interactive:
# If an interactive debugger is attached,
# don't use timeouts, and don't capture ctrl-c.
timeout = None
signal.signal(signal.SIGINT, lambda sigid, frame: None)
runner_cls = mozrunner.runners.get(mozinfo.info.get('appname', 'firefox'),
@@ -727,17 +737,17 @@ class RefTest(object):
process_class=mozprocess.ProcessHandlerMixin,
cmdargs=cmdargs,
env=env,
process_args=kp_kwargs)
runner.start(debug_args=debug_args,
interactive=interactive,
outputTimeout=timeout)
proc = runner.process_handler
- outputHandler.proc_name = 'GECKO({})'.format(proc.pid)
+ self.outputHandler.proc_name = 'GECKO({})'.format(proc.pid)
# Used to defer a possible IOError exception from Marionette
marionette_exception = None
if self.use_marionette:
marionette_args = {
'socket_timeout': options.marionette_socket_timeout,
'startup_timeout': options.marionette_startup_timeout,
@@ -764,48 +774,48 @@ class RefTest(object):
# wrong with the process, like a crash or the socket is no
# longer open. We defer raising this specific error so that
# post-test checks for leaks and crashes are performed and
# reported first.
marionette_exception = sys.exc_info()
status = runner.wait()
runner.process_handler = None
- outputHandler.proc_name = None
+ self.outputHandler.proc_name = None
if status:
msg = "TEST-UNEXPECTED-FAIL | %s | application terminated with exit code %s" % \
(self.lastTestSeen, status)
# use process_output so message is logged verbatim
self.log.process_output(None, msg)
crashed = mozcrash.log_crashes(self.log, os.path.join(profile.profile, 'minidumps'),
- symbolsPath, test=self.lastTestSeen)
+ options.symbolsPath, test=self.lastTestSeen)
if not status and crashed:
status = 1
runner.cleanup()
self.cleanup(profile.profile)
if marionette_exception is not None:
exc, value, tb = marionette_exception
raise exc, value, tb
self.log.info("Process mode: {}".format('e10s' if options.e10s else 'non-e10s'))
- return status, outputHandler.results
+ return status
def getActiveTests(self, manifests, options, testDumpFile=None):
# These prefs will cause reftest.jsm to parse the manifests,
# dump the resulting tests to a file, and exit.
prefs = {
'reftest.manifests': json.dumps(manifests),
'reftest.manifests.dumpTests': testDumpFile or self.testDumpFile,
}
- cmdargs = [] # ['-headless']
- status, _ = self.runApp(options, cmdargs=cmdargs, prefs=prefs)
+ cmdargs = []
+ self.runApp(options, cmdargs=cmdargs, prefs=prefs)
with open(self.testDumpFile, 'r') as fh:
tests = json.load(fh)
if os.path.isfile(self.testDumpFile):
mozfile.remove(self.testDumpFile)
for test in tests:
@@ -823,50 +833,61 @@ class RefTest(object):
return tests
def runSerialTests(self, manifests, options, cmdargs=None):
debuggerInfo = None
if options.debugger:
debuggerInfo = mozdebug.get_debugger_info(options.debugger, options.debuggerArgs,
options.debuggerInteractive)
- tests = None
- if self.parse_manifest:
- tests = self.getActiveTests(manifests, options)
-
- ids = [t['identifier'] for t in tests]
- self.log.suite_start(ids, name=options.suite)
+ def run(**kwargs):
+ status = self.runApp(
+ options,
+ manifests=manifests,
+ cmdargs=cmdargs,
+ # We generally want the JS harness or marionette
+ # to handle timeouts if they can.
+ # The default JS harness timeout is currently
+ # 300 seconds (default options.timeout).
+ # The default Marionette socket timeout is
+ # currently 360 seconds.
+ # Give the JS harness extra time to deal with
+ # its own timeouts and try to usually exceed
+ # the 360 second marionette socket timeout.
+ # See bug 479518 and bug 1414063.
+ timeout=options.timeout + 70.0,
+ debuggerInfo=debuggerInfo,
+ **kwargs)
- status, results = self.runApp(
- options,
- tests=tests,
- manifests=manifests,
- cmdargs=cmdargs,
- # We generally want the JS harness or marionette
- # to handle timeouts if they can.
- # The default JS harness timeout is currently
- # 300 seconds (default options.timeout).
- # The default Marionette socket timeout is
- # currently 360 seconds.
- # Give the JS harness extra time to deal with
- # its own timeouts and try to usually exceed
- # the 360 second marionette socket timeout.
- # See bug 479518 and bug 1414063.
- timeout=options.timeout + 70.0,
- symbolsPath=options.symbolsPath,
- debuggerInfo=debuggerInfo
- )
- mozleak.process_leak_log(self.leakLogFile,
- leak_thresholds=options.leakThresholds,
- stack_fixer=get_stack_fixer_function(options.utilityPath,
- options.symbolsPath))
+ mozleak.process_leak_log(self.leakLogFile,
+ leak_thresholds=options.leakThresholds,
+ stack_fixer=get_stack_fixer_function(options.utilityPath,
+ options.symbolsPath))
+ return status
+
+ if not self.run_by_manifest:
+ return run()
- if self.parse_manifest:
- self.log.suite_end(extra={'results': results})
- return status
+ tests = self.getActiveTests(manifests, options)
+ tests_by_manifest = defaultdict(list)
+ ids_by_manifest = defaultdict(list)
+ for t in tests:
+ tests_by_manifest[t['manifest']].append(t)
+ ids_by_manifest[t['manifest']].append(t['identifier'])
+
+ self.log.suite_start(ids_by_manifest, name=options.suite)
+
+ overall = 0
+ for manifest, tests in tests_by_manifest.items():
+ self.log.info("Running tests in {}".format(manifest))
+ status = run(tests=tests)
+ overall = overall or status
+
+ self.log.suite_end(extra={'results': self.outputHandler.results})
+ return overall
def copyExtraFilesToProfile(self, options, profile):
"Copy extra files or dirs specified on the command line to the testing profile."
profileDir = profile.profile
for f in options.extraProfileFiles:
abspath = self.getFullPath(f)
if os.path.isfile(abspath):
if os.path.basename(abspath) == 'user.js':
@@ -884,17 +905,17 @@ class RefTest(object):
shutil.copytree(abspath, dest)
else:
self.log.warning(
"runreftest.py | Failed to copy %s to profile" % abspath)
continue
def run_test_harness(parser, options):
- reftest = RefTest()
+ reftest = RefTest(options.suite)
parser.validate(options, reftest)
# We have to validate options.app here for the case when the mach
# command is able to find it after argument parsing. This can happen
# when running from a tests.zip.
if not options.app:
parser.error("could not find the application path, --appname must be specified")
