--- a/testing/mozharness/configs/b2g/taskcluster_emulator_automation.py
+++ b/testing/mozharness/configs/b2g/taskcluster_emulator_automation.py
@@ -28,17 +28,18 @@ config = {
"run_file_names": {
"jsreftest": "runreftestb2g.py",
"mochitest": "runtestsb2g.py",
"mochitest-chrome": "runtestsb2g.py",
"reftest": "runreftestb2g.py",
"crashtest": "runreftestb2g.py",
"xpcshell": "runtestsb2g.py",
- "cppunittest": "remotecppunittests.py"
+ "cppunittest": "remotecppunittests.py",
+ "marionette": "runtests.py"
},
"suite_definitions": {
"cppunittest": {
"options": [
"--dm_trans=adb",
"--symbols-path=%(symbols_path)s",
"--xre-path=%(xre_path)s",
"--addEnv",
@@ -170,12 +171,25 @@ config = {
"--busybox=%(busybox)s",
"--total-chunks=%(total_chunks)s",
"--this-chunk=%(this_chunk)s",
"--log-raw=%(raw_log_file)s",
"--log-errorsummary=%(error_summary_file)s",
],
"run_filename": "runtestsb2g.py",
"testsdir": "xpcshell"
+ },
+ "marionette": {
+ "options": [
+ "--type=b2g",
+ "--log-raw=%(raw_log_file)s",
+ "--log-errorsummary=%(error_summary_file)s",
+ "--symbols-path=%(symbols_path)s",
+ "--logcat-dir=%(logcat_dir)s",
+ "--emulator=%(emulator)s",
+ "--homedir=%(homedir)s"
+ ],
+ "run_filename": "runtests.py",
+ "testsdir": "marionette/client/marionette"
}
},
"vcs_output_timeout": 1760,
}
\ No newline at end of file
--- a/testing/mozharness/configs/marionette/automation_emulator_config.py
+++ b/testing/mozharness/configs/marionette/automation_emulator_config.py
@@ -1,9 +1,11 @@
# This is a template config file for marionette production.
+# TODO: This could be removed after B2G ICS emulator buildbot builds is turned
+# off, Bug 1209180.
import os
HG_SHARE_BASE_DIR = "/builds/hg-shared"
config = {
# marionette options
"test_type": "b2g",
--- a/testing/mozharness/mozharness/mozilla/testing/unittest.py
+++ b/testing/mozharness/mozharness/mozilla/testing/unittest.py
@@ -4,17 +4,17 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
import os
import re
from mozharness.mozilla.testing.errors import TinderBoxPrintRe
-from mozharness.base.log import OutputParser, WARNING, INFO, CRITICAL
+from mozharness.base.log import OutputParser, WARNING, INFO, CRITICAL, ERROR
from mozharness.mozilla.buildbot import TBPL_WARNING, TBPL_FAILURE, TBPL_RETRY
from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE
SUITE_CATEGORIES = ['mochitest', 'reftest', 'xpcshell']
def tbox_print_summary(pass_count, fail_count, known_fail_count=None,
crashed=False, leaked=False):
@@ -45,44 +45,58 @@ def tbox_print_summary(pass_count, fail_
class TestSummaryOutputParserHelper(OutputParser):
def __init__(self, regex=re.compile(r'(passed|failed|todo): (\d+)'), **kwargs):
self.regex = regex
self.failed = 0
self.passed = 0
self.todo = 0
self.last_line = None
+ self.tbpl_status = TBPL_SUCCESS
+ self.worst_log_level = INFO
super(TestSummaryOutputParserHelper, self).__init__(**kwargs)
def parse_single_line(self, line):
super(TestSummaryOutputParserHelper, self).parse_single_line(line)
self.last_line = line
m = self.regex.search(line)
if m:
try:
setattr(self, m.group(1), int(m.group(2)))
except ValueError:
# ignore bad values
pass
- def evaluate_parser(self):
+ def evaluate_parser(self, return_code, success_codes=None):
+ if return_code == 0 and self.passed > 0 and self.failed == 0:
+ self.tbpl_status = TBPL_SUCCESS
+ elif return_code == 10 and self.failed > 0:
+ self.tbpl_status = TBPL_WARNING
+ else:
+ self.tbpl_status = TBPL_FAILURE
+ self.worst_log_level = ERROR
+
+ return (self.tbpl_status, self.worst_log_level)
+
+ def print_summary(self, suite_name):
# generate the TinderboxPrint line for TBPL
emphasize_fail_text = '<em class="testfail">%s</em>'
failed = "0"
if self.passed == 0 and self.failed == 0:
self.tsummary = emphasize_fail_text % "T-FAIL"
else:
if self.failed > 0:
failed = emphasize_fail_text % str(self.failed)
self.tsummary = "%d/%s/%d" % (self.passed, failed, self.todo)
- def print_summary(self, suite_name):
- self.evaluate_parser()
self.info("TinderboxPrint: %s: %s\n" % (suite_name, self.tsummary))
+ def append_tinderboxprint_line(self, suite_name):
+ self.print_summary(suite_name)
+
class DesktopUnittestOutputParser(OutputParser):
"""
A class that extends OutputParser such that it can parse the number of
passed/failed/todo tests from the output.
"""
def __init__(self, suite_category, **kwargs):
--- a/testing/mozharness/scripts/b2g_emulator_unittest.py
+++ b/testing/mozharness/scripts/b2g_emulator_unittest.py
@@ -18,21 +18,22 @@ from mozharness.base.log import ERROR, W
from mozharness.base.script import (
BaseScript,
PreScriptAction,
)
from mozharness.base.vcs.vcsbase import VCSMixin
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
from mozharness.mozilla.testing.errors import LogcatErrorList
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
+from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
from mozharness.mozilla.buildbot import TBPL_SUCCESS
class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin):
- test_suites = ('jsreftest', 'reftest', 'mochitest', 'mochitest-chrome', 'xpcshell', 'crashtest', 'cppunittest')
+ test_suites = ('jsreftest', 'reftest', 'mochitest', 'mochitest-chrome', 'xpcshell', 'crashtest', 'cppunittest', 'marionette')
config_options = [[
["--type"],
{"action": "store",
"dest": "test_type",
"default": "browser",
"help": "The type of tests to run",
}
], [
@@ -185,16 +186,18 @@ class B2GEmulatorTest(TestingMixin, VCSM
dirs['abs_crashtest_dir'] = os.path.join(
dirs['abs_test_install_dir'], 'reftest')
dirs['abs_jsreftest_dir'] = os.path.join(
dirs['abs_test_install_dir'], 'reftest')
dirs['abs_xpcshell_dir'] = os.path.join(
dirs['abs_test_install_dir'], 'xpcshell')
dirs['abs_cppunittest_dir'] = os.path.join(
dirs['abs_test_install_dir'], 'cppunittest')
+ dirs['abs_marionette_dir'] = os.path.join(
+ dirs['abs_test_install_dir'], 'marionette', 'marionette')
for key in dirs.keys():
if key not in abs_dirs:
abs_dirs[key] = dirs[key]
self.abs_dirs = abs_dirs
return self.abs_dirs
def download_and_extract(self):
target_suite = self.config['test_suite']
@@ -245,16 +248,17 @@ class B2GEmulatorTest(TestingMixin, VCSM
'b2gpath': dirs['abs_b2g-distro_dir'],
'emulator': emulator_type,
'logcat_dir': dirs['abs_work_dir'],
'modules_dir': dirs['abs_modules_dir'],
'remote_webserver': self.config['remote_webserver'],
'xre_path': os.path.join(dirs['abs_xre_dir'], 'bin'),
'utility_path': os.path.join(dirs['abs_test_install_dir'], 'bin'),
'symbols_path': self.symbols_path,
+ 'homedir': os.path.join(dirs['abs_emulator_dir'], 'b2g-distro'),
'busybox': self.busybox_path,
'total_chunks': self.config.get('total_chunks'),
'this_chunk': self.config.get('this_chunk'),
'test_path': self.config.get('test_path'),
'certificate_path': dirs['abs_certs_dir'],
'raw_log_file': raw_log_file,
'error_summary_file': error_summary_file,
}
@@ -269,40 +273,39 @@ class B2GEmulatorTest(TestingMixin, VCSM
str_format_values=str_format_values)
cmd.extend(opt for opt in options if not opt.endswith('None'))
tests = self.query_tests_args(self.config["suite_definitions"][suite].get("tests"),
try_tests,
str_format_values=str_format_values)
cmd.extend(opt for opt in tests if not opt.endswith('None'))
+ if self.test_manifest:
+ cmd.append(self.test_manifest)
+
return cmd
def _query_adb(self):
return self.which('adb') or \
os.getenv('ADB_PATH') or \
os.path.join(self.query_abs_dirs()['abs_b2g-distro_dir'],
'out', 'host', 'linux-x86', 'bin', 'adb')
def preflight_run_tests(self):
super(B2GEmulatorTest, self).preflight_run_tests()
suite = self.config['test_suite']
+ dirs = self.query_abs_dirs()
# set default test manifest by suite if none specified
- if not self.test_manifest:
- if suite == 'reftest':
- self.test_manifest = os.path.join('tests', 'layout',
- 'reftests', 'reftest.list')
- elif suite == 'xpcshell':
- self.test_manifest = os.path.join('tests', 'xpcshell_b2g.ini')
- elif suite == 'crashtest':
- self.test_manifest = os.path.join('tests', 'testing',
- 'crashtest', 'crashtests.list')
- elif suite == 'jsreftest':
- self.test_manifest = os.path.join('jsreftest', 'tests',
- 'jstests.list')
+ if self.test_manifest:
+ if suite == 'marionette':
+ self.test_manifest = os.path.join(dirs['abs_test_install_dir'],
+ 'marionette', 'tests',
+ 'testing', 'marionette',
+ 'client', 'marionette',
+ 'tests', self.test_manifest)
if not os.path.isfile(self.adb_path):
self.fatal("The adb binary '%s' is not a valid file!" % self.adb_path)
def install(self):
# The emulator was extracted during the download_and_extract step;
# there's no separate binary to install. We call pass to prevent
# the base implementation from running here.
@@ -347,20 +350,25 @@ class B2GEmulatorTest(TestingMixin, VCSM
if self.query_minidump_stackwalk():
env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir']
if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
self.mkdir_p(env['MOZ_UPLOAD_DIR'])
env = self.query_env(partial_env=env)
success_codes = self._get_success_codes(suite_name)
- parser = self.get_test_output_parser(suite_name,
- config=self.config,
- log_obj=self.log_obj,
- error_list=error_list)
+ if suite_name == "marionette":
+ parser = TestSummaryOutputParserHelper(config=self.config,
+ log_obj=self.log_obj,
+ error_list=self.error_list)
+ else:
+ parser = self.get_test_output_parser(suite_name,
+ config=self.config,
+ log_obj=self.log_obj,
+ error_list=error_list)
return_code = self.run_command(cmd, cwd=cwd, env=env,
output_timeout=1000,
output_parser=parser,
success_codes=success_codes)
logcat = os.path.join(dirs['abs_work_dir'], 'emulator-5554.log')
qemu = os.path.join(dirs['abs_work_dir'], 'qemu.log')
--- a/testing/mozharness/scripts/marionette.py
+++ b/testing/mozharness/scripts/marionette.py
@@ -21,16 +21,18 @@ from mozharness.base.vcs.vcsbase import
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
from mozharness.mozilla.gaia import GaiaMixin
from mozharness.mozilla.testing.errors import LogcatErrorList
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
from mozharness.mozilla.structuredlog import StructuredOutputParser
+# TODO: we could remove emulator specific code after B2G ICS emulator buildbot
+# builds is turned off, Bug 1209180.
class MarionetteTest(TestingMixin, MercurialScript, BlobUploadMixin, TransferMixin, GaiaMixin):
config_options = [[
["--application"],
{"action": "store",
"dest": "application",
"default": None,
"help": "application name of binary"
}
--- a/testing/taskcluster/tasks/tests/b2g_emulator_marionette.yml
+++ b/testing/taskcluster/tasks/tests/b2g_emulator_marionette.yml
@@ -6,23 +6,23 @@ task:
name: '[TC] Marionette Framework Unit Tests'
description: Marionette Framework Unit Tests test run
workerType: b2gtest-emulator
payload:
command:
- entrypoint
- >
- python ./mozharness/scripts/marionette.py
- --no-read-buildbot-config
- --config-file ./mozharness/configs/marionette/automation_emulator_config.py
- --config-file ./mozharness_configs/remove_executables.py
- --download-symbols ondemand
+ python ./mozharness/scripts/b2g_emulator_unittest.py
+ --config-file ./mozharness/configs/b2g/taskcluster_emulator_automation.py
+ --test-suite marionette
+ --test-manifest unit-tests.ini
--installer-url {{build_url}}
--test-packages-url {{test_packages_url}}
+ --xre-url https://api.pub.build.mozilla.org/tooltool/sha512/cefa8c00db04969d3a50e2a5509bd4ea1dc17d256a651a9518cb28dad72e87a1dbbcd3c88ef770be0edf0ab73d2d73925140df93618ffb7fab81b789d312f547
artifacts:
'public/build':
type: directory
path: '/home/worker/artifacts/'
expires: '{{#from_now}}1 year{{/from_now}}'
extra:
treeherder:
--- a/testing/taskcluster/tasks/tests/b2g_emulator_marionette_webapi.yml
+++ b/testing/taskcluster/tasks/tests/b2g_emulator_marionette_webapi.yml
@@ -6,24 +6,23 @@ task:
name: '[TC] Marionette WebAPI Tests'
description: Marionette WebAPI test run
workerType: b2gtest-emulator
payload:
command:
- entrypoint
- >
- python ./mozharness/scripts/marionette.py
- --no-read-buildbot-config
- --config-file ./mozharness/configs/marionette/automation_emulator_config.py
- --config-file ./mozharness_configs/remove_executables.py
- --download-symbols ondemand
+ python ./mozharness/scripts/b2g_emulator_unittest.py
+ --config-file ./mozharness/configs/b2g/taskcluster_emulator_automation.py
+ --test-suite marionette
--test-manifest webapi-tests.ini
--installer-url {{build_url}}
--test-packages-url {{test_packages_url}}
+ --xre-url https://api.pub.build.mozilla.org/tooltool/sha512/cefa8c00db04969d3a50e2a5509bd4ea1dc17d256a651a9518cb28dad72e87a1dbbcd3c88ef770be0edf0ab73d2d73925140df93618ffb7fab81b789d312f547
artifacts:
'public/build':
type: directory
path: '/home/worker/artifacts/'
expires: '{{#from_now}}1 year{{/from_now}}'
extra:
treeherderEnv: