--- a/testing/talos/talos/unittests/test_config.py
+++ b/testing/talos/talos/unittests/test_config.py
@@ -1,1118 +1,1118 @@
-from __future__ import absolute_import
-
-import copy
-import os
-
-import mock
-
-import mozunit
-import pytest
-import conftest
-
-from talos.config import (
- get_active_tests, get_test, get_config, get_browser_config,
- get_configs, ConfigurationError, DEFAULTS)
-from talos.test import PageloaderTest
-
-ORIGINAL_DEFAULTS = copy.deepcopy(DEFAULTS)
-
-
-class mock_test(PageloaderTest):
- keys = ['tpmanifest', 'tpcycles', 'tppagecycles', 'tprender', 'tpchrome',
- 'tpmozafterpaint', 'fnbpaint', 'tploadnocache', 'firstpaint', 'userready',
- 'testeventmap', 'base_vs_ref', 'mainthread', 'resolution', 'cycles',
- 'gecko_profile', 'gecko_profile_interval', 'gecko_profile_entries',
- 'tptimeout', 'win_counters', 'w7_counters', 'linux_counters', 'mac_counters',
- 'tpscrolltest', 'xperf_counters', 'timeout', 'shutdown', 'responsiveness',
- 'profile_path', 'xperf_providers', 'xperf_user_providers', 'xperf_stackwalk',
- 'format_pagename', 'filters', 'preferences', 'extensions', 'setup', 'cleanup',
- 'lower_is_better', 'alert_threshold', 'unit', 'webextensions', 'profile',
- 'tpmozafterpaint', 'url']
-
- tpmozafterpaint = 'value'
- firstpaint = 'value'
- userready = 'value'
- fnbpaint = 'value'
-
-
-class Test_get_active_tests(object):
- def test_raises_exception_for_undefined_test(self):
- with pytest.raises(ConfigurationError):
- get_active_tests({'activeTests': 'undefined_test'})
-
- with pytest.raises(ConfigurationError):
- get_active_tests({'activeTests': ' undefined_test '})
-
- with pytest.raises(ConfigurationError):
- get_active_tests({'activeTests': 'undef_test:undef_test2:undef_test3'})
-
-
-class Test_get_test(object):
- global_overrides = {
- 'tpmozafterpaint': 'overriden',
- 'firstpaint': 'overriden',
- 'userready': 'overriden',
- 'fnbpaint': 'overriden'
- }
-
- config = {'webserver': 'test_webserver'}
-
- def test_doesnt_override_specific_keys_unless_they_are_null(self):
- test_instance = mock_test()
- test_dict = get_test({}, self.global_overrides, [], test_instance)
-
- assert test_dict['tpmozafterpaint'] == 'value'
- assert test_dict['firstpaint'] == 'value'
- assert test_dict['userready'] == 'value'
- assert test_dict['fnbpaint'] == 'value'
-
- # nulls still get overriden
- test_instance = mock_test(
- tpmozafterpaint=None, firstpaint=None, userready=None, fnbpaint=None)
- test_dict = get_test({}, self.global_overrides, [], test_instance)
-
- assert test_dict['tpmozafterpaint'] == 'overriden'
- assert test_dict['firstpaint'] == 'overriden'
- assert test_dict['userready'] == 'overriden'
- assert test_dict['fnbpaint'] == 'overriden'
-
- @mock.patch('talos.config.open', create=True)
- def test_interpolate_keys(self, mock_open):
- mock_open.return_value = mock.MagicMock(readlines=lambda: [])
-
- test_instance = mock_test(url='${talos}/test_page.html',
- tpmanifest='${talos}/file.manifest')
-
- test_dict = get_test(self.config, self.global_overrides, [], test_instance)
- assert test_dict['url'].startswith('http://test_webserver/')
- assert '${talos}' not in test_dict['url']
- assert '${talos}' not in test_dict['tpmanifest']
-
- def test_build_tpmanifest(self, tmpdir):
- manifest_file = tmpdir.join('file.manifest').ensure(file=True)
- test_instance = mock_test(url='test_page.html',
- tpmanifest=str(manifest_file))
-
- test_dict = get_test(self.config, self.global_overrides, [], test_instance)
- assert test_dict['tpmanifest'].endswith('.develop')
-
- def test_add_counters(self):
- test_instance = mock_test(
- linux_counters=None,
- mac_counters=[],
- win_counters=['counter_a'],
- w7_counters=['counter_a', 'counter_b'],
- xperf_counters=['counter_a', 'counter_extra']
- )
-
- counters = ['counter_a', 'counter_b', 'counter_c']
- test_dict = get_test(
- self.config, self.global_overrides, counters, test_instance)
-
- assert test_dict['linux_counters'] == counters
- assert test_dict['mac_counters'] == counters
- assert test_dict['win_counters'] == counters
- assert test_dict['w7_counters'] == counters
- assert set(test_dict['xperf_counters']) == set(counters + ['counter_extra'])
-
-
-class Test_get_browser_config(object):
- required = ('preferences', 'extensions', 'browser_path', 'browser_wait',
- 'extra_args', 'buildid', 'env', 'init_url', 'webserver')
- optional = ['bcontroller_config',
- 'branch_name',
- 'child_process',
- 'develop',
- 'e10s',
- 'process',
- 'framework',
- 'repository',
- 'sourcestamp',
- 'symbols_path',
- 'test_timeout',
- 'xperf_path',
- 'error_filename',
- 'no_upload_results',
- 'enable_stylo',
- 'disable_stylo',
- 'stylothreads',
- 'subtests']
-
- def test_that_contains_title(self):
- config_no_optionals = dict.fromkeys(self.required, '')
- config_no_optionals.update(title='is_mandatory')
-
- browser_config = get_browser_config(config_no_optionals)
- assert browser_config['title'] == 'is_mandatory'
-
- def test_raises_keyerror_for_missing_title(self):
- config_missing_title = dict.fromkeys(self.required, '')
-
- with pytest.raises(KeyError):
- get_browser_config(config_missing_title)
-
- def test_raises_keyerror_for_required_keys(self):
- config_missing_required = dict.fromkeys(self.required, '')
- config_missing_required.update(title='is_mandatory')
- del config_missing_required['preferences']
-
- with pytest.raises(KeyError):
- get_browser_config(config_missing_required)
-
- def test_doesnt_raise_on_missing_optionals(self):
- config_missing_optionals = dict.fromkeys(self.required, '')
- config_missing_optionals['title'] = 'is_mandatory'
-
- try:
- get_browser_config(config_missing_optionals)
- except KeyError:
- pytest.fail('Must not raise exception on missing optional')
-
- def test_browser_keys_are_subset_from_config(self):
- config_extensive = dict.fromkeys(self.required, '')
- config_extensive.update(dict.fromkeys(self.optional, ''))
- config_extensive['title'] = 'is_mandatory'
- config_extensive['extra_custom_key'] = 'value'
-
- browser_config = get_browser_config(config_extensive)
- assert browser_config != config_extensive
- assert set(browser_config.keys()).issubset(set(config_extensive.keys()))
-
-
-class Test_get_config(object):
- @classmethod
- def setup_class(cls):
- cls.argv = '--suite other-e10s --mainthread -e /some/random/path'.split()
- cls.argv_unprovided_tests = '-e /some/random/path'.split()
- cls.argv_unknown_suite = '--suite random-unknown-suite -e /some/random/path'.split()
- cls.argv_overrides_defaults = '''
- --suite other-e10s
- --executablePath /some/random/path
- --cycles 20
- --geckoProfile
- --geckoProfileInterval 1000
- --geckoProfileEntries 1000
- --mainthread
- --tpcycles 20
- --mozAfterPaint
- --firstPaint
- --firstNonBlankPaint
- --userReady
- --tppagecycles 20
- '''.split()
-
- cls.argv_ts_paint = '--activeTests ts_paint -e /some/random/path'.split()
- cls.argv_ts_paint_webext = '--activeTests ts_paint_webext -e /some/random/path'.split()
- cls.argv_ts_paint_heavy = '--activeTests ts_paint_heavy -e /some/random/path'.split()
- cls.argv_sessionrestore = '--activeTests sessionrestore -e /some/random/path'.split()
- cls.argv_sessionrestore_no_auto_restore = \
- '--activeTests sessionrestore_no_auto_restore -e /some/random/path'.split()
- cls.argv_sessionrestore_many_windows = \
- '--activeTests sessionrestore_many_windows -e /some/random/path'.split()
- cls.argv_tresize = '--activeTests tresize -e /some/random/path'.split()
- cls.argv_tpaint = '--activeTests tpaint -e /some/random/path'.split()
- cls.argv_cpstartup = '--activeTests cpstartup -e /some/random/path'.split()
- cls.argv_tabpaint = '--activeTests tabpaint -e /some/random/path'.split()
- cls.argv_tps = '--activeTests tps -e /some/random/path'.split()
- cls.argv_tart = '--activeTests tart -e /some/random/path'.split()
- cls.argv_cart = '--activeTests cart -e /some/random/path'.split()
- cls.argv_damp = '--activeTests damp -e /some/random/path'.split()
- cls.argv_glterrain = '--activeTests glterrain -e /some/random/path'.split()
- cls.argv_glvideo = '--activeTests glvideo -e /some/random/path'.split()
- cls.argv_tp5n = '--activeTests tp5n -e /some/random/path'.split()
- cls.argv_tp5o = '--activeTests tp5o -e /some/random/path'.split()
- cls.argv_tp5o_webext = '--activeTests tp5o_webext -e /some/random/path'.split()
- cls.argv_tp5o_scroll = '--activeTests tp5o_scroll -e /some/random/path'.split()
- cls.argv_v8_7 = '--activeTests v8_7 -e /some/random/path'.split()
- cls.argv_kraken = '--activeTests kraken -e /some/random/path'.split()
- cls.argv_basic_compositor_video = \
- '--activeTests basic_compositor_video -e /some/random/path'.split()
- cls.argv_tcanvasmark = '--activeTests tcanvasmark -e /some/random/path'.split()
- cls.argv_dromaeo_css = '--activeTests dromaeo_css -e /some/random/path'.split()
- cls.argv_dromaeo_dom = '--activeTests dromaeo_dom -e /some/random/path'.split()
- cls.argv_tsvgm = '--activeTests tsvgm -e /some/random/path'.split()
- cls.argv_tsvgx = '--activeTests tsvgx -e /some/random/path'.split()
- cls.argv_tsvg_static = '--activeTests tsvg_static -e /some/random/path'.split()
- cls.argv_tsvgr_opacity = '--activeTests tsvgr_opacity -e /some/random/path'.split()
- cls.argv_tscrollx = '--activeTests tscrollx -e /some/random/path'.split()
- cls.argv_a11yr = '--activeTests a11yr -e /some/random/path'.split()
- cls.argv_speedometer = '--activeTests speedometer -e /some/random/path'.split()
- cls.argv_perf_reftest = '--activeTests perf_reftest -e /some/random/path'.split()
- cls.argv_perf_reftest_singletons = \
- '--activeTests perf_reftest_singletons -e /some/random/path'.split()
- cls.argv_tp6_google = '--activeTests tp6_google -e /some/random/path'.split()
- cls.argv_tp6_google_heavy = '--activeTests tp6_google_heavy -e /some/random/path'.split()
- cls.argv_tp6_youtube = '--activeTests tp6_youtube -e /some/random/path'.split()
- cls.argv_tp6_youtube_heavy = '--activeTests tp6_youtube_heavy -e /some/random/path'.split()
- cls.argv_tp6_amazon = '--activeTests tp6_amazon -e /some/random/path'.split()
- cls.argv_tp6_amazon_heavy = '--activeTests tp6_amazon_heavy -e /some/random/path'.split()
- cls.argv_tp6_facebook = '--activeTests tp6_facebook -e /some/random/path'.split()
- cls.argv_tp6_facebook_heavy = \
- '--activeTests tp6_facebook_heavy -e /some/random/path'.split()
-
- @classmethod
- def teardown_class(cls):
- conftest.remove_develop_files()
-
- def test_correctly_overrides_test_valus(self):
- config = get_config(self.argv)
- assert bool(config) is True
-
- # no null values
- null_keys = [key for key, val in config.iteritems() if val is None]
- assert len(null_keys) == 0
-
- # expected keys are there
- assert config['browser_path'] == '/some/random/path'
- assert config['suite'] == 'other-e10s'
- assert config['mainthread'] is True
-
- # default values overriden
- config = get_config(self.argv_overrides_defaults)
- assert config['basetest'] == ORIGINAL_DEFAULTS['basetest']
-
- def test_config_has_tests(self):
- config = get_config(self.argv)
- assert len(config['tests']) > 0
-
- def test_global_variable_isnt_modified(self):
- get_config(self.argv)
- assert ORIGINAL_DEFAULTS == DEFAULTS
-
- def test_raises_except_if_unprovided_tests_on_cli(self):
- with pytest.raises(ConfigurationError):
- get_config(self.argv_unprovided_tests)
-
- with pytest.raises(ConfigurationError):
- get_config(self.argv_unknown_suite)
-
- def test_ts_paint_has_expected_attributes(self):
- config = get_config(self.argv_ts_paint)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'ts_paint'
- assert test_config['cycles'] == 20
- assert test_config['timeout'] == 150
- assert test_config['gecko_profile_startup'] is True
- assert test_config['gecko_profile_entries'] == 10000000
- assert test_config['url'] != 'startup_test/tspaint_test.html' # interpolation was done
- assert test_config['shutdown'] is False
- assert test_config['xperf_counters'] == []
- # TODO: these don't work; is this a bug?
- # assert test_config['win7_counters'] == []
- assert test_config['filters'] is not None
- assert test_config['tpmozafterpaint'] is True
- # assert test_config['mainthread'] is False
- # assert test_config['responsiveness'] is False
- # assert test_config['unit'] == 'ms'
-
- def test_ts_paint_webext_has_expected_attributes(self):
- config = get_config(self.argv_ts_paint_webext)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'ts_paint_webext'
- assert test_config['cycles'] == 20
- assert test_config['timeout'] == 150
- assert test_config['gecko_profile_startup'] is True
- assert test_config['gecko_profile_entries'] == 10000000
- assert test_config['url'] != 'startup_test/tspaint_test.html' # interpolation was done
- assert test_config['shutdown'] is False
- assert test_config['xperf_counters'] == []
- # TODO: these don't work; is this a bug?
- # assert test_config['win7_counters'] == []
- assert test_config['filters'] is not None
- assert test_config['tpmozafterpaint'] is True
- # assert test_config['mainthread'] is False
- # assert test_config['responsiveness'] is False
- # assert test_config['unit'] == 'ms'
- # TODO: this isn't overriden
- # assert test_config['webextensions'] != '${talos}/webextensions/dummy/dummy-signed.xpi'
- assert test_config['preferences'] == {'xpinstall.signatures.required': False}
-
- def test_ts_paint_heavy_has_expected_attributes(self):
- config = get_config(self.argv_ts_paint_heavy)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'ts_paint_heavy'
- assert test_config['cycles'] == 20
- assert test_config['timeout'] == 150
- assert test_config['gecko_profile_startup'] is True
- assert test_config['gecko_profile_entries'] == 10000000
- assert test_config['url'] != 'startup_test/tspaint_test.html' # interpolation was done
- assert test_config['shutdown'] is False
- assert test_config['xperf_counters'] == []
- # TODO: this doesn't work; is this a bug?
- # assert test_config['win7_counters'] == []
- assert test_config['filters'] is not None
- assert test_config['tpmozafterpaint'] is True
- # assert test_config['mainthread'] is False
- # assert test_config['responsiveness'] is False
- # assert test_config['unit'] == 'ms'
- assert test_config['profile'] == 'simple'
-
- def test_sessionrestore_has_expected_attributes(self):
- config = get_config(self.argv_sessionrestore)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'sessionrestore'
- assert test_config['cycles'] == 10
- assert test_config['timeout'] == 900
- assert test_config['gecko_profile_startup'] is True
- assert test_config['gecko_profile_entries'] == 10000000
- assert test_config['shutdown'] is False
- assert test_config['reinstall'] == [
- 'sessionstore.jsonlz4', 'sessionstore.js', 'sessionCheckpoints.json']
- assert test_config['url'] == 'about:home'
- assert test_config['preferences'] == {'browser.startup.page': 3}
- # assert test_config['unit'] == 'ms'
-
- def test_sesssionrestore_no_auto_restore_has_expected_attributes(self):
- config = get_config(self.argv_sessionrestore_no_auto_restore)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'sessionrestore_no_auto_restore'
- assert test_config['cycles'] == 10
- assert test_config['timeout'] == 900
- assert test_config['gecko_profile_startup'] is True
- assert test_config['gecko_profile_entries'] == 10000000
- assert test_config['shutdown'] is False
- assert test_config['reinstall'] == [
- 'sessionstore.jsonlz4', 'sessionstore.js', 'sessionCheckpoints.json']
- assert test_config['url'] == 'about:home'
- assert test_config['preferences'] == {'browser.startup.page': 1}
- # assert test_config['unit'] == 'ms'
-
- def test_sessionrestore_many_windows_has_expected_attributes(self):
- config = get_config(self.argv_sessionrestore_many_windows)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'sessionrestore_many_windows'
- assert test_config['cycles'] == 10
- assert test_config['timeout'] == 900
- assert test_config['gecko_profile_startup'] is True
- assert test_config['gecko_profile_entries'] == 10000000
- assert test_config['shutdown'] is False
- assert test_config['reinstall'] == [
- 'sessionstore.jsonlz4', 'sessionstore.js', 'sessionCheckpoints.json']
- assert test_config['url'] == 'about:home'
- assert test_config['preferences'] == {'browser.startup.page': 3}
- # assert test_config['unit'] == 'ms'
-
- def test_tresize_has_expected_attributes(self):
- config = get_config(self.argv_tresize)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tresize'
- assert test_config['cycles'] == 20
- assert test_config['url'] != 'startup_test/tresize/addon/content/tresize-test.html'
- assert test_config['timeout'] == 150
- assert test_config['gecko_profile_interval'] == 2
- assert test_config['gecko_profile_entries'] == 1000000
- assert test_config['tpmozafterpaint'] is True
- assert test_config['filters'] is not None
- # assert test_config['unit'] == 'ms'
-
- def test_tpaint_has_expected_attributes(self):
- config = get_config(self.argv_tpaint)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tpaint'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 20
- assert test_config['timeout'] == 300
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 2000000
- assert test_config['tpmozafterpaint'] is True
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
- assert test_config['preferences'] == {
- 'security.data_uri.block_toplevel_data_uri_navigations': False}
-
- def test_cpstartup_has_expected_attributes(self):
- config = get_config(self.argv_cpstartup)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'cpstartup'
- assert test_config['tpcycles'] == 1
- assert test_config['tpmanifest'] != '${talos}/tests/cpstartup/cpstartup.manifest'
- assert test_config['tppagecycles'] == 20
- assert test_config['gecko_profile_entries'] == 1000000
- assert test_config['tploadnocache'] is True
- assert test_config['unit'] == 'ms'
- assert test_config['preferences'] == {
- 'browser.link.open_newwindow': 3,
- 'browser.link.open_newwindow.restriction': 2,
- }
-
- def test_tabpaint_has_expected_attributes(self):
- config = get_config(self.argv_tabpaint)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tabpaint'
- assert test_config['tpcycles'] == 1
- assert test_config['tpmanifest'] != '${talos}/tests/tabpaint/tabpaint.manifest'
- assert test_config['tppagecycles'] == 20
- assert test_config['gecko_profile_entries'] == 1000000
- assert test_config['tploadnocache'] is True
- assert test_config['unit'] == 'ms'
- assert test_config['preferences'] == {
- 'browser.link.open_newwindow': 3,
- 'browser.link.open_newwindow.restriction': 2,
- }
-
- def test_tps_has_expected_attributes(self):
- config = get_config(self.argv_tps)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tps'
- assert test_config['tpcycles'] == 1
- assert test_config['tpmanifest'] != '${talos}/tests/tabswitch/tps.manifest'
- assert test_config['tppagecycles'] == 5
- assert test_config['gecko_profile_entries'] == 5000000
- assert test_config['tploadnocache'] is True
- assert test_config['preferences'] == {
- 'addon.test.tabswitch.urlfile': os.path.join('${talos}',
- 'tests',
- 'tp5o.html'),
- 'addon.test.tabswitch.webserver': '${webserver}',
- 'addon.test.tabswitch.maxurls': -1,
- }
- assert test_config['unit'] == 'ms'
-
- def test_tart_has_expected_attributes(self):
- config = get_config(self.argv_tart)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tart'
- assert test_config['tpmanifest'] != '${talos}/tests/tart/tart.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['tploadnocache'] is True
- assert test_config['tpmozafterpaint'] is False
- assert test_config['gecko_profile_interval'] == 10
- assert test_config['gecko_profile_entries'] == 1000000
- assert 'win_counters' not in test_config
- assert 'w7_counters' not in test_config
- assert 'linux_counters' not in test_config
- assert 'mac_counters' not in test_config
- assert test_config['preferences'] == {
- 'layout.frame_rate': 0,
- 'docshell.event_starvation_delay_hint': 1,
- 'dom.send_after_paint_to_content': False
- }
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
-
- def test_cart_has_expected_attributes(self):
- config = get_config(self.argv_cart)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'cart'
- assert test_config['tpmanifest'] != '${talos}/tests/tart/cart.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['tploadnocache'] is True
- assert test_config['tpmozafterpaint'] is False
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 10000000
- assert 'win_counters' not in test_config
- assert 'w7_counters' not in test_config
- assert 'linux_counters' not in test_config
- assert 'mac_counters' not in test_config
- assert test_config['preferences'] == {
- 'layout.frame_rate': 0,
- 'docshell.event_starvation_delay_hint': 1,
- 'dom.send_after_paint_to_content': False
- }
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
-
- def test_damp_has_expected_attributes(self):
- config = get_config(self.argv_damp)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'damp'
- assert test_config['tpmanifest'] != '${talos}/tests/devtools/damp.manifest'
- assert test_config['cycles'] == 5
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 5
- assert test_config['tploadnocache'] is True
- assert test_config['tpmozafterpaint'] is False
- assert test_config['gecko_profile_interval'] == 10
- assert test_config['gecko_profile_entries'] == 1000000
- assert 'win_counters' not in test_config
- assert 'w7_counters' not in test_config
- assert 'linux_counters' not in test_config
- assert 'mac_counters' not in test_config
- assert test_config['filters'] is not None
- assert test_config['preferences'] == {
- 'devtools.memory.enabled': True,
- 'addon.test.damp.webserver': '${webserver}'
- }
- assert test_config['unit'] == 'ms'
-
- def test_glterrain_has_expected_attributes(self):
- config = get_config(self.argv_glterrain)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'glterrain'
- assert test_config['tpmanifest'] != '${talos}/tests/webgl/glterrain.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['tploadnocache'] is True
- assert test_config['tpmozafterpaint'] is False
- assert test_config['tpchrome'] is False
- assert test_config['gecko_profile_interval'] == 10
- assert test_config['gecko_profile_entries'] == 2000000
- assert 'win_counters' not in test_config
- assert 'w7_counters' not in test_config
- assert 'linux_counters' not in test_config
- assert 'mac_counters' not in test_config
- assert test_config['preferences'] == {
- 'layout.frame_rate': 0,
- 'docshell.event_starvation_delay_hint': 1,
- 'dom.send_after_paint_to_content': False
- }
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'frame interval'
-
- def test_glvideo_has_expected_attributes(self):
- config = get_config(self.argv_glvideo)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'glvideo'
- assert test_config['tpmanifest'] != '${talos}/tests/webgl/glvideo.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 5
- assert test_config['tploadnocache'] is True
- assert test_config['tpmozafterpaint'] is False
- assert test_config['tpchrome'] is False
- assert test_config['gecko_profile_interval'] == 2
- assert test_config['gecko_profile_entries'] == 2000000
- assert 'win_counters' not in test_config
- assert 'w7_counters' not in test_config
- assert 'linux_counters' not in test_config
- assert 'mac_counters' not in test_config
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
-
- @mock.patch('talos.config.build_manifest', conftest.patched_build_manifest)
- def test_tp5n_has_expected_attributes(self):
- config = get_config(self.argv_tp5n)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tp5n'
- assert test_config['resolution'] == 20
- assert test_config['shutdown'] is False
- assert test_config['tpmanifest'] != '${talos}/tests/tp5n/tp5n.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 1
- assert test_config['cycles'] == 1
- assert test_config['tpmozafterpaint'] is True
- assert test_config['tptimeout'] == 5000
- assert test_config['mainthread'] is True
- assert test_config['w7_counters'] == []
- assert test_config['win_counters'] == []
- assert test_config['linux_counters'] == []
- assert test_config['mac_counters'] == []
- assert test_config['xperf_counters'] == [
- 'main_startup_fileio', 'main_startup_netio',
- 'main_normal_fileio', 'main_normal_netio',
- 'nonmain_startup_fileio', 'nonmain_normal_fileio',
- 'nonmain_normal_netio', 'mainthread_readcount',
- 'mainthread_readbytes', 'mainthread_writecount',
- 'mainthread_writebytes'
- ]
- assert test_config['xperf_providers'] == [
- 'PROC_THREAD', 'LOADER', 'HARD_FAULTS', 'FILENAME',
- 'FILE_IO', 'FILE_IO_INIT'
- ]
- assert test_config['xperf_user_providers'] == [
- 'Mozilla Generic Provider',
- 'Microsoft-Windows-TCPIP'
- ]
- assert test_config['xperf_stackwalk'] == [
- 'FileCreate', 'FileRead', 'FileWrite', 'FileFlush',
- 'FileClose'
- ]
- assert test_config['filters'] is not None
- assert test_config['timeout'] == 1800
- assert test_config['preferences'] == {
- 'extensions.enabledScopes': '',
- 'talos.logfile': 'browser_output.txt'
- }
- assert test_config['unit'] == 'ms'
-
- @mock.patch('talos.config.build_manifest', conftest.patched_build_manifest)
- def test_tp5o_has_expected_attributes(self):
- config = get_config(self.argv_tp5o)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tp5o'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['cycles'] == 1
- assert test_config['tpmozafterpaint'] is True
- assert test_config['tptimeout'] == 5000
- assert test_config['mainthread'] is False
- assert test_config['tpmanifest'] != '${talos}/tests/tp5n/tp5o.manifest'
- assert test_config['win_counters'] == ['% Processor Time']
- assert test_config['w7_counters'] == ['% Processor Time']
- assert test_config['linux_counters'] == ['XRes']
- assert test_config['mac_counters'] == []
- assert test_config['responsiveness'] is True
- assert test_config['gecko_profile_interval'] == 2
- assert test_config['gecko_profile_entries'] == 4000000
- assert test_config['filters'] is not None
- assert test_config['timeout'] == 1800
- assert test_config['unit'] == 'ms'
-
- @mock.patch('talos.config.build_manifest', conftest.patched_build_manifest)
- def test_tp5o_webext_has_expected_attributes(self):
- config = get_config(self.argv_tp5o_webext)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tp5o_webext'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['cycles'] == 1
- assert test_config['tpmozafterpaint'] is True
- assert test_config['tptimeout'] == 5000
- assert test_config['mainthread'] is False
- assert test_config['tpmanifest'] != '${talos}/tests/tp5n/tp5o.manifest'
- assert test_config['win_counters'] == ['% Processor Time']
- assert test_config['w7_counters'] == ['% Processor Time']
- assert test_config['linux_counters'] == ['XRes']
- assert test_config['mac_counters'] == []
- assert test_config['responsiveness'] is True
- assert test_config['gecko_profile_interval'] == 2
- assert test_config['gecko_profile_entries'] == 4000000
- assert test_config['filters'] is not None
- assert test_config['timeout'] == 1800
- assert test_config['unit'] == 'ms'
- assert test_config['webextensions'] == '${talos}/webextensions/dummy/dummy-signed.xpi'
- assert test_config['preferences'] == {'xpinstall.signatures.required': False}
-
- @mock.patch('talos.config.build_manifest', conftest.patched_build_manifest)
- def test_tp5o_scroll_has_expected_attributes(self):
- config = get_config(self.argv_tp5o_scroll)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tp5o_scroll'
- assert test_config['tpmanifest'] != '${talos}/tests/tp5n/tp5o.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 12
- assert test_config['gecko_profile_interval'] == 2
- assert test_config['gecko_profile_entries'] == 2000000
- assert test_config['tpscrolltest'] is True
- assert test_config['tpmozafterpaint'] is False
- assert test_config['preferences'] == {
- 'layout.frame_rate': 0,
- 'docshell.event_starvation_delay_hint': 1,
- 'dom.send_after_paint_to_content': False,
- 'layout.css.scroll-behavior.spring-constant': "'10'",
- 'toolkit.framesRecording.bufferSize': 10000
- }
- assert test_config['filters'] is not None
- assert test_config['unit'] == '1/FPS'
-
- def test_v8_7_has_expected_attributes(self):
- config = get_config(self.argv_v8_7)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'v8_7'
- assert test_config['tpmanifest'] != '${talos}/tests/v8_7/v8.manifest'
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 1000000
- assert test_config['tpcycles'] == 1
- assert test_config['resolution'] == 20
- assert test_config['tpmozafterpaint'] is False
- assert test_config['preferences'] == {'dom.send_after_paint_to_content': False}
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'score'
- assert test_config['lower_is_better'] is False
-
- def test_kraken_has_expected_attributes(self):
- config = get_config(self.argv_kraken)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'kraken'
- assert test_config['tpmanifest'] != '${talos}/tests/kraken/kraken.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 1
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 5000000
- assert test_config['tpmozafterpaint'] is False
- assert test_config['tpchrome'] is False
- assert test_config['preferences'] == {'dom.send_after_paint_to_content': False}
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'score'
-
- def test_basic_compositor_video_has_expected_attributes(self):
- config = get_config(self.argv_basic_compositor_video)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'basic_compositor_video'
- assert test_config['tpmanifest'] != '${talos}/tests/video/video.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 12
- assert test_config['tpchrome'] is False
- assert test_config['timeout'] == 10000
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 2000000
- assert test_config['preferences'] == {
- 'full-screen-api.allow-trusted-requests-only': False,
- 'layers.acceleration.force-enabled': False,
- 'layers.acceleration.disabled': True,
- 'layout.frame_rate': 0,
- 'docshell.event_starvation_delay_hint': 1,
- 'full-screen-api.warning.timeout': 500,
- 'media.ruin-av-sync.enabled': True
- }
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms/frame'
- assert test_config['lower_is_better'] is True
-
- def test_tcanvasmark_has_expected_attributes(self):
- config = get_config(self.argv_tcanvasmark)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tcanvasmark'
- assert test_config['tpmanifest'] != '${talos}/tests/canvasmark/canvasmark.manifest'
- assert 'win_counters' not in test_config
- assert 'w7_counters' not in test_config
- assert 'linux_counters' not in test_config
- assert 'mac_counters' not in test_config
- assert test_config['tpcycles'] == 5
- assert test_config['tppagecycles'] == 1
- assert test_config['timeout'] == 900
- assert test_config['gecko_profile_interval'] == 10
- assert test_config['gecko_profile_entries'] == 2500000
- assert test_config['tpmozafterpaint'] is False
- assert test_config['preferences'] == {'dom.send_after_paint_to_content': False}
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'score'
- assert test_config['lower_is_better'] is False
-
- def test_dromaeo_css_has_expected_attributes(self):
- config = get_config(self.argv_dromaeo_css)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'dromaeo_css'
- assert test_config['tpcycles'] == 1
- assert test_config['filters'] is not None
- assert test_config['lower_is_better'] is False
- assert test_config['alert_threshold'] == 5.0
- assert test_config['tpchrome'] is False
- assert test_config['gecko_profile_interval'] == 2
- assert test_config['gecko_profile_entries'] == 10000000
- assert test_config['tpmanifest'] != '${talos}/tests/dromaeo/css.manifest'
- assert test_config['unit'] == 'score'
-
- def test_dromaeo_dom_has_expected_attributes(self):
- config = get_config(self.argv_dromaeo_dom)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'dromaeo_dom'
- assert test_config['tpcycles'] == 1
- assert test_config['filters'] is not None
- assert test_config['lower_is_better'] is False
- assert test_config['alert_threshold'] == 5.0
- assert test_config['tpchrome'] is False
- assert test_config['gecko_profile_interval'] == 2
- assert test_config['gecko_profile_entries'] == 10000000
- assert test_config['tpmanifest'] != '${talos}/tests/dromaeo/dom.manifest'
- assert test_config['unit'] == 'score'
-
- def test_tsvgm_has_expected_attributes(self):
- config = get_config(self.argv_tsvgm)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tsvgm'
- assert test_config['tpmanifest'] != '${talos}/tests/svgx/svgm.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 7
- assert test_config['tpmozafterpaint'] is False
- assert test_config['tpchrome'] is False
- assert test_config['gecko_profile_interval'] == 10
- assert test_config['gecko_profile_entries'] == 1000000
- assert test_config['preferences'] == {
- 'layout.frame_rate': 0,
- 'docshell.event_starvation_delay_hint': 1,
- 'dom.send_after_paint_to_content': False
- }
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
-
- def test_tsvgx_has_expected_attributes(self):
- config = get_config(self.argv_tsvgx)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tsvgx'
- assert test_config['tpmanifest'] != '${talos}/tests/svgx/svgx.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['tpmozafterpaint'] is False
- assert test_config['tpchrome'] is False
- assert test_config['gecko_profile_interval'] == 10
- assert test_config['gecko_profile_entries'] == 1000000
- assert test_config['preferences'] == {
- 'layout.frame_rate': 0,
- 'docshell.event_starvation_delay_hint': 1,
- 'dom.send_after_paint_to_content': False
- }
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
-
- def test_tsvg_static_has_expected_attributes(self):
- config = get_config(self.argv_tsvg_static)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tsvg_static'
- assert test_config['tpmanifest'] != '${talos}/tests/svg_static/svg_static.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['tpmozafterpaint'] is True
- assert test_config['tpchrome'] is False
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 10000000
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
-
- def test_tsvgr_opacity_has_expected_attributes(self):
- config = get_config(self.argv_tsvgr_opacity)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tsvgr_opacity'
- assert test_config['tpmanifest'] != '${talos}/tests/svg_opacity/svg_opacity.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['tpmozafterpaint'] is True
- assert test_config['tpchrome'] is False
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 10000000
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
-
- def test_tscrollx_has_expected_attributes(self):
- config = get_config(self.argv_tscrollx)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tscrollx'
- assert test_config['tpmanifest'] != '${talos}/tests/scroll/scroll.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['tpmozafterpaint'] is False
- assert test_config['tpchrome'] is False
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 1000000
- assert test_config['preferences'] == {
- 'layout.frame_rate': 0,
- 'docshell.event_starvation_delay_hint': 1,
- 'dom.send_after_paint_to_content': False,
- 'layout.css.scroll-behavior.spring-constant': "'10'",
- 'toolkit.framesRecording.bufferSize': 10000
- }
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
-
- def test_a11yr_has_expect_attributes(self):
- config = get_config(self.argv_a11yr)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'a11yr'
- assert test_config['tpmanifest'] != '${talos}/tests/a11y/a11y.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['tpmozafterpaint'] is True
- assert test_config['tpchrome'] is False
- assert test_config['preferences'] == {'dom.send_after_paint_to_content': False}
- assert test_config['unit'] == 'ms'
- assert test_config['alert_threshold'] == 5.0
-
- def test_speedometer_has_expected_attributes(self):
- config = get_config(self.argv_speedometer)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'speedometer'
- assert test_config['tpmanifest'] != '${talos}/tests/speedometer/speedometer.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 5
- assert test_config['tpmozafterpaint'] is False
- assert test_config['tpchrome'] is False
- assert test_config['format_pagename'] is False
- assert test_config['lower_is_better'] is False
- assert test_config['unit'] == 'score'
-
- def test_perf_reftest_has_expected_attributes(self):
- config = get_config(self.argv_perf_reftest)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'perf_reftest'
- assert test_config['base_vs_ref'] is True
- assert test_config['tpmanifest'] != '${talos}/tests/perf-reftest/perf_reftest.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 10
- assert test_config['tptimeout'] == 30000
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 2000000
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
- assert test_config['lower_is_better'] is True
- assert test_config['alert_threshold'] == 5.0
-
- def test_perf_reftest_singletons_has_expected_attributes(self):
- config = get_config(self.argv_perf_reftest_singletons)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'perf_reftest_singletons'
- assert test_config['tpmanifest'] != \
- '${talos}/tests/perf-reftest-singletons/perf_reftest_singletons.manifest'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 15
- assert test_config['tptimeout'] == 30000
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 2000000
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
- assert test_config['lower_is_better'] is True
- assert test_config['alert_threshold'] == 5.0
-
- def test_tp6_google_has_expected_attributes(self):
- config = get_config(self.argv_tp6_google)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tp6_google'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 2000000
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
- assert test_config['lower_is_better'] is True
- assert test_config['fnbpaint'] is True
- assert test_config['tpmanifest'] != \
- '${talos}/tests/quantum_pageload/quantum_pageload_google.manifest'
-
- def test_tp6_google_heavy_has_expected_attributes(self):
- config = get_config(self.argv_tp6_google_heavy)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tp6_google_heavy'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 2000000
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
- assert test_config['lower_is_better'] is True
- assert test_config['fnbpaint'] is True
- assert test_config['profile'] == 'simple'
-
- def test_tp6_youtube_has_expected_attributes(self):
- config = get_config(self.argv_tp6_youtube)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tp6_youtube'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 2000000
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
- assert test_config['lower_is_better'] is True
- assert test_config['fnbpaint'] is True
- assert test_config['tpmanifest'] != \
- '${talos}/tests/quantum_pageload/quantum_pageload_youtube.manifest'
-
- def test_tp6_youtube_heavy_has_expected_attributes(self):
- config = get_config(self.argv_tp6_youtube_heavy)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tp6_youtube_heavy'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 2000000
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
- assert test_config['lower_is_better'] is True
- assert test_config['fnbpaint'] is True
- assert test_config['profile'] == 'simple'
-
- def test_tp6_amazon_has_expected_attributes(self):
- config = get_config(self.argv_tp6_amazon)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tp6_amazon'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 2000000
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
- assert test_config['lower_is_better'] is True
- assert test_config['fnbpaint'] is True
- assert test_config['tpmanifest'] != \
- '${talos}/tests/quantum_pageload/quantum_pageload_amazon.manifest'
-
- def test_tp6_amazon_heavy_has_expected_attributes(self):
- config = get_config(self.argv_tp6_amazon_heavy)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tp6_amazon_heavy'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 2000000
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
- assert test_config['lower_is_better'] is True
- assert test_config['fnbpaint'] is True
- assert test_config['profile'] == 'simple'
-
- def test_tp6_facebook_has_expected_attributes(self):
- config = get_config(self.argv_tp6_facebook)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tp6_facebook'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 2000000
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
- assert test_config['lower_is_better'] is True
- assert test_config['fnbpaint'] is True
- assert test_config['tpmanifest'] != \
- '${talos}/tests/quantum_pageload/quantum_pageload_facebook.manifest'
-
- def test_tp6_facebook_heavy_has_expected_attributes(self):
- config = get_config(self.argv_tp6_facebook_heavy)
- test_config = config['tests'][0]
-
- assert test_config['name'] == 'tp6_facebook_heavy'
- assert test_config['tpcycles'] == 1
- assert test_config['tppagecycles'] == 25
- assert test_config['gecko_profile_interval'] == 1
- assert test_config['gecko_profile_entries'] == 2000000
- assert test_config['filters'] is not None
- assert test_config['unit'] == 'ms'
- assert test_config['lower_is_better'] is True
- assert test_config['fnbpaint'] is True
- assert test_config['profile'] == 'simple'
-
-
-@mock.patch('talos.config.get_browser_config')
-@mock.patch('talos.config.get_config')
-def test_get_configs(get_config_mock, get_browser_config_mock):
- # unpacks in right order
- get_config_mock.return_value = 'first'
- get_browser_config_mock.return_value = 'second'
-
- first, second = get_configs()
- assert (first, second) == ('first', 'second')
-
-
-if __name__ == '__main__':
- mozunit.main()
+from __future__ import absolute_import
+
+import copy
+import os
+
+import mock
+
+import mozunit
+import pytest
+import conftest
+
+from talos.config import (
+ get_active_tests, get_test, get_config, get_browser_config,
+ get_configs, ConfigurationError, DEFAULTS)
+from talos.test import PageloaderTest
+
+ORIGINAL_DEFAULTS = copy.deepcopy(DEFAULTS)
+
+
+class mock_test(PageloaderTest):
+ keys = ['tpmanifest', 'tpcycles', 'tppagecycles', 'tprender', 'tpchrome',
+ 'tpmozafterpaint', 'fnbpaint', 'tploadnocache', 'firstpaint', 'userready',
+ 'testeventmap', 'base_vs_ref', 'mainthread', 'resolution', 'cycles',
+ 'gecko_profile', 'gecko_profile_interval', 'gecko_profile_entries',
+ 'tptimeout', 'win_counters', 'w7_counters', 'linux_counters', 'mac_counters',
+ 'tpscrolltest', 'xperf_counters', 'timeout', 'shutdown', 'responsiveness',
+ 'profile_path', 'xperf_providers', 'xperf_user_providers', 'xperf_stackwalk',
+ 'format_pagename', 'filters', 'preferences', 'extensions', 'setup', 'cleanup',
+ 'lower_is_better', 'alert_threshold', 'unit', 'webextensions', 'profile',
+ 'tpmozafterpaint', 'url']
+
+ tpmozafterpaint = 'value'
+ firstpaint = 'value'
+ userready = 'value'
+ fnbpaint = 'value'
+
+
+class Test_get_active_tests(object):
+ def test_raises_exception_for_undefined_test(self):
+ with pytest.raises(ConfigurationError):
+ get_active_tests({'activeTests': 'undefined_test'})
+
+ with pytest.raises(ConfigurationError):
+ get_active_tests({'activeTests': ' undefined_test '})
+
+ with pytest.raises(ConfigurationError):
+ get_active_tests({'activeTests': 'undef_test:undef_test2:undef_test3'})
+
+
+class Test_get_test(object):
+ global_overrides = {
+ 'tpmozafterpaint': 'overriden',
+ 'firstpaint': 'overriden',
+ 'userready': 'overriden',
+ 'fnbpaint': 'overriden'
+ }
+
+ config = {'webserver': 'test_webserver'}
+
+ def test_doesnt_override_specific_keys_unless_they_are_null(self):
+ test_instance = mock_test()
+ test_dict = get_test({}, self.global_overrides, [], test_instance)
+
+ assert test_dict['tpmozafterpaint'] == 'value'
+ assert test_dict['firstpaint'] == 'value'
+ assert test_dict['userready'] == 'value'
+ assert test_dict['fnbpaint'] == 'value'
+
+ # nulls still get overriden
+ test_instance = mock_test(
+ tpmozafterpaint=None, firstpaint=None, userready=None, fnbpaint=None)
+ test_dict = get_test({}, self.global_overrides, [], test_instance)
+
+ assert test_dict['tpmozafterpaint'] == 'overriden'
+ assert test_dict['firstpaint'] == 'overriden'
+ assert test_dict['userready'] == 'overriden'
+ assert test_dict['fnbpaint'] == 'overriden'
+
+ @mock.patch('talos.config.open', create=True)
+ def test_interpolate_keys(self, mock_open):
+ mock_open.return_value = mock.MagicMock(readlines=lambda: [])
+
+ test_instance = mock_test(url='${talos}/test_page.html',
+ tpmanifest='${talos}/file.manifest')
+
+ test_dict = get_test(self.config, self.global_overrides, [], test_instance)
+ assert test_dict['url'].startswith('http://test_webserver/')
+ assert '${talos}' not in test_dict['url']
+ assert '${talos}' not in test_dict['tpmanifest']
+
+ def test_build_tpmanifest(self, tmpdir):
+ manifest_file = tmpdir.join('file.manifest').ensure(file=True)
+ test_instance = mock_test(url='test_page.html',
+ tpmanifest=str(manifest_file))
+
+ test_dict = get_test(self.config, self.global_overrides, [], test_instance)
+ assert test_dict['tpmanifest'].endswith('.develop')
+
+ def test_add_counters(self):
+ test_instance = mock_test(
+ linux_counters=None,
+ mac_counters=[],
+ win_counters=['counter_a'],
+ w7_counters=['counter_a', 'counter_b'],
+ xperf_counters=['counter_a', 'counter_extra']
+ )
+
+ counters = ['counter_a', 'counter_b', 'counter_c']
+ test_dict = get_test(
+ self.config, self.global_overrides, counters, test_instance)
+
+ assert test_dict['linux_counters'] == counters
+ assert test_dict['mac_counters'] == counters
+ assert test_dict['win_counters'] == counters
+ assert test_dict['w7_counters'] == counters
+ assert set(test_dict['xperf_counters']) == set(counters + ['counter_extra'])
+
+
+class Test_get_browser_config(object):
+ required = ('preferences', 'extensions', 'browser_path', 'browser_wait',
+ 'extra_args', 'buildid', 'env', 'init_url', 'webserver')
+ optional = ['bcontroller_config',
+ 'branch_name',
+ 'child_process',
+ 'develop',
+ 'e10s',
+ 'process',
+ 'framework',
+ 'repository',
+ 'sourcestamp',
+ 'symbols_path',
+ 'test_timeout',
+ 'xperf_path',
+ 'error_filename',
+ 'no_upload_results',
+ 'enable_stylo',
+ 'disable_stylo',
+ 'stylothreads',
+ 'subtests']
+
+ def test_that_contains_title(self):
+ config_no_optionals = dict.fromkeys(self.required, '')
+ config_no_optionals.update(title='is_mandatory')
+
+ browser_config = get_browser_config(config_no_optionals)
+ assert browser_config['title'] == 'is_mandatory'
+
+ def test_raises_keyerror_for_missing_title(self):
+ config_missing_title = dict.fromkeys(self.required, '')
+
+ with pytest.raises(KeyError):
+ get_browser_config(config_missing_title)
+
+ def test_raises_keyerror_for_required_keys(self):
+ config_missing_required = dict.fromkeys(self.required, '')
+ config_missing_required.update(title='is_mandatory')
+ del config_missing_required['preferences']
+
+ with pytest.raises(KeyError):
+ get_browser_config(config_missing_required)
+
+ def test_doesnt_raise_on_missing_optionals(self):
+ config_missing_optionals = dict.fromkeys(self.required, '')
+ config_missing_optionals['title'] = 'is_mandatory'
+
+ try:
+ get_browser_config(config_missing_optionals)
+ except KeyError:
+ pytest.fail('Must not raise exception on missing optional')
+
+ def test_browser_keys_are_subset_from_config(self):
+ config_extensive = dict.fromkeys(self.required, '')
+ config_extensive.update(dict.fromkeys(self.optional, ''))
+ config_extensive['title'] = 'is_mandatory'
+ config_extensive['extra_custom_key'] = 'value'
+
+ browser_config = get_browser_config(config_extensive)
+ assert browser_config != config_extensive
+ assert set(browser_config.keys()).issubset(set(config_extensive.keys()))
+
+
+class Test_get_config(object):
+ @classmethod
+ def setup_class(cls):
+ cls.argv = '--suite other-e10s --mainthread -e /some/random/path'.split()
+ cls.argv_unprovided_tests = '-e /some/random/path'.split()
+ cls.argv_unknown_suite = '--suite random-unknown-suite -e /some/random/path'.split()
+ cls.argv_overrides_defaults = '''
+ --suite other-e10s
+ --executablePath /some/random/path
+ --cycles 20
+ --geckoProfile
+ --geckoProfileInterval 1000
+ --geckoProfileEntries 1000
+ --mainthread
+ --tpcycles 20
+ --mozAfterPaint
+ --firstPaint
+ --firstNonBlankPaint
+ --userReady
+ --tppagecycles 20
+ '''.split()
+
+ cls.argv_ts_paint = '--activeTests ts_paint -e /some/random/path'.split()
+ cls.argv_ts_paint_webext = '--activeTests ts_paint_webext -e /some/random/path'.split()
+ cls.argv_ts_paint_heavy = '--activeTests ts_paint_heavy -e /some/random/path'.split()
+ cls.argv_sessionrestore = '--activeTests sessionrestore -e /some/random/path'.split()
+ cls.argv_sessionrestore_no_auto_restore = \
+ '--activeTests sessionrestore_no_auto_restore -e /some/random/path'.split()
+ cls.argv_sessionrestore_many_windows = \
+ '--activeTests sessionrestore_many_windows -e /some/random/path'.split()
+ cls.argv_tresize = '--activeTests tresize -e /some/random/path'.split()
+ cls.argv_tpaint = '--activeTests tpaint -e /some/random/path'.split()
+ cls.argv_cpstartup = '--activeTests cpstartup -e /some/random/path'.split()
+ cls.argv_tabpaint = '--activeTests tabpaint -e /some/random/path'.split()
+ cls.argv_tps = '--activeTests tps -e /some/random/path'.split()
+ cls.argv_tart = '--activeTests tart -e /some/random/path'.split()
+ cls.argv_cart = '--activeTests cart -e /some/random/path'.split()
+ cls.argv_damp = '--activeTests damp -e /some/random/path'.split()
+ cls.argv_glterrain = '--activeTests glterrain -e /some/random/path'.split()
+ cls.argv_glvideo = '--activeTests glvideo -e /some/random/path'.split()
+ cls.argv_tp5n = '--activeTests tp5n -e /some/random/path'.split()
+ cls.argv_tp5o = '--activeTests tp5o -e /some/random/path'.split()
+ cls.argv_tp5o_webext = '--activeTests tp5o_webext -e /some/random/path'.split()
+ cls.argv_tp5o_scroll = '--activeTests tp5o_scroll -e /some/random/path'.split()
+ cls.argv_v8_7 = '--activeTests v8_7 -e /some/random/path'.split()
+ cls.argv_kraken = '--activeTests kraken -e /some/random/path'.split()
+ cls.argv_basic_compositor_video = \
+ '--activeTests basic_compositor_video -e /some/random/path'.split()
+ cls.argv_tcanvasmark = '--activeTests tcanvasmark -e /some/random/path'.split()
+ cls.argv_dromaeo_css = '--activeTests dromaeo_css -e /some/random/path'.split()
+ cls.argv_dromaeo_dom = '--activeTests dromaeo_dom -e /some/random/path'.split()
+ cls.argv_tsvgm = '--activeTests tsvgm -e /some/random/path'.split()
+ cls.argv_tsvgx = '--activeTests tsvgx -e /some/random/path'.split()
+ cls.argv_tsvg_static = '--activeTests tsvg_static -e /some/random/path'.split()
+ cls.argv_tsvgr_opacity = '--activeTests tsvgr_opacity -e /some/random/path'.split()
+ cls.argv_tscrollx = '--activeTests tscrollx -e /some/random/path'.split()
+ cls.argv_a11yr = '--activeTests a11yr -e /some/random/path'.split()
+ cls.argv_speedometer = '--activeTests speedometer -e /some/random/path'.split()
+ cls.argv_perf_reftest = '--activeTests perf_reftest -e /some/random/path'.split()
+ cls.argv_perf_reftest_singletons = \
+ '--activeTests perf_reftest_singletons -e /some/random/path'.split()
+ cls.argv_tp6_google = '--activeTests tp6_google -e /some/random/path'.split()
+ cls.argv_tp6_google_heavy = '--activeTests tp6_google_heavy -e /some/random/path'.split()
+ cls.argv_tp6_youtube = '--activeTests tp6_youtube -e /some/random/path'.split()
+ cls.argv_tp6_youtube_heavy = '--activeTests tp6_youtube_heavy -e /some/random/path'.split()
+ cls.argv_tp6_amazon = '--activeTests tp6_amazon -e /some/random/path'.split()
+ cls.argv_tp6_amazon_heavy = '--activeTests tp6_amazon_heavy -e /some/random/path'.split()
+ cls.argv_tp6_facebook = '--activeTests tp6_facebook -e /some/random/path'.split()
+ cls.argv_tp6_facebook_heavy = \
+ '--activeTests tp6_facebook_heavy -e /some/random/path'.split()
+
+ @classmethod
+ def teardown_class(cls):
+ conftest.remove_develop_files()
+
+ def test_correctly_overrides_test_valus(self):
+ config = get_config(self.argv)
+ assert bool(config) is True
+
+ # no null values
+ null_keys = [key for key, val in config.iteritems() if val is None]
+ assert len(null_keys) == 0
+
+ # expected keys are there
+ assert config['browser_path'] == '/some/random/path'
+ assert config['suite'] == 'other-e10s'
+ assert config['mainthread'] is True
+
+ # default values overriden
+ config = get_config(self.argv_overrides_defaults)
+ assert config['basetest'] == ORIGINAL_DEFAULTS['basetest']
+
+ def test_config_has_tests(self):
+ config = get_config(self.argv)
+ assert len(config['tests']) > 0
+
+ def test_global_variable_isnt_modified(self):
+ get_config(self.argv)
+ assert ORIGINAL_DEFAULTS == DEFAULTS
+
+ def test_raises_except_if_unprovided_tests_on_cli(self):
+ with pytest.raises(ConfigurationError):
+ get_config(self.argv_unprovided_tests)
+
+ with pytest.raises(ConfigurationError):
+ get_config(self.argv_unknown_suite)
+
+ def test_ts_paint_has_expected_attributes(self):
+ config = get_config(self.argv_ts_paint)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'ts_paint'
+ assert test_config['cycles'] == 20
+ assert test_config['timeout'] == 150
+ assert test_config['gecko_profile_startup'] is True
+ assert test_config['gecko_profile_entries'] == 10000000
+ assert test_config['url'] != 'startup_test/tspaint_test.html' # interpolation was done
+ assert test_config['shutdown'] is False
+ assert test_config['xperf_counters'] == []
+ # TODO: these don't work; is this a bug?
+ # assert test_config['win7_counters'] == []
+ assert test_config['filters'] is not None
+ assert test_config['tpmozafterpaint'] is True
+ # assert test_config['mainthread'] is False
+ # assert test_config['responsiveness'] is False
+ # assert test_config['unit'] == 'ms'
+
+ def test_ts_paint_webext_has_expected_attributes(self):
+ config = get_config(self.argv_ts_paint_webext)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'ts_paint_webext'
+ assert test_config['cycles'] == 20
+ assert test_config['timeout'] == 150
+ assert test_config['gecko_profile_startup'] is True
+ assert test_config['gecko_profile_entries'] == 10000000
+ assert test_config['url'] != 'startup_test/tspaint_test.html' # interpolation was done
+ assert test_config['shutdown'] is False
+ assert test_config['xperf_counters'] == []
+ # TODO: these don't work; is this a bug?
+ # assert test_config['win7_counters'] == []
+ assert test_config['filters'] is not None
+ assert test_config['tpmozafterpaint'] is True
+ # assert test_config['mainthread'] is False
+ # assert test_config['responsiveness'] is False
+ # assert test_config['unit'] == 'ms'
+ # TODO: this isn't overriden
+ # assert test_config['webextensions'] != '${talos}/webextensions/dummy/dummy-signed.xpi'
+ assert test_config['preferences'] == {'xpinstall.signatures.required': False}
+
+ def test_ts_paint_heavy_has_expected_attributes(self):
+ config = get_config(self.argv_ts_paint_heavy)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'ts_paint_heavy'
+ assert test_config['cycles'] == 20
+ assert test_config['timeout'] == 150
+ assert test_config['gecko_profile_startup'] is True
+ assert test_config['gecko_profile_entries'] == 10000000
+ assert test_config['url'] != 'startup_test/tspaint_test.html' # interpolation was done
+ assert test_config['shutdown'] is False
+ assert test_config['xperf_counters'] == []
+ # TODO: this doesn't work; is this a bug?
+ # assert test_config['win7_counters'] == []
+ assert test_config['filters'] is not None
+ assert test_config['tpmozafterpaint'] is True
+ # assert test_config['mainthread'] is False
+ # assert test_config['responsiveness'] is False
+ # assert test_config['unit'] == 'ms'
+ assert test_config['profile'] == 'simple'
+
+ def test_sessionrestore_has_expected_attributes(self):
+ config = get_config(self.argv_sessionrestore)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'sessionrestore'
+ assert test_config['cycles'] == 10
+ assert test_config['timeout'] == 900
+ assert test_config['gecko_profile_startup'] is True
+ assert test_config['gecko_profile_entries'] == 10000000
+ assert test_config['shutdown'] is False
+ assert test_config['reinstall'] == [
+ 'sessionstore.jsonlz4', 'sessionstore.js', 'sessionCheckpoints.json']
+ assert test_config['url'] == 'about:home'
+ assert test_config['preferences'] == {'browser.startup.page': 3}
+ # assert test_config['unit'] == 'ms'
+
+ def test_sesssionrestore_no_auto_restore_has_expected_attributes(self):
+ config = get_config(self.argv_sessionrestore_no_auto_restore)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'sessionrestore_no_auto_restore'
+ assert test_config['cycles'] == 10
+ assert test_config['timeout'] == 900
+ assert test_config['gecko_profile_startup'] is True
+ assert test_config['gecko_profile_entries'] == 10000000
+ assert test_config['shutdown'] is False
+ assert test_config['reinstall'] == [
+ 'sessionstore.jsonlz4', 'sessionstore.js', 'sessionCheckpoints.json']
+ assert test_config['url'] == 'about:home'
+ assert test_config['preferences'] == {'browser.startup.page': 1}
+ # assert test_config['unit'] == 'ms'
+
+ def test_sessionrestore_many_windows_has_expected_attributes(self):
+ config = get_config(self.argv_sessionrestore_many_windows)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'sessionrestore_many_windows'
+ assert test_config['cycles'] == 10
+ assert test_config['timeout'] == 900
+ assert test_config['gecko_profile_startup'] is True
+ assert test_config['gecko_profile_entries'] == 10000000
+ assert test_config['shutdown'] is False
+ assert test_config['reinstall'] == [
+ 'sessionstore.jsonlz4', 'sessionstore.js', 'sessionCheckpoints.json']
+ assert test_config['url'] == 'about:home'
+ assert test_config['preferences'] == {'browser.startup.page': 3}
+ # assert test_config['unit'] == 'ms'
+
+ def test_tresize_has_expected_attributes(self):
+ config = get_config(self.argv_tresize)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tresize'
+ assert test_config['cycles'] == 20
+ assert test_config['url'] != 'startup_test/tresize/addon/content/tresize-test.html'
+ assert test_config['timeout'] == 150
+ assert test_config['gecko_profile_interval'] == 2
+ assert test_config['gecko_profile_entries'] == 1000000
+ assert test_config['tpmozafterpaint'] is True
+ assert test_config['filters'] is not None
+ # assert test_config['unit'] == 'ms'
+
+ def test_tpaint_has_expected_attributes(self):
+ config = get_config(self.argv_tpaint)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tpaint'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 20
+ assert test_config['timeout'] == 300
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert test_config['tpmozafterpaint'] is True
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+ assert test_config['preferences'] == {
+ 'security.data_uri.block_toplevel_data_uri_navigations': False}
+
+ def test_cpstartup_has_expected_attributes(self):
+ config = get_config(self.argv_cpstartup)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'cpstartup'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tpmanifest'] != '${talos}/tests/cpstartup/cpstartup.manifest'
+ assert test_config['tppagecycles'] == 20
+ assert test_config['gecko_profile_entries'] == 1000000
+ assert test_config['tploadnocache'] is True
+ assert test_config['unit'] == 'ms'
+ assert test_config['preferences'] == {
+ 'browser.link.open_newwindow': 3,
+ 'browser.link.open_newwindow.restriction': 2,
+ }
+
+ def test_tabpaint_has_expected_attributes(self):
+ config = get_config(self.argv_tabpaint)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tabpaint'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tpmanifest'] != '${talos}/tests/tabpaint/tabpaint.manifest'
+ assert test_config['tppagecycles'] == 20
+ assert test_config['gecko_profile_entries'] == 1000000
+ assert test_config['tploadnocache'] is True
+ assert test_config['unit'] == 'ms'
+ assert test_config['preferences'] == {
+ 'browser.link.open_newwindow': 3,
+ 'browser.link.open_newwindow.restriction': 2,
+ }
+
+ def test_tps_has_expected_attributes(self):
+ config = get_config(self.argv_tps)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tps'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tpmanifest'] != '${talos}/tests/tabswitch/tps.manifest'
+ assert test_config['tppagecycles'] == 5
+ assert test_config['gecko_profile_entries'] == 5000000
+ assert test_config['tploadnocache'] is True
+ assert test_config['preferences'] == {
+ 'addon.test.tabswitch.urlfile': os.path.join('${talos}',
+ 'tests',
+ 'tp5o.html'),
+ 'addon.test.tabswitch.webserver': '${webserver}',
+ 'addon.test.tabswitch.maxurls': -1,
+ }
+ assert test_config['unit'] == 'ms'
+
+ def test_tart_has_expected_attributes(self):
+ config = get_config(self.argv_tart)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tart'
+ assert test_config['tpmanifest'] != '${talos}/tests/tart/tart.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['tploadnocache'] is True
+ assert test_config['tpmozafterpaint'] is False
+ assert test_config['gecko_profile_interval'] == 10
+ assert test_config['gecko_profile_entries'] == 1000000
+ assert 'win_counters' not in test_config
+ assert 'w7_counters' not in test_config
+ assert 'linux_counters' not in test_config
+ assert 'mac_counters' not in test_config
+ assert test_config['preferences'] == {
+ 'layout.frame_rate': 0,
+ 'docshell.event_starvation_delay_hint': 1,
+ 'dom.send_after_paint_to_content': False
+ }
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+
+ def test_cart_has_expected_attributes(self):
+ config = get_config(self.argv_cart)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'cart'
+ assert test_config['tpmanifest'] != '${talos}/tests/tart/cart.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['tploadnocache'] is True
+ assert test_config['tpmozafterpaint'] is False
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 10000000
+ assert 'win_counters' not in test_config
+ assert 'w7_counters' not in test_config
+ assert 'linux_counters' not in test_config
+ assert 'mac_counters' not in test_config
+ assert test_config['preferences'] == {
+ 'layout.frame_rate': 0,
+ 'docshell.event_starvation_delay_hint': 1,
+ 'dom.send_after_paint_to_content': False
+ }
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+
+ def test_damp_has_expected_attributes(self):
+ config = get_config(self.argv_damp)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'damp'
+ assert test_config['tpmanifest'] != '${talos}/tests/devtools/damp.manifest'
+ assert test_config['cycles'] == 5
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 5
+ assert test_config['tploadnocache'] is True
+ assert test_config['tpmozafterpaint'] is False
+ assert test_config['gecko_profile_interval'] == 10
+ assert test_config['gecko_profile_entries'] == 1000000
+ assert 'win_counters' not in test_config
+ assert 'w7_counters' not in test_config
+ assert 'linux_counters' not in test_config
+ assert 'mac_counters' not in test_config
+ assert test_config['filters'] is not None
+ assert test_config['preferences'] == {
+ 'devtools.memory.enabled': True,
+ 'addon.test.damp.webserver': '${webserver}'
+ }
+ assert test_config['unit'] == 'ms'
+
+ def test_glterrain_has_expected_attributes(self):
+ config = get_config(self.argv_glterrain)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'glterrain'
+ assert test_config['tpmanifest'] != '${talos}/tests/webgl/glterrain.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['tploadnocache'] is True
+ assert test_config['tpmozafterpaint'] is False
+ assert test_config['tpchrome'] is False
+ assert test_config['gecko_profile_interval'] == 10
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert 'win_counters' not in test_config
+ assert 'w7_counters' not in test_config
+ assert 'linux_counters' not in test_config
+ assert 'mac_counters' not in test_config
+ assert test_config['preferences'] == {
+ 'layout.frame_rate': 0,
+ 'docshell.event_starvation_delay_hint': 1,
+ 'dom.send_after_paint_to_content': False
+ }
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'frame interval'
+
+ def test_glvideo_has_expected_attributes(self):
+ config = get_config(self.argv_glvideo)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'glvideo'
+ assert test_config['tpmanifest'] != '${talos}/tests/webgl/glvideo.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 5
+ assert test_config['tploadnocache'] is True
+ assert test_config['tpmozafterpaint'] is False
+ assert test_config['tpchrome'] is False
+ assert test_config['gecko_profile_interval'] == 2
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert 'win_counters' not in test_config
+ assert 'w7_counters' not in test_config
+ assert 'linux_counters' not in test_config
+ assert 'mac_counters' not in test_config
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+
+ @mock.patch('talos.config.build_manifest', conftest.patched_build_manifest)
+ def test_tp5n_has_expected_attributes(self):
+ config = get_config(self.argv_tp5n)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tp5n'
+ assert test_config['resolution'] == 20
+ assert test_config['shutdown'] is False
+ assert test_config['tpmanifest'] != '${talos}/tests/tp5n/tp5n.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 1
+ assert test_config['cycles'] == 1
+ assert test_config['tpmozafterpaint'] is True
+ assert test_config['tptimeout'] == 5000
+ assert test_config['mainthread'] is True
+ assert test_config['w7_counters'] == []
+ assert test_config['win_counters'] == []
+ assert test_config['linux_counters'] == []
+ assert test_config['mac_counters'] == []
+ assert test_config['xperf_counters'] == [
+ 'main_startup_fileio', 'main_startup_netio',
+ 'main_normal_fileio', 'main_normal_netio',
+ 'nonmain_startup_fileio', 'nonmain_normal_fileio',
+ 'nonmain_normal_netio', 'mainthread_readcount',
+ 'mainthread_readbytes', 'mainthread_writecount',
+ 'mainthread_writebytes'
+ ]
+ assert test_config['xperf_providers'] == [
+ 'PROC_THREAD', 'LOADER', 'HARD_FAULTS', 'FILENAME',
+ 'FILE_IO', 'FILE_IO_INIT'
+ ]
+ assert test_config['xperf_user_providers'] == [
+ 'Mozilla Generic Provider',
+ 'Microsoft-Windows-TCPIP'
+ ]
+ assert test_config['xperf_stackwalk'] == [
+ 'FileCreate', 'FileRead', 'FileWrite', 'FileFlush',
+ 'FileClose'
+ ]
+ assert test_config['filters'] is not None
+ assert test_config['timeout'] == 1800
+ assert test_config['preferences'] == {
+ 'extensions.enabledScopes': '',
+ 'talos.logfile': 'browser_output.txt'
+ }
+ assert test_config['unit'] == 'ms'
+
+ @mock.patch('talos.config.build_manifest', conftest.patched_build_manifest)
+ def test_tp5o_has_expected_attributes(self):
+ config = get_config(self.argv_tp5o)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tp5o'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['cycles'] == 1
+ assert test_config['tpmozafterpaint'] is True
+ assert test_config['tptimeout'] == 5000
+ assert test_config['mainthread'] is False
+ assert test_config['tpmanifest'] != '${talos}/tests/tp5n/tp5o.manifest'
+ assert test_config['win_counters'] == ['% Processor Time']
+ assert test_config['w7_counters'] == ['% Processor Time']
+ assert test_config['linux_counters'] == ['XRes']
+ assert test_config['mac_counters'] == []
+ assert test_config['responsiveness'] is True
+ assert test_config['gecko_profile_interval'] == 2
+ assert test_config['gecko_profile_entries'] == 4000000
+ assert test_config['filters'] is not None
+ assert test_config['timeout'] == 1800
+ assert test_config['unit'] == 'ms'
+
+ @mock.patch('talos.config.build_manifest', conftest.patched_build_manifest)
+ def test_tp5o_webext_has_expected_attributes(self):
+ config = get_config(self.argv_tp5o_webext)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tp5o_webext'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['cycles'] == 1
+ assert test_config['tpmozafterpaint'] is True
+ assert test_config['tptimeout'] == 5000
+ assert test_config['mainthread'] is False
+ assert test_config['tpmanifest'] != '${talos}/tests/tp5n/tp5o.manifest'
+ assert test_config['win_counters'] == ['% Processor Time']
+ assert test_config['w7_counters'] == ['% Processor Time']
+ assert test_config['linux_counters'] == ['XRes']
+ assert test_config['mac_counters'] == []
+ assert test_config['responsiveness'] is True
+ assert test_config['gecko_profile_interval'] == 2
+ assert test_config['gecko_profile_entries'] == 4000000
+ assert test_config['filters'] is not None
+ assert test_config['timeout'] == 1800
+ assert test_config['unit'] == 'ms'
+ assert test_config['webextensions'] == '${talos}/webextensions/dummy/dummy-signed.xpi'
+ assert test_config['preferences'] == {'xpinstall.signatures.required': False}
+
+ @mock.patch('talos.config.build_manifest', conftest.patched_build_manifest)
+ def test_tp5o_scroll_has_expected_attributes(self):
+ config = get_config(self.argv_tp5o_scroll)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tp5o_scroll'
+ assert test_config['tpmanifest'] != '${talos}/tests/tp5n/tp5o.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 12
+ assert test_config['gecko_profile_interval'] == 2
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert test_config['tpscrolltest'] is True
+ assert test_config['tpmozafterpaint'] is False
+ assert test_config['preferences'] == {
+ 'layout.frame_rate': 0,
+ 'docshell.event_starvation_delay_hint': 1,
+ 'dom.send_after_paint_to_content': False,
+ 'layout.css.scroll-behavior.spring-constant': "'10'",
+ 'toolkit.framesRecording.bufferSize': 10000
+ }
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == '1/FPS'
+
+ def test_v8_7_has_expected_attributes(self):
+ config = get_config(self.argv_v8_7)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'v8_7'
+ assert test_config['tpmanifest'] != '${talos}/tests/v8_7/v8.manifest'
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 1000000
+ assert test_config['tpcycles'] == 1
+ assert test_config['resolution'] == 20
+ assert test_config['tpmozafterpaint'] is False
+ assert test_config['preferences'] == {'dom.send_after_paint_to_content': False}
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'score'
+ assert test_config['lower_is_better'] is False
+
+ def test_kraken_has_expected_attributes(self):
+ config = get_config(self.argv_kraken)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'kraken'
+ assert test_config['tpmanifest'] != '${talos}/tests/kraken/kraken.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 1
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 5000000
+ assert test_config['tpmozafterpaint'] is False
+ assert test_config['tpchrome'] is False
+ assert test_config['preferences'] == {'dom.send_after_paint_to_content': False}
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'score'
+
+ def test_basic_compositor_video_has_expected_attributes(self):
+ config = get_config(self.argv_basic_compositor_video)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'basic_compositor_video'
+ assert test_config['tpmanifest'] != '${talos}/tests/video/video.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 12
+ assert test_config['tpchrome'] is False
+ assert test_config['timeout'] == 10000
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert test_config['preferences'] == {
+ 'full-screen-api.allow-trusted-requests-only': False,
+ 'layers.acceleration.force-enabled': False,
+ 'layers.acceleration.disabled': True,
+ 'layout.frame_rate': 0,
+ 'docshell.event_starvation_delay_hint': 1,
+ 'full-screen-api.warning.timeout': 500,
+ 'media.ruin-av-sync.enabled': True
+ }
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms/frame'
+ assert test_config['lower_is_better'] is True
+
+ def test_tcanvasmark_has_expected_attributes(self):
+ config = get_config(self.argv_tcanvasmark)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tcanvasmark'
+ assert test_config['tpmanifest'] != '${talos}/tests/canvasmark/canvasmark.manifest'
+ assert 'win_counters' not in test_config
+ assert 'w7_counters' not in test_config
+ assert 'linux_counters' not in test_config
+ assert 'mac_counters' not in test_config
+ assert test_config['tpcycles'] == 5
+ assert test_config['tppagecycles'] == 1
+ assert test_config['timeout'] == 900
+ assert test_config['gecko_profile_interval'] == 10
+ assert test_config['gecko_profile_entries'] == 2500000
+ assert test_config['tpmozafterpaint'] is False
+ assert test_config['preferences'] == {'dom.send_after_paint_to_content': False}
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'score'
+ assert test_config['lower_is_better'] is False
+
+ def test_dromaeo_css_has_expected_attributes(self):
+ config = get_config(self.argv_dromaeo_css)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'dromaeo_css'
+ assert test_config['tpcycles'] == 1
+ assert test_config['filters'] is not None
+ assert test_config['lower_is_better'] is False
+ assert test_config['alert_threshold'] == 5.0
+ assert test_config['tpchrome'] is False
+ assert test_config['gecko_profile_interval'] == 2
+ assert test_config['gecko_profile_entries'] == 10000000
+ assert test_config['tpmanifest'] != '${talos}/tests/dromaeo/css.manifest'
+ assert test_config['unit'] == 'score'
+
+ def test_dromaeo_dom_has_expected_attributes(self):
+ config = get_config(self.argv_dromaeo_dom)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'dromaeo_dom'
+ assert test_config['tpcycles'] == 1
+ assert test_config['filters'] is not None
+ assert test_config['lower_is_better'] is False
+ assert test_config['alert_threshold'] == 5.0
+ assert test_config['tpchrome'] is False
+ assert test_config['gecko_profile_interval'] == 2
+ assert test_config['gecko_profile_entries'] == 10000000
+ assert test_config['tpmanifest'] != '${talos}/tests/dromaeo/dom.manifest'
+ assert test_config['unit'] == 'score'
+
+ def test_tsvgm_has_expected_attributes(self):
+ config = get_config(self.argv_tsvgm)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tsvgm'
+ assert test_config['tpmanifest'] != '${talos}/tests/svgx/svgm.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 7
+ assert test_config['tpmozafterpaint'] is False
+ assert test_config['tpchrome'] is False
+ assert test_config['gecko_profile_interval'] == 10
+ assert test_config['gecko_profile_entries'] == 1000000
+ assert test_config['preferences'] == {
+ 'layout.frame_rate': 0,
+ 'docshell.event_starvation_delay_hint': 1,
+ 'dom.send_after_paint_to_content': False
+ }
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+
+ def test_tsvgx_has_expected_attributes(self):
+ config = get_config(self.argv_tsvgx)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tsvgx'
+ assert test_config['tpmanifest'] != '${talos}/tests/svgx/svgx.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['tpmozafterpaint'] is False
+ assert test_config['tpchrome'] is False
+ assert test_config['gecko_profile_interval'] == 10
+ assert test_config['gecko_profile_entries'] == 1000000
+ assert test_config['preferences'] == {
+ 'layout.frame_rate': 0,
+ 'docshell.event_starvation_delay_hint': 1,
+ 'dom.send_after_paint_to_content': False
+ }
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+
+ def test_tsvg_static_has_expected_attributes(self):
+ config = get_config(self.argv_tsvg_static)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tsvg_static'
+ assert test_config['tpmanifest'] != '${talos}/tests/svg_static/svg_static.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['tpmozafterpaint'] is True
+ assert test_config['tpchrome'] is False
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 10000000
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+
+ def test_tsvgr_opacity_has_expected_attributes(self):
+ config = get_config(self.argv_tsvgr_opacity)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tsvgr_opacity'
+ assert test_config['tpmanifest'] != '${talos}/tests/svg_opacity/svg_opacity.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['tpmozafterpaint'] is True
+ assert test_config['tpchrome'] is False
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 10000000
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+
+ def test_tscrollx_has_expected_attributes(self):
+ config = get_config(self.argv_tscrollx)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tscrollx'
+ assert test_config['tpmanifest'] != '${talos}/tests/scroll/scroll.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['tpmozafterpaint'] is False
+ assert test_config['tpchrome'] is False
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 1000000
+ assert test_config['preferences'] == {
+ 'layout.frame_rate': 0,
+ 'docshell.event_starvation_delay_hint': 1,
+ 'dom.send_after_paint_to_content': False,
+ 'layout.css.scroll-behavior.spring-constant': "'10'",
+ 'toolkit.framesRecording.bufferSize': 10000
+ }
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+
+ def test_a11yr_has_expect_attributes(self):
+ config = get_config(self.argv_a11yr)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'a11yr'
+ assert test_config['tpmanifest'] != '${talos}/tests/a11y/a11y.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['tpmozafterpaint'] is True
+ assert test_config['tpchrome'] is False
+ assert test_config['preferences'] == {'dom.send_after_paint_to_content': False}
+ assert test_config['unit'] == 'ms'
+ assert test_config['alert_threshold'] == 5.0
+
+ def test_speedometer_has_expected_attributes(self):
+ config = get_config(self.argv_speedometer)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'speedometer'
+ assert test_config['tpmanifest'] != '${talos}/tests/speedometer/speedometer.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 5
+ assert test_config['tpmozafterpaint'] is False
+ assert test_config['tpchrome'] is False
+ assert test_config['format_pagename'] is False
+ assert test_config['lower_is_better'] is False
+ assert test_config['unit'] == 'score'
+
+ def test_perf_reftest_has_expected_attributes(self):
+ config = get_config(self.argv_perf_reftest)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'perf_reftest'
+ assert test_config['base_vs_ref'] is True
+ assert test_config['tpmanifest'] != '${talos}/tests/perf-reftest/perf_reftest.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 10
+ assert test_config['tptimeout'] == 30000
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+ assert test_config['lower_is_better'] is True
+ assert test_config['alert_threshold'] == 5.0
+
+ def test_perf_reftest_singletons_has_expected_attributes(self):
+ config = get_config(self.argv_perf_reftest_singletons)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'perf_reftest_singletons'
+ assert test_config['tpmanifest'] != \
+ '${talos}/tests/perf-reftest-singletons/perf_reftest_singletons.manifest'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 15
+ assert test_config['tptimeout'] == 30000
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+ assert test_config['lower_is_better'] is True
+ assert test_config['alert_threshold'] == 5.0
+
+ def test_tp6_google_has_expected_attributes(self):
+ config = get_config(self.argv_tp6_google)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tp6_google'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+ assert test_config['lower_is_better'] is True
+ assert test_config['fnbpaint'] is True
+ assert test_config['tpmanifest'] != \
+ '${talos}/tests/quantum_pageload/quantum_pageload_google.manifest'
+
+ def test_tp6_google_heavy_has_expected_attributes(self):
+ config = get_config(self.argv_tp6_google_heavy)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tp6_google_heavy'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+ assert test_config['lower_is_better'] is True
+ assert test_config['fnbpaint'] is True
+ assert test_config['profile'] == 'simple'
+
+ def test_tp6_youtube_has_expected_attributes(self):
+ config = get_config(self.argv_tp6_youtube)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tp6_youtube'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+ assert test_config['lower_is_better'] is True
+ assert test_config['fnbpaint'] is True
+ assert test_config['tpmanifest'] != \
+ '${talos}/tests/quantum_pageload/quantum_pageload_youtube.manifest'
+
+ def test_tp6_youtube_heavy_has_expected_attributes(self):
+ config = get_config(self.argv_tp6_youtube_heavy)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tp6_youtube_heavy'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+ assert test_config['lower_is_better'] is True
+ assert test_config['fnbpaint'] is True
+ assert test_config['profile'] == 'simple'
+
+ def test_tp6_amazon_has_expected_attributes(self):
+ config = get_config(self.argv_tp6_amazon)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tp6_amazon'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+ assert test_config['lower_is_better'] is True
+ assert test_config['fnbpaint'] is True
+ assert test_config['tpmanifest'] != \
+ '${talos}/tests/quantum_pageload/quantum_pageload_amazon.manifest'
+
+ def test_tp6_amazon_heavy_has_expected_attributes(self):
+ config = get_config(self.argv_tp6_amazon_heavy)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tp6_amazon_heavy'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+ assert test_config['lower_is_better'] is True
+ assert test_config['fnbpaint'] is True
+ assert test_config['profile'] == 'simple'
+
+ def test_tp6_facebook_has_expected_attributes(self):
+ config = get_config(self.argv_tp6_facebook)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tp6_facebook'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+ assert test_config['lower_is_better'] is True
+ assert test_config['fnbpaint'] is True
+ assert test_config['tpmanifest'] != \
+ '${talos}/tests/quantum_pageload/quantum_pageload_facebook.manifest'
+
+ def test_tp6_facebook_heavy_has_expected_attributes(self):
+ config = get_config(self.argv_tp6_facebook_heavy)
+ test_config = config['tests'][0]
+
+ assert test_config['name'] == 'tp6_facebook_heavy'
+ assert test_config['tpcycles'] == 1
+ assert test_config['tppagecycles'] == 25
+ assert test_config['gecko_profile_interval'] == 1
+ assert test_config['gecko_profile_entries'] == 2000000
+ assert test_config['filters'] is not None
+ assert test_config['unit'] == 'ms'
+ assert test_config['lower_is_better'] is True
+ assert test_config['fnbpaint'] is True
+ assert test_config['profile'] == 'simple'
+
+
+@mock.patch('talos.config.get_browser_config')
+@mock.patch('talos.config.get_config')
+def test_get_configs(get_config_mock, get_browser_config_mock):
+ # unpacks in right order
+ get_config_mock.return_value = 'first'
+ get_browser_config_mock.return_value = 'second'
+
+ first, second = get_configs()
+ assert (first, second) == ('first', 'second')
+
+
+if __name__ == '__main__':
+ mozunit.main()