--- a/testing/mozharness/examples/action_config_script.py
+++ b/testing/mozharness/examples/action_config_script.py
@@ -14,32 +14,30 @@ from mozharness.base.script import BaseS
# ActionsConfigExample {{{1
class ActionsConfigExample(BaseScript):
config_options = [[
['--beverage', ],
{"action": "store",
"dest": "beverage",
- "type": "string",
"help": "Specify your beverage of choice",
}
], [
['--ship-style', ],
{"action": "store",
"dest": "ship_style",
- "type": "choice",
"choices": ["1", "2", "3"],
"help": "Specify the type of ship",
}
], [
['--long-sleep-time', ],
{"action": "store",
"dest": "long_sleep_time",
- "type": "int",
+ "type": int,
"help": "Specify how long to sleep",
}
]]
def __init__(self, require_config_file=False):
super(ActionsConfigExample, self).__init__(
config_options=self.config_options,
all_actions=[
--- a/testing/mozharness/mozharness/base/config.py
+++ b/testing/mozharness/mozharness/base/config.py
@@ -3,70 +3,53 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""Generic config parsing and dumping, the way I remember it from scripts
gone by.
The config should be built from script-level defaults, overlaid by
-config-file defaults, overlaid by command line options.
+config-file defaults, overlaid by command line arguments.
(For buildbot-analogues that would be factory-level defaults,
builder-level defaults, and build request/scheduler settings.)
The config should then be locked (set to read-only, to prevent runtime
alterations). Afterwards we should dump the config to a file that is
uploaded with the build, and can be used to debug or replicate the build
at a later time.
TODO:
* check_required_settings or something -- run at init, assert that
these settings are set.
"""
+from argparse import ArgumentParser, Action
from copy import deepcopy
-from optparse import OptionParser, Option, OptionGroup
import os
import sys
import urllib2
import socket
import time
try:
import simplejson as json
except ImportError:
import json
from mozharness.base.log import DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL
-# optparse {{{1
-class ExtendedOptionParser(OptionParser):
- """OptionParser, but with ExtendOption as the option_class.
- """
- def __init__(self, **kwargs):
- kwargs['option_class'] = ExtendOption
- OptionParser.__init__(self, **kwargs)
-
-
-class ExtendOption(Option):
- """from http://docs.python.org/library/optparse.html?highlight=optparse#adding-new-actions"""
- ACTIONS = Option.ACTIONS + ("extend",)
- STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",)
- TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",)
- ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + ("extend",)
-
- def take_action(self, action, dest, opt, value, values, parser):
- if action == "extend":
- lvalue = value.split(",")
- values.ensure_value(dest, []).extend(lvalue)
- else:
- Option.take_action(
- self, action, dest, opt, value, values, parser)
+# argparse {{{1
+class ExtendAction(Action):
+ def __call__(self, parser, namespace, values, option_string=None):
+ items = getattr(namespace, self.dest) or []
+ items.extend(values.split(','))
+ setattr(namespace, self.dest, items)
def make_immutable(item):
if isinstance(item, list) or isinstance(item, tuple):
result = LockedTuple(item)
elif isinstance(item, dict):
result = ReadOnlyDict(item)
result.lock()
@@ -208,17 +191,17 @@ def download_config_file(url, file_name)
class BaseConfig(object):
"""Basic config setting/getting.
"""
def __init__(self, config=None, initial_config_file=None, config_options=None,
all_actions=None, default_actions=None,
volatile_config=None, option_args=None,
require_config_file=False,
append_env_variables_from_configs=False,
- usage="usage: %prog [options]"):
+ usage=None):
self._config = {}
self.all_cfg_files_and_dicts = []
self.actions = []
self.config_lock = False
self.require_config_file = require_config_file
# It allows to append env variables from multiple config files
self.append_env_variables_from_configs = append_env_variables_from_configs
@@ -243,142 +226,127 @@ class BaseConfig(object):
self.set_config(config)
if initial_config_file:
initial_config = parse_config_file(initial_config_file)
self.all_cfg_files_and_dicts.append(
(initial_config_file, initial_config)
)
self.set_config(initial_config)
# Since initial_config_file is only set when running unit tests,
- # if no option_args have been specified, then the parser will
- # parse sys.argv which in this case would be the command line
- # options specified to run the tests, e.g. nosetests -v. Clearly,
- # the options passed to nosetests (such as -v) should not be
- # interpreted by mozharness as mozharness options, so we specify
- # a dummy command line with no options, so that the parser does
- # not add anything from the test invocation command line
- # arguments to the mozharness options.
+ # default option_args to [] to avoid parsing sys.argv (which are
+ # specified for nosetests).
if option_args is None:
- option_args=['dummy_mozharness_script_with_no_command_line_options.py']
+ option_args = []
if config_options is None:
config_options = []
self._create_config_parser(config_options, usage)
# we allow manually passing of option args for things like nosetests
self.parse_args(args=option_args)
def get_read_only_config(self):
return ReadOnlyDict(self._config)
def _create_config_parser(self, config_options, usage):
- self.config_parser = ExtendedOptionParser(usage=usage)
- self.config_parser.add_option(
- "--work-dir", action="store", dest="work_dir",
- type="string", default="build",
+ self.config_parser = ArgumentParser(usage=usage)
+ self.config_parser.register('action', 'extend', ExtendAction)
+ self.config_parser.add_argument(
+ "--work-dir", default="build",
help="Specify the work_dir (subdir of base_work_dir)"
)
- self.config_parser.add_option(
- "--base-work-dir", action="store", dest="base_work_dir",
- type="string", default=os.getcwd(),
+ self.config_parser.add_argument(
+ "--base-work-dir", default=os.getcwd(),
help="Specify the absolute path of the parent of the working directory"
)
- self.config_parser.add_option(
+ self.config_parser.add_argument(
"-c", "--config-file", "--cfg", action="extend", dest="config_files",
- type="string", help="Specify a config file; can be repeated"
+ help="Specify a config file; can be repeated"
)
- self.config_parser.add_option(
+ self.config_parser.add_argument(
"-C", "--opt-config-file", "--opt-cfg", action="extend",
- dest="opt_config_files", type="string", default=[],
+ dest="opt_config_files", default=[],
help="Specify an optional config file, like --config-file but with no "
"error if the file is missing; can be repeated"
)
- self.config_parser.add_option(
+ self.config_parser.add_argument(
"--dump-config", action="store_true",
- dest="dump_config",
help="List and dump the config generated from this run to "
"a JSON file."
)
- self.config_parser.add_option(
+ self.config_parser.add_argument(
"--dump-config-hierarchy", action="store_true",
- dest="dump_config_hierarchy",
help="Like --dump-config but will list and dump which config "
"files were used making up the config and specify their own "
"keys/values that were not overwritten by another cfg -- "
"held the highest hierarchy."
)
# Logging
- log_option_group = OptionGroup(self.config_parser, "Logging")
- log_option_group.add_option(
- "--log-level", action="store",
- type="choice", dest="log_level", default=INFO,
+ log_option_group = self.config_parser.add_argument_group("Logging")
+ log_option_group.add_argument(
+ "--log-level", default=INFO,
choices=[DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL],
help="Set log level (debug|info|warning|error|critical|fatal)"
)
- log_option_group.add_option(
+ log_option_group.add_argument(
"-q", "--quiet", action="store_false", dest="log_to_console",
default=True, help="Don't log to the console"
)
- log_option_group.add_option(
- "--append-to-log", action="store_true",
- dest="append_to_log", default=False,
+ log_option_group.add_argument(
+ "--append-to-log", action="store_true", default=False,
help="Append to the log"
)
- log_option_group.add_option(
+ log_option_group.add_argument(
"--multi-log", action="store_const", const="multi",
dest="log_type", help="Log using MultiFileLogger"
)
- log_option_group.add_option(
+ log_option_group.add_argument(
"--simple-log", action="store_const", const="simple",
dest="log_type", help="Log using SimpleFileLogger"
)
- self.config_parser.add_option_group(log_option_group)
# Actions
- action_option_group = OptionGroup(
- self.config_parser, "Actions",
- "Use these options to list or enable/disable actions."
- )
- action_option_group.add_option(
+ action_option_group = self.config_parser.add_argument_group(
+ "Actions", "Use these options to list or enable/disable actions.")
+ action_option_group.add_argument(
"--list-actions", action="store_true",
- dest="list_actions",
help="List all available actions, then exit"
)
- action_option_group.add_option(
+ action_option_group.add_argument(
"--add-action", action="extend",
dest="add_actions", metavar="ACTIONS",
help="Add action %s to the list of actions" % self.all_actions
)
- action_option_group.add_option(
+ action_option_group.add_argument(
"--no-action", action="extend",
dest="no_actions", metavar="ACTIONS",
help="Don't perform action"
)
for action in self.all_actions:
- action_option_group.add_option(
+ action_option_group.add_argument(
"--%s" % action, action="append_const",
dest="actions", const=action,
help="Add %s to the limited list of actions" % action
)
- action_option_group.add_option(
+ action_option_group.add_argument(
"--no-%s" % action, action="append_const",
dest="no_actions", const=action,
help="Remove %s from the list of actions to perform" % action
)
- self.config_parser.add_option_group(action_option_group)
+
# Child-specified options
# TODO error checking for overlapping options
if config_options:
for option in config_options:
- self.config_parser.add_option(*option[0], **option[1])
+ self.config_parser.add_argument(*option[0], **option[1])
# Initial-config-specified options
config_options = self._config.get('config_options', None)
if config_options:
for option in config_options:
- self.config_parser.add_option(*option[0], **option[1])
+ self.config_parser.add_argument(*option[0], **option[1])
def set_config(self, config, overwrite=False):
"""This is probably doable some other way."""
if self._config and not overwrite:
self._config.update(config)
else:
self._config = config
return self._config
@@ -409,25 +377,25 @@ class BaseConfig(object):
raise SystemExit(-1)
def list_actions(self):
print "Actions available:"
for a in self.all_actions:
print " " + ("*" if a in self.default_actions else " "), a
raise SystemExit(0)
- def get_cfgs_from_files(self, all_config_files, options):
+ def get_cfgs_from_files(self, all_config_files, args):
"""Returns the configuration derived from the list of configuration
files. The result is represented as a list of `(filename,
config_dict)` tuples; they will be combined with keys in later
dictionaries taking precedence over earlier.
`all_config_files` is all files specified with `--config-file` and
- `--opt-config-file`; `options` is the argparse options object giving
- access to any other command-line options.
+ `--opt-config-file`; `args` is the argparse Namespace object giving
+ access to any other command-line arguments.
This function is also responsible for downloading any configuration
files specified by URL. It uses ``parse_config_file`` in this module
to parse individual files.
This method can be overridden in a subclass to add extra logic to the
way that self.config is made up. See
`mozharness.mozilla.building.buildbase.BuildingConfig` for an example.
@@ -440,50 +408,48 @@ class BaseConfig(object):
file_path = os.path.join(os.getcwd(), file_name)
download_config_file(cf, file_path)
all_cfg_files_and_dicts.append(
(file_path, parse_config_file(file_path))
)
else:
all_cfg_files_and_dicts.append((cf, parse_config_file(cf)))
except Exception:
- if cf in options.opt_config_files:
+ if cf in args.opt_config_files:
print(
"WARNING: optional config file not found %s" % cf
)
else:
raise
return all_cfg_files_and_dicts
def parse_args(self, args=None):
"""Parse command line arguments in a generic way.
- Return the parser object after adding the basic options, so
+ Return the parser object after adding the basic arguments, so
child objects can manipulate it.
"""
self.command_line = ' '.join(sys.argv)
if args is None:
args = sys.argv[1:]
- (options, args) = self.config_parser.parse_args(args)
-
- defaults = self.config_parser.defaults.copy()
+ args = self.config_parser.parse_args(args)
- if not options.config_files:
+ if not args.config_files:
if self.require_config_file:
- if options.list_actions:
+ if args.list_actions:
self.list_actions()
print("Required config file not set! (use --config-file option)")
raise SystemExit(-1)
else:
# this is what get_cfgs_from_files returns. It will represent each
# config file name and its assoctiated dict
# eg ('builds/branch_specifics.py', {'foo': 'bar'})
# let's store this to self for things like --interpret-config-files
self.all_cfg_files_and_dicts.extend(self.get_cfgs_from_files(
# append opt_config to allow them to overwrite previous configs
- options.config_files + options.opt_config_files, options=options
+ args.config_files + args.opt_config_files, args
))
config = {}
if self.append_env_variables_from_configs:
# We only append values from various configs for the 'env' entry
# For everything else we follow the standard behaviour
for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts):
for v in c_dict.keys():
if v == 'env' and v in config:
@@ -494,53 +460,52 @@ class BaseConfig(object):
for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts):
config.update(c_dict)
# assign or update self._config depending on if it exists or not
# NOTE self._config will be passed to ReadOnlyConfig's init -- a
# dict subclass with immutable locking capabilities -- and serve
# as the keys/values that make up that instance. Ultimately,
# this becomes self.config during BaseScript's init
self.set_config(config)
- for key in defaults.keys():
- value = getattr(options, key)
+
+ for key, value in vars(args).items():
if value is None:
continue
# Don't override config_file defaults with config_parser defaults
- if key in defaults and value == defaults[key] and key in self._config:
+ if value == self.config_parser.get_default(key) and key in self._config:
continue
self._config[key] = value
# The idea behind the volatile_config is we don't want to save this
# info over multiple runs. This defaults to the action-specific
- # config options, but can be anything.
+ # config args, but can be anything.
for key in self.volatile_config.keys():
if self._config.get(key) is not None:
self.volatile_config[key] = self._config[key]
del(self._config[key])
self.update_actions()
- if options.list_actions:
+ if args.list_actions:
self.list_actions()
# Keep? This is for saving the volatile config in the dump_config
self._config['volatile_config'] = self.volatile_config
- self.options = options
self.args = args
- return (self.options, self.args)
+ return self.args
def update_actions(self):
""" Update actions after reading in config.
Seems a little complex, but the logic goes:
First, if default_actions is specified in the config, set our
default actions even if the script specifies other default actions.
- Without any other action-specific options, run with default actions.
+ Without any other action-specific arguments, run with default actions.
If we specify --ACTION or --only-ACTION once or multiple times,
we want to override the default_actions list with the one(s) we list.
Otherwise, if we specify --add-action ACTION, we want to add an
action to the list.
Finally, if we specify --no-ACTION, remove that from the list of
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -15,16 +15,17 @@ import json
import os
import pprint
import subprocess
import time
import uuid
import copy
import glob
import shlex
+from argparse import Action
from itertools import chain
# import the power of mozharness ;)
import sys
from datetime import datetime
import re
from mozharness.base.config import BaseConfig, parse_config_file
from mozharness.base.log import ERROR, OutputParser, FATAL
@@ -322,17 +323,17 @@ class BuildingConfig(BaseConfig):
build_pool_configs = parse_config_file(pool_cfg_file)
all_config_dicts.append(
(pool_cfg_file, build_pool_configs[options.build_pool])
)
return all_config_dicts
# noinspection PyUnusedLocal
-class BuildOptionParser(object):
+class BuildOptionParser(Action):
# TODO add nosetests for this class
platform = None
bits = None
config_file_search_path = [
'.', os.path.join(sys.path[0], '..', 'configs'),
os.path.join(sys.path[0], '..', '..', 'configs')
]
@@ -383,16 +384,21 @@ class BuildOptionParser(object):
'artifact': 'builds/releng_sub_%s_configs/%s_artifact.py',
'debug-artifact': 'builds/releng_sub_%s_configs/%s_debug_artifact.py',
'devedition': 'builds/releng_sub_%s_configs/%s_devedition.py',
'dmd': 'builds/releng_sub_%s_configs/%s_dmd.py',
}
build_pool_cfg_file = 'builds/build_pool_specifics.py'
branch_cfg_file = 'builds/branch_specifics.py'
+ def __call__(self, parser, namespace, values, option_string=None):
+ func = getattr(self, 'set_{}'.format(self.dest))
+ func(self, namespace, values)
+
+
@classmethod
def _query_pltfrm_and_bits(cls, target_option, options):
""" determine platform and bits
This can be from either from a supplied --platform and --bits
or parsed from given config file names.
"""
error_msg = (
@@ -434,21 +440,21 @@ class BuildOptionParser(object):
cls.platform = 'android'
break
else:
sys.exit(error_msg % (target_option, 'platform', '--platform',
'"linux", "windows", "mac", or "android"'))
return cls.bits, cls.platform
@classmethod
- def find_variant_cfg_path(cls, opt, value, parser):
+ def find_variant_cfg_path(cls, opt, value, namespace):
valid_variant_cfg_path = None
# first let's see if we were given a valid short-name
if cls.build_variants.get(value):
- bits, pltfrm = cls._query_pltfrm_and_bits(opt, parser.values)
+ bits, pltfrm = cls._query_pltfrm_and_bits(opt, namespace)
prospective_cfg_path = cls.build_variants[value] % (pltfrm, bits)
else:
# this is either an incomplete path or an invalid key in
# build_variants
prospective_cfg_path = value
if os.path.exists(prospective_cfg_path):
# now let's see if we were given a valid pathname
@@ -460,120 +466,110 @@ class BuildOptionParser(object):
if os.path.exists(os.path.join(path, prospective_cfg_path)):
# success! we found a config file
valid_variant_cfg_path = os.path.join(path,
prospective_cfg_path)
break
return valid_variant_cfg_path, prospective_cfg_path
@classmethod
- def set_build_variant(cls, option, opt, value, parser):
+ def set_build_variant(cls, option, namespace, value):
""" sets an extra config file.
This is done by either taking an existing filepath or by taking a valid
shortname coupled with known platform/bits.
"""
valid_variant_cfg_path, prospective_cfg_path = cls.find_variant_cfg_path(
- '--custom-build-variant-cfg', value, parser)
+ '--custom-build-variant-cfg', value, namespace)
if not valid_variant_cfg_path:
# either the value was an indeterminable path or an invalid short
# name
sys.exit("Whoops!\n'--custom-build-variant' was passed but an "
"appropriate config file could not be determined. Tried "
"using: '%s' but it was either not:\n\t-- a valid "
"shortname: %s \n\t-- a valid path in %s \n\t-- a "
"valid variant for the given platform and bits." % (
prospective_cfg_path,
str(cls.build_variants.keys()),
str(cls.config_file_search_path)))
- parser.values.config_files.append(valid_variant_cfg_path)
- setattr(parser.values, option.dest, value) # the pool
+ namespace.config_files.append(valid_variant_cfg_path)
+ setattr(namespace, option.dest, value) # the pool
@classmethod
- def set_build_pool(cls, option, opt, value, parser):
+ def set_build_pool(cls, option, namespace, value):
# first let's add the build pool file where there may be pool
# specific keys/values. Then let's store the pool name
- parser.values.config_files.append(cls.build_pool_cfg_file)
- setattr(parser.values, option.dest, value) # the pool
+ namespace.config_files.append(cls.build_pool_cfg_file)
+ setattr(namespace, option.dest, value) # the pool
@classmethod
- def set_build_branch(cls, option, opt, value, parser):
+ def set_branch(cls, option, namespace, value):
# first let's add the branch_specific file where there may be branch
# specific keys/values. Then let's store the branch name we are using
- parser.values.config_files.append(cls.branch_cfg_file)
- setattr(parser.values, option.dest, value) # the branch name
+ namespace.config_files.append(cls.branch_cfg_file)
+ setattr(namespace, option.dest, value) # the branch name
@classmethod
- def set_platform(cls, option, opt, value, parser):
+ def set_platform(cls, option, namespace, value):
cls.platform = value
- setattr(parser.values, option.dest, value)
+ setattr(namespace, option.dest, value)
@classmethod
- def set_bits(cls, option, opt, value, parser):
+ def set_bits(cls, option, namespace, value):
cls.bits = value
- setattr(parser.values, option.dest, value)
+ setattr(namespace, option.dest, value)
# this global depends on BuildOptionParser and therefore can not go at the
# top of the file
BUILD_BASE_CONFIG_OPTIONS = [
[['--developer-run', '--skip-buildbot-actions'], {
"action": "store_false",
"dest": "is_automation",
"default": True,
"help": "If this is running outside of Mozilla's build"
"infrastructure, use this option. It ignores actions"
"that are not needed and adds config checks."}],
[['--platform'], {
- "action": "callback",
- "callback": BuildOptionParser.set_platform,
- "type": "string",
+ "action": BuildOptionParser,
"dest": "platform",
"help": "Sets the platform we are running this against"
" valid values: 'windows', 'mac', 'linux'"}],
[['--bits'], {
- "action": "callback",
- "callback": BuildOptionParser.set_bits,
- "type": "string",
+ "action": BuildOptionParser,
"dest": "bits",
"help": "Sets which bits we are building this against"
" valid values: '32', '64'"}],
[['--custom-build-variant-cfg'], {
- "action": "callback",
- "callback": BuildOptionParser.set_build_variant,
- "type": "string",
+ "action": BuildOptionParser,
"dest": "build_variant",
"help": "Sets the build type and will determine appropriate"
" additional config to use. Either pass a config path"
" or use a valid shortname from: "
"%s" % (BuildOptionParser.build_variants.keys(),)}],
[['--build-pool'], {
- "action": "callback",
- "callback": BuildOptionParser.set_build_pool,
- "type": "string",
+ "action": BuildOptionParser,
"dest": "build_pool",
"help": "This will update the config with specific pool"
" environment keys/values. The dicts for this are"
" in %s\nValid values: staging or"
" production" % ('builds/build_pool_specifics.py',)}],
[['--branch'], {
- "action": "callback",
- "callback": BuildOptionParser.set_build_branch,
- "type": "string",
+ "action": BuildOptionParser,
"dest": "branch",
"help": "This sets the branch we will be building this for."
" If this branch is in branch_specifics.py, update our"
" config with specific keys/values from that. See"
" %s for possibilites" % (
BuildOptionParser.branch_cfg_file,
)}],
[['--scm-level'], {
"action": "store",
- "type": "int",
+ "type": int,
"dest": "scm_level",
"default": 1,
"help": "This sets the SCM level for the branch being built."
" See https://www.mozilla.org/en-US/about/"
"governance/policies/commit/access-policy/"}],
[['--enable-pgo'], {
"action": "store_true",
"dest": "pgo_build",
--- a/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py
+++ b/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py
@@ -26,60 +26,53 @@ class MultiLocaleBuild(LocalesMixin, Mer
""" This class targets Fennec multilocale builds.
We were considering this for potential Firefox desktop multilocale.
Now that we have a different approach for B2G multilocale,
it's most likely misnamed. """
config_options = [[
["--locale"],
{"action": "extend",
"dest": "locales",
- "type": "string",
"help": "Specify the locale(s) to repack"
}
], [
["--objdir"],
{"action": "store",
"dest": "objdir",
- "type": "string",
"default": "objdir",
"help": "Specify the objdir"
}
], [
["--l10n-base"],
{"action": "store",
"dest": "hg_l10n_base",
- "type": "string",
"help": "Specify the L10n repo base directory"
}
], [
["--l10n-tag"],
{"action": "store",
"dest": "hg_l10n_tag",
- "type": "string",
"help": "Specify the L10n tag"
}
], [
["--tag-override"],
{"action": "store",
"dest": "tag_override",
- "type": "string",
"help": "Override the tags set for all repos"
}
], [
["--user-repo-override"],
{"action": "store",
"dest": "user_repo_override",
- "type": "string",
"help": "Override the user repo path for all repos"
}
], [
["--l10n-dir"],
{"action": "store",
"dest": "l10n_dir",
- "type": "string",
"default": "l10n",
"help": "Specify the l10n dir name"
}
]]
def __init__(self, require_config_file=True):
LocalesMixin.__init__(self)
MercurialScript.__init__(self, config_options=self.config_options,
--- a/testing/mozharness/mozharness/mozilla/testing/device.py
+++ b/testing/mozharness/mozharness/mozilla/testing/device.py
@@ -435,27 +435,25 @@ device_config_options = [[
["--device-heartbeat-port"],
{"action": "store",
"dest": "device_heartbeat_port",
"help": "Specify the heartbeat port of the device."
}
], [
["--device-protocol"],
{"action": "store",
- "type": "choice",
"dest": "device_protocol",
"choices": DEVICE_PROTOCOL_DICT.keys(),
"help": "Specify the device communication protocol."
}
], [
["--device-type"],
# A bit useless atm, but we can add new device types as we add support
# for them.
{"action": "store",
- "type": "choice",
"choices": ["non-tegra", "tegra250"],
"default": "non-tegra",
"dest": "device_type",
"help": "Specify the device type."
}
], [
["--devicemanager-path"],
{"action": "store",
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -114,17 +114,16 @@ class Talos(TestingMixin, MercurialScrip
[["--branch-name"],
{"action": "store",
"dest": "branch",
"help": "Graphserver branch to report to"
}],
[["--system-bits"],
{"action": "store",
"dest": "system_bits",
- "type": "choice",
"default": "32",
"choices": ['32', '64'],
"help": "Testing 32 or 64 (for talos json plugins)"
}],
[["--add-option"],
{"action": "extend",
"dest": "talos_extra_options",
"default": None,
@@ -133,17 +132,17 @@ class Talos(TestingMixin, MercurialScrip
[["--geckoProfile"], {
"dest": "gecko_profile",
"action": "store_true",
"default": False,
"help": "Whether or not to profile the test run and save the profile results"
}],
[["--geckoProfileInterval"], {
"dest": "gecko_profile_interval",
- "type": "int",
+ "type": int,
"default": 0,
"help": "The interval between samples taken by the profiler (milliseconds)"
}],
[["--enable-stylo"], {
"action": "store_true",
"dest": "enable_stylo",
"default": False,
"help": "Run tests with Stylo enabled"
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@@ -90,17 +90,16 @@ testing_config_options = [
{"action": "store",
"dest": "jsshell_url",
"default": None,
"help": "URL to the jsshell to install",
}],
[["--download-symbols"],
{"action": "store",
"dest": "download_symbols",
- "type": "choice",
"choices": ['ondemand', 'true'],
"help": "Download and extract crash reporter symbols.",
}],
] + copy.deepcopy(virtualenv_config_options) \
+ copy.deepcopy(try_config_options) \
+ copy.deepcopy(verify_config_options)
--- a/testing/mozharness/scripts/desktop_l10n.py
+++ b/testing/mozharness/scripts/desktop_l10n.py
@@ -74,91 +74,80 @@ runtime_config_tokens = ('buildid', 'ver
class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
VCSMixin, SigningMixin, PurgeMixin, BaseScript,
BalrogMixin, MarMixin, VirtualenvMixin, TransferMixin):
"""Manages desktop repacks"""
config_options = [[
['--balrog-config', ],
{"action": "extend",
"dest": "config_files",
- "type": "string",
"help": "Specify the balrog configuration file"}
], [
['--branch-config', ],
{"action": "extend",
"dest": "config_files",
- "type": "string",
"help": "Specify the branch configuration file"}
], [
['--environment-config', ],
{"action": "extend",
"dest": "config_files",
- "type": "string",
"help": "Specify the environment (staging, production, ...) configuration file"}
], [
['--platform-config', ],
{"action": "extend",
"dest": "config_files",
- "type": "string",
"help": "Specify the platform configuration file"}
], [
['--locale', ],
{"action": "extend",
"dest": "locales",
- "type": "string",
"help": "Specify the locale(s) to sign and update. Optionally pass"
" revision separated by colon, en-GB:default."}
], [
['--locales-file', ],
{"action": "store",
"dest": "locales_file",
- "type": "string",
"help": "Specify a file to determine which locales to sign and update"}
], [
['--tag-override', ],
{"action": "store",
"dest": "tag_override",
- "type": "string",
"help": "Override the tags set for all repos"}
], [
['--revision', ],
{"action": "store",
"dest": "revision",
- "type": "string",
"help": "Override the gecko revision to use (otherwise use buildbot supplied"
" value, or en-US revision) "}
], [
['--user-repo-override', ],
{"action": "store",
"dest": "user_repo_override",
- "type": "string",
"help": "Override the user repo path for all repos"}
], [
['--release-config-file', ],
{"action": "store",
"dest": "release_config_file",
- "type": "string",
"help": "Specify the release config file to use"}
], [
['--this-chunk', ],
{"action": "store",
"dest": "this_locale_chunk",
- "type": "int",
+ "type": int,
"help": "Specify which chunk of locales to run"}
], [
['--total-chunks', ],
{"action": "store",
"dest": "total_locale_chunks",
- "type": "int",
+ "type": int,
"help": "Specify the total number of chunks of locales"}
], [
['--en-us-installer-url', ],
{"action": "store",
"dest": "en_us_installer_url",
- "type": "string",
"help": "Specify the url of the en-us binary"}
], [
["--disable-mock"], {
"dest": "disable_mock",
"action": "store_true",
"help": "do not run under mock despite what gecko-config says"}
]]
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -48,73 +48,65 @@ SUITE_NO_E10S = ['xpcshell']
# DesktopUnittest {{{1
class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMixin,
CodeCoverageMixin):
config_options = [
[['--mochitest-suite', ], {
"action": "extend",
"dest": "specified_mochitest_suites",
- "type": "string",
"help": "Specify which mochi suite to run. "
"Suites are defined in the config file.\n"
"Examples: 'all', 'plain1', 'plain5', 'chrome', or 'a11y'"}
],
[['--reftest-suite', ], {
"action": "extend",
"dest": "specified_reftest_suites",
- "type": "string",
"help": "Specify which reftest suite to run. "
"Suites are defined in the config file.\n"
"Examples: 'all', 'crashplan', or 'jsreftest'"}
],
[['--xpcshell-suite', ], {
"action": "extend",
"dest": "specified_xpcshell_suites",
- "type": "string",
"help": "Specify which xpcshell suite to run. "
"Suites are defined in the config file\n."
"Examples: 'xpcshell'"}
],
[['--cppunittest-suite', ], {
"action": "extend",
"dest": "specified_cppunittest_suites",
- "type": "string",
"help": "Specify which cpp unittest suite to run. "
"Suites are defined in the config file\n."
"Examples: 'cppunittest'"}
],
[['--gtest-suite', ], {
"action": "extend",
"dest": "specified_gtest_suites",
- "type": "string",
"help": "Specify which gtest suite to run. "
"Suites are defined in the config file\n."
"Examples: 'gtest'"}
],
[['--jittest-suite', ], {
"action": "extend",
"dest": "specified_jittest_suites",
- "type": "string",
"help": "Specify which jit-test suite to run. "
"Suites are defined in the config file\n."
"Examples: 'jittest'"}
],
[['--mozbase-suite', ], {
"action": "extend",
"dest": "specified_mozbase_suites",
- "type": "string",
"help": "Specify which mozbase suite to run. "
"Suites are defined in the config file\n."
"Examples: 'mozbase'"}
],
[['--mozmill-suite', ], {
"action": "extend",
"dest": "specified_mozmill_suites",
- "type": "string",
"help": "Specify which mozmill suite to run. "
"Suites are defined in the config file\n."
"Examples: 'mozmill'"}
],
[['--run-all-suites', ], {
"action": "store_true",
"dest": "run_all_suites",
"default": False,
--- a/testing/mozharness/scripts/fx_desktop_build.py
+++ b/testing/mozharness/scripts/fx_desktop_build.py
@@ -12,16 +12,17 @@ and developer machines alike
author: Jordan Lund
"""
import copy
import pprint
import sys
import os
+from argparse import Namespace
# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))
import mozharness.base.script as script
from mozharness.mozilla.building.buildbase import BUILD_BASE_CONFIG_OPTIONS, \
BuildingConfig, BuildOptionParser, BuildScript
from mozharness.base.config import parse_config_file
@@ -163,17 +164,17 @@ class FxDesktopBuild(BuildScript, TryToo
# helpers
def _update_build_variant(self, rw_config, variant='artifact'):
""" Intended for use in _pre_config_lock """
c = self.config
variant_cfg_path, _ = BuildOptionParser.find_variant_cfg_path(
'--custom-build-variant-cfg',
variant,
- rw_config.config_parser
+ Namespace(**rw_config._config)
)
if not variant_cfg_path:
self.fatal('Could not find appropriate config file for variant %s' % variant)
# Update other parts of config to keep dump-config accurate
# Only dump-config is affected because most config info is set during
# initial parsing
variant_cfg_dict = parse_config_file(variant_cfg_path)
rw_config.all_cfg_files_and_dicts.append((variant_cfg_path, variant_cfg_dict))
--- a/testing/mozharness/scripts/marionette.py
+++ b/testing/mozharness/scripts/marionette.py
@@ -55,17 +55,16 @@ class MarionetteTest(TestingMixin, Mercu
"dest": "marionette_address",
"default": None,
"help": "The host:port of the Marionette server running inside Gecko. "
"Unused for emulator testing",
}
], [
["--emulator"],
{"action": "store",
- "type": "choice",
"choices": ['arm', 'x86'],
"dest": "emulator",
"default": None,
"help": "Use an emulator for testing",
}
], [
["--test-manifest"],
{"action": "store",
--- a/testing/mozharness/scripts/merge_day/gecko_migration.py
+++ b/testing/mozharness/scripts/merge_day/gecko_migration.py
@@ -37,42 +37,37 @@ VALID_MIGRATION_BEHAVIORS = (
# GeckoMigration {{{1
class GeckoMigration(MercurialScript, BalrogMixin, VirtualenvMixin,
BuildbotMixin, MercurialRepoManipulationMixin):
config_options = [
[['--hg-user', ], {
"action": "store",
"dest": "hg_user",
- "type": "string",
"default": "ffxbld <release@mozilla.com>",
"help": "Specify what user to use to commit to hg.",
}],
[['--balrog-api-root', ], {
"action": "store",
"dest": "balrog_api_root",
- "type": "string",
"help": "Specify Balrog API root URL.",
}],
[['--balrog-username', ], {
"action": "store",
"dest": "balrog_username",
- "type": "string",
"help": "Specify what user to connect to Balrog with.",
}],
[['--balrog-credentials-file', ], {
"action": "store",
"dest": "balrog_credentials_file",
- "type": "string",
"help": "The file containing the Balrog credentials.",
}],
[['--remove-locale', ], {
"action": "extend",
"dest": "remove_locales",
- "type": "string",
"help": "Comma separated list of locales to remove from the 'to' repo.",
}],
]
gecko_repos = None
def __init__(self, require_config_file=True):
super(GeckoMigration, self).__init__(
config_options=virtualenv_config_options + self.config_options,
--- a/testing/mozharness/scripts/mobile_l10n.py
+++ b/testing/mozharness/scripts/mobile_l10n.py
@@ -47,81 +47,74 @@ from mozharness.mozilla.taskcluster_help
class MobileSingleLocale(MockMixin, LocalesMixin, ReleaseMixin,
MobileSigningMixin, TransferMixin, TooltoolMixin,
BuildbotMixin, PurgeMixin, MercurialScript, BalrogMixin,
VirtualenvMixin):
config_options = [[
['--locale', ],
{"action": "extend",
"dest": "locales",
- "type": "string",
"help": "Specify the locale(s) to sign and update"
}
], [
['--locales-file', ],
{"action": "store",
"dest": "locales_file",
- "type": "string",
"help": "Specify a file to determine which locales to sign and update"
}
], [
['--tag-override', ],
{"action": "store",
"dest": "tag_override",
- "type": "string",
"help": "Override the tags set for all repos"
}
], [
['--user-repo-override', ],
{"action": "store",
"dest": "user_repo_override",
- "type": "string",
"help": "Override the user repo path for all repos"
}
], [
['--release-config-file', ],
{"action": "store",
"dest": "release_config_file",
- "type": "string",
"help": "Specify the release config file to use"
}
], [
['--key-alias', ],
{"action": "store",
"dest": "key_alias",
- "type": "choice",
"default": "nightly",
"choices": ["nightly", "release"],
"help": "Specify the signing key alias"
}
], [
['--this-chunk', ],
{"action": "store",
"dest": "this_locale_chunk",
- "type": "int",
+ "type": int,
"help": "Specify which chunk of locales to run"
}
], [
['--total-chunks', ],
{"action": "store",
"dest": "total_locale_chunks",
- "type": "int",
+ "type": int,
"help": "Specify the total number of chunks of locales"
}
], [
["--disable-mock"],
{"dest": "disable_mock",
"action": "store_true",
"help": "do not run under mock despite what gecko-config says",
}
], [
['--revision', ],
{"action": "store",
"dest": "revision",
- "type": "string",
"help": "Override the gecko revision to use (otherwise use buildbot supplied"
" value, or en-US revision) "}
]]
def __init__(self, require_config_file=True):
buildscript_kwargs = {
'all_actions': [
"clobber",
--- a/testing/mozharness/scripts/mobile_partner_repack.py
+++ b/testing/mozharness/scripts/mobile_partner_repack.py
@@ -28,74 +28,66 @@ SUPPORTED_PLATFORMS = ["android"]
# MobilePartnerRepack {{{1
class MobilePartnerRepack(LocalesMixin, ReleaseMixin, MobileSigningMixin,
TransferMixin, MercurialScript):
config_options = [[
['--locale', ],
{"action": "extend",
"dest": "locales",
- "type": "string",
"help": "Specify the locale(s) to repack"
}
], [
['--partner', ],
{"action": "extend",
"dest": "partners",
- "type": "string",
"help": "Specify the partner(s) to repack"
}
], [
['--locales-file', ],
{"action": "store",
"dest": "locales_file",
- "type": "string",
"help": "Specify a json file to determine which locales to repack"
}
], [
['--tag-override', ],
{"action": "store",
"dest": "tag_override",
- "type": "string",
"help": "Override the tags set for all repos"
}
], [
['--platform', ],
{"action": "extend",
"dest": "platforms",
- "type": "choice",
"choices": SUPPORTED_PLATFORMS,
"help": "Specify the platform(s) to repack"
}
], [
['--user-repo-override', ],
{"action": "store",
"dest": "user_repo_override",
- "type": "string",
"help": "Override the user repo path for all repos"
}
], [
['--release-config-file', ],
{"action": "store",
"dest": "release_config_file",
- "type": "string",
"help": "Specify the release config file to use"
}
], [
['--version', ],
{"action": "store",
"dest": "version",
- "type": "string",
"help": "Specify the current version"
}
], [
['--buildnum', ],
{"action": "store",
"dest": "buildnum",
- "type": "int",
+ "type": int,
"default": 1,
"metavar": "INT",
"help": "Specify the current release build num (e.g. build1, build2)"
}
]]
def __init__(self, require_config_file=True):
self.release_config = {}
--- a/testing/mozharness/scripts/release/antivirus.py
+++ b/testing/mozharness/scripts/release/antivirus.py
@@ -31,23 +31,23 @@ class AntivirusScan(BaseScript, Virtuale
[["--exclude"], {
"dest": "excludes",
"action": "append",
"help": "List of filename patterns to exclude. See script source for default",
}],
[["-d", "--download-parallelization"], {
"dest": "download_parallelization",
"default": 6,
- "type": "int",
+ "type": int,
"help": "Number of concurrent file downloads",
}],
[["-s", "--scan-parallelization"], {
"dest": "scan_parallelization",
"default": 4,
- "type": "int",
+ "type": int,
"help": "Number of concurrent file scans",
}],
[["--tools-repo"], {
"dest": "tools_repo",
"default": "https://hg.mozilla.org/build/tools",
}],
[["--tools-revision"], {
"dest": "tools_revision",
--- a/testing/mozharness/scripts/release/beet_mover.py
+++ b/testing/mozharness/scripts/release/beet_mover.py
@@ -36,17 +36,16 @@ def get_hash(content, hash_type="md5"):
CONFIG_OPTIONS = [
[["--template"], {
"dest": "template",
"help": "Specify jinja2 template file",
}],
[['--locale', ], {
"action": "extend",
"dest": "locales",
- "type": "string",
"help": "Specify the locale(s) to upload."}],
[["--platform"], {
"dest": "platform",
"help": "Specify the platform of the build",
}],
[["--version"], {
"dest": "version",
"help": "full release version based on gecko and tag/stage identifier. e.g. '44.0b1'"
@@ -83,17 +82,17 @@ CONFIG_OPTIONS = [
[["--exclude"], {
"dest": "excludes",
"action": "append",
"help": "List of filename patterns to exclude. See script source for default",
}],
[["-s", "--scan-parallelization"], {
"dest": "scan_parallelization",
"default": 4,
- "type": "int",
+ "type": int,
"help": "Number of concurrent file scans",
}],
]
DEFAULT_EXCLUDES = [
r"^.*tests.*$",
r"^.*crashreporter.*$",
r"^.*\.zip(\.asc)?$",
--- a/testing/mozharness/scripts/release/postrelease_version_bump.py
+++ b/testing/mozharness/scripts/release/postrelease_version_bump.py
@@ -22,60 +22,52 @@ from mozharness.mozilla.repo_manipulatio
# PostReleaseVersionBump {{{1
class PostReleaseVersionBump(MercurialScript, BuildbotMixin,
MercurialRepoManipulationMixin):
config_options = [
[['--hg-user', ], {
"action": "store",
"dest": "hg_user",
- "type": "string",
"default": "ffxbld <release@mozilla.com>",
"help": "Specify what user to use to commit to hg.",
}],
[['--next-version', ], {
"action": "store",
"dest": "next_version",
- "type": "string",
"help": "Next version used in version bump",
}],
[['--ssh-user', ], {
"action": "store",
"dest": "ssh_user",
- "type": "string",
"help": "SSH username with hg.mozilla.org permissions",
}],
[['--ssh-key', ], {
"action": "store",
"dest": "ssh_key",
- "type": "string",
"help": "Path to SSH key.",
}],
[['--product', ], {
"action": "store",
"dest": "product",
- "type": "string",
"help": "Product name",
}],
[['--version', ], {
"action": "store",
"dest": "version",
- "type": "string",
"help": "Version",
}],
[['--build-number', ], {
"action": "store",
"dest": "build_number",
- "type": "string",
"help": "Build number",
}],
[['--revision', ], {
"action": "store",
"dest": "revision",
- "type": "string",
"help": "HG revision to tag",
}],
]
def __init__(self, require_config_file=True):
super(PostReleaseVersionBump, self).__init__(
config_options=self.config_options,
all_actions=[
--- a/testing/mozharness/scripts/release/push-candidate-to-releases.py
+++ b/testing/mozharness/scripts/release/push-candidate-to-releases.py
@@ -54,17 +54,17 @@ class ReleasePusher(BaseScript, Virtuale
],
"action": "append",
"help": "List of patterns to exclude from copy. The list can be "
"extended by passing multiple --exclude arguments.",
}],
[["-j", "--parallelization"], {
"dest": "parallelization",
"default": 20,
- "type": "int",
+ "type": int,
"help": "Number of copy requests to run concurrently",
}],
] + virtualenv_config_options
def __init__(self, aws_creds):
BaseScript.__init__(self,
config_options=self.config_options,
require_config_file=False,
--- a/testing/mozharness/scripts/release/updates.py
+++ b/testing/mozharness/scripts/release/updates.py
@@ -30,30 +30,27 @@ from mozharness.mozilla.release import g
# UpdatesBumper {{{1
class UpdatesBumper(MercurialScript, BuildbotMixin,
MercurialRepoManipulationMixin):
config_options = [
[['--hg-user', ], {
"action": "store",
"dest": "hg_user",
- "type": "string",
"default": "ffxbld <release@mozilla.com>",
"help": "Specify what user to use to commit to hg.",
}],
[['--ssh-user', ], {
"action": "store",
"dest": "ssh_user",
- "type": "string",
"help": "SSH username with hg.mozilla.org permissions",
}],
[['--ssh-key', ], {
"action": "store",
"dest": "ssh_key",
- "type": "string",
"help": "Path to SSH key.",
}],
]
def __init__(self, require_config_file=True):
super(UpdatesBumper, self).__init__(
config_options=self.config_options,
all_actions=[
--- a/testing/mozharness/test/test_base_config.py
+++ b/testing/mozharness/test/test_base_config.py
@@ -245,16 +245,17 @@ class TestReadOnlyDict(unittest.TestCase
r = self.get_locked_ROD()
c = deepcopy(r)
c['e'] = 'hey'
self.assertEqual(c['e'], 'hey', "can't set var in ROD after deepcopy")
class TestActions(unittest.TestCase):
all_actions = ['a', 'b', 'c', 'd', 'e']
+ config_options = [[['args'], {'nargs': '*'}]]
default_actions = ['b', 'c', 'd']
def test_verify_actions(self):
c = config.BaseConfig(initial_config_file='test/test.json')
try:
c.verify_actions(['not_a_real_action'])
except:
pass
@@ -270,39 +271,43 @@ class TestActions(unittest.TestCase):
all_actions=self.all_actions,
initial_config_file='test/test.json')
self.assertEqual(self.default_actions, c.get_actions(),
msg="default_actions broken")
def test_no_action1(self):
c = config.BaseConfig(default_actions=self.default_actions,
all_actions=self.all_actions,
+ config_options=self.config_options,
initial_config_file='test/test.json')
c.parse_args(args=['foo', '--no-action', 'a'])
self.assertEqual(self.default_actions, c.get_actions(),
msg="--no-ACTION broken")
def test_no_action2(self):
c = config.BaseConfig(default_actions=self.default_actions,
all_actions=self.all_actions,
+ config_options=self.config_options,
initial_config_file='test/test.json')
c.parse_args(args=['foo', '--no-c'])
self.assertEqual(['b', 'd'], c.get_actions(),
msg="--no-ACTION broken")
def test_add_action(self):
c = config.BaseConfig(default_actions=self.default_actions,
all_actions=self.all_actions,
+ config_options=self.config_options,
initial_config_file='test/test.json')
c.parse_args(args=['foo', '--add-action', 'e'])
self.assertEqual(['b', 'c', 'd', 'e'], c.get_actions(),
msg="--add-action ACTION broken")
def test_only_action(self):
c = config.BaseConfig(default_actions=self.default_actions,
all_actions=self.all_actions,
+ config_options=self.config_options,
initial_config_file='test/test.json')
c.parse_args(args=['foo', '--a', '--e'])
self.assertEqual(['a', 'e'], c.get_actions(),
msg="--ACTION broken")
if __name__ == '__main__':
unittest.main()