Bug 1401199 - [taskgraph] Use default parameter values when strict=False, r?dustin
MozReview-Commit-ID: 9XWlLeGcPeQ
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -228,17 +228,17 @@ def get_decision_parameters(options):
# to run tasks that would otherwise be optimized, ues try_task_config.
'try_option_syntax': True,
# since no try jobs have been specified, the standard target task will
# be applied, and tasks should be optimized out of that.
None: True,
}[parameters['try_mode']]
- return Parameters(parameters)
+ return Parameters(**parameters)
def write_artifact(filename, data):
logger.info('writing artifact file `{}`'.format(filename))
if not os.path.isdir(ARTIFACTS_DIR):
os.mkdir(ARTIFACTS_DIR)
path = os.path.join(ARTIFACTS_DIR, filename)
if filename.endswith('.yml'):
--- a/taskcluster/taskgraph/parameters.py
+++ b/taskcluster/taskgraph/parameters.py
@@ -2,88 +2,115 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import json
+import time
import yaml
-from mozbuild.util import ReadOnlyDict
+from datetime import datetime
+
+from mozbuild.util import ReadOnlyDict, memoize
+from mozversioncontrol import get_repository_object
+
+from . import GECKO
class ParameterMismatch(Exception):
"""Raised when a parameters.yml has extra or missing parameters."""
+@memoize
+def get_head_ref():
+ return get_repository_object(GECKO).head_ref
+
+
# Please keep this list sorted and in sync with taskcluster/docs/parameters.rst
-PARAMETER_NAMES = set([
- 'base_repository',
- 'build_date',
- 'filters',
- 'head_ref',
- 'head_repository',
- 'head_rev',
- 'include_nightly',
- 'level',
- 'message',
- 'moz_build_date',
- 'optimize_target_tasks',
- 'owner',
- 'project',
- 'pushdate',
- 'pushlog_id',
- 'release_history',
- 'target_tasks_method',
- 'try_mode',
- 'try_options',
- 'try_task_config',
-])
+# Parameters are of the form: {name: default}
+PARAMETERS = {
+ 'base_repository': 'https://hg.mozilla.org/mozilla-unified',
+ 'build_date': lambda: int(time.time()),
+ 'filters': ['check_servo', 'target_tasks_method'],
+ 'head_ref': get_head_ref,
+ 'head_repository': 'https://hg.mozilla.org/mozilla-central',
+ 'head_rev': get_head_ref,
+ 'include_nightly': False,
+ 'level': '3',
+ 'message': '',
+ 'moz_build_date': lambda: datetime.now().strftime("%Y%m%d%H%M%S"),
+ 'optimize_target_tasks': True,
+ 'owner': 'nobody@mozilla.com',
+ 'project': 'mozilla-central',
+ 'pushdate': lambda: int(time.time()),
+ 'pushlog_id': '0',
+ 'release_history': {},
+ 'target_tasks_method': 'default',
+ 'try_mode': None,
+ 'try_options': None,
+ 'try_task_config': None,
+}
class Parameters(ReadOnlyDict):
"""An immutable dictionary with nicer KeyError messages on failure"""
+
+ def __init__(self, strict=True, **kwargs):
+ self.strict = strict
+
+ if not self.strict:
+ # apply defaults to missing parameters
+ for name, default in PARAMETERS.items():
+ if name not in kwargs:
+ if callable(default):
+ default = default()
+ kwargs[name] = default
+
+ ReadOnlyDict.__init__(self, **kwargs)
+
def check(self):
names = set(self)
+ valid = set(PARAMETERS.keys())
msg = []
- missing = PARAMETER_NAMES - names
+ missing = valid - names
if missing:
msg.append("missing parameters: " + ", ".join(missing))
- extra = names - PARAMETER_NAMES
- if extra:
+ extra = names - valid
+ if extra and self.strict:
msg.append("extra parameters: " + ", ".join(extra))
if msg:
raise ParameterMismatch("; ".join(msg))
def __getitem__(self, k):
- if k not in PARAMETER_NAMES:
+ if k not in PARAMETERS.keys():
raise KeyError("no such parameter {!r}".format(k))
try:
return super(Parameters, self).__getitem__(k)
except KeyError:
raise KeyError("taskgraph parameter {!r} not found".format(k))
-def load_parameters_file(filename):
+def load_parameters_file(filename, strict=True):
"""
Load parameters from a path, url, decision task-id or project.
Examples:
task-id=fdtgsD5DQUmAQZEaGMvQ4Q
project=mozilla-central
"""
import urllib
from taskgraph.util.taskcluster import get_artifact_url, find_task_id
if not filename:
- return Parameters()
+ return Parameters(strict=strict)
try:
# reading parameters from a local parameters.yml file
f = open(filename)
except IOError:
# fetching parameters.yml using task task-id, project or supplied url
task_id = None
if filename.startswith("task-id="):
@@ -92,13 +119,13 @@ def load_parameters_file(filename):
index = "gecko.v2.{}.latest.firefox.decision".format(filename.split("=")[1])
task_id = find_task_id(index)
if task_id:
filename = get_artifact_url(task_id, 'public/parameters.yml')
f = urllib.urlopen(filename)
if filename.endswith('.yml'):
- return Parameters(**yaml.safe_load(f))
+ return Parameters(strict=strict, **yaml.safe_load(f))
elif filename.endswith('.json'):
- return Parameters(**json.load(f))
+ return Parameters(strict=strict, **json.load(f))
else:
raise TypeError("Parameters file `{}` is not JSON or YAML".format(filename))
--- a/taskcluster/taskgraph/test/test_parameters.py
+++ b/taskcluster/taskgraph/test/test_parameters.py
@@ -1,23 +1,28 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import unittest
-from taskgraph.parameters import Parameters, load_parameters_file, PARAMETER_NAMES
+from taskgraph.parameters import (
+ Parameters,
+ ParameterMismatch,
+ load_parameters_file,
+ PARAMETERS,
+)
from mozunit import main, MockedOpen
class TestParameters(unittest.TestCase):
- vals = {n: n for n in PARAMETER_NAMES}
+ vals = {n: n for n in PARAMETERS.keys()}
def test_Parameters_immutable(self):
p = Parameters(**self.vals)
def assign():
p['head_ref'] = 20
self.assertRaises(Exception, assign)
@@ -35,21 +40,27 @@ class TestParameters(unittest.TestCase):
self.assertEqual(p['head_ref'], 10)
def test_Parameters_check(self):
p = Parameters(**self.vals)
p.check() # should not raise
def test_Parameters_check_missing(self):
p = Parameters()
- self.assertRaises(Exception, lambda: p.check())
+ self.assertRaises(ParameterMismatch, lambda: p.check())
+
+ p = Parameters(strict=False)
+ p.check() # should not raise
def test_Parameters_check_extra(self):
p = Parameters(xyz=10, **self.vals)
- self.assertRaises(Exception, lambda: p.check())
+ self.assertRaises(ParameterMismatch, lambda: p.check())
+
+ p = Parameters(strict=False, xyz=10, **self.vals)
+ p.check() # should not raise
def test_load_parameters_file_yaml(self):
with MockedOpen({"params.yml": "some: data\n"}):
self.assertEqual(
load_parameters_file('params.yml'),
{'some': 'data'})
def test_load_parameters_file_json(self):