Bug 1401199 - [taskgraph] Use default parameter values when strict=False, r?dustin draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Fri, 29 Sep 2017 11:35:30 -0400
changeset 672875 6c0860b17efbd27d88fde3ad8e03eed49814f519
parent 672691 9f731e41bb287b0f459bf3609dc3401f7257ce89
child 672876 2c26727445a2d8a49fa9da8466b5c55e429f6fc8
push id82406
push userahalberstadt@mozilla.com
push dateFri, 29 Sep 2017 21:17:09 +0000
reviewersdustin
bugs1401199
milestone58.0a1
Bug 1401199 - [taskgraph] Use default parameter values when strict=False, r?dustin MozReview-Commit-ID: 9XWlLeGcPeQ
taskcluster/taskgraph/decision.py
taskcluster/taskgraph/parameters.py
taskcluster/taskgraph/test/test_parameters.py
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -228,17 +228,17 @@ def get_decision_parameters(options):
         # to run tasks that would otherwise be optimized, ues try_task_config.
         'try_option_syntax': True,
 
         # since no try jobs have been specified, the standard target task will
         # be applied, and tasks should be optimized out of that.
         None: True,
     }[parameters['try_mode']]
 
-    return Parameters(parameters)
+    return Parameters(**parameters)
 
 
 def write_artifact(filename, data):
     logger.info('writing artifact file `{}`'.format(filename))
     if not os.path.isdir(ARTIFACTS_DIR):
         os.mkdir(ARTIFACTS_DIR)
     path = os.path.join(ARTIFACTS_DIR, filename)
     if filename.endswith('.yml'):
--- a/taskcluster/taskgraph/parameters.py
+++ b/taskcluster/taskgraph/parameters.py
@@ -2,88 +2,115 @@
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
+import time
 import yaml
-from mozbuild.util import ReadOnlyDict
+from datetime import datetime
+
+from mozbuild.util import ReadOnlyDict, memoize
+from mozversioncontrol import get_repository_object
+
+from . import GECKO
 
 
 class ParameterMismatch(Exception):
     """Raised when a parameters.yml has extra or missing parameters."""
 
 
+@memoize
+def get_head_ref():
+    return get_repository_object(GECKO).head_ref
+
+
 # Please keep this list sorted and in sync with taskcluster/docs/parameters.rst
-PARAMETER_NAMES = set([
-    'base_repository',
-    'build_date',
-    'filters',
-    'head_ref',
-    'head_repository',
-    'head_rev',
-    'include_nightly',
-    'level',
-    'message',
-    'moz_build_date',
-    'optimize_target_tasks',
-    'owner',
-    'project',
-    'pushdate',
-    'pushlog_id',
-    'release_history',
-    'target_tasks_method',
-    'try_mode',
-    'try_options',
-    'try_task_config',
-])
+# Parameters are of the form: {name: default}
+PARAMETERS = {
+    'base_repository': 'https://hg.mozilla.org/mozilla-unified',
+    'build_date': lambda: int(time.time()),
+    'filters': ['check_servo', 'target_tasks_method'],
+    'head_ref': get_head_ref,
+    'head_repository': 'https://hg.mozilla.org/mozilla-central',
+    'head_rev': get_head_ref,
+    'include_nightly': False,
+    'level': '3',
+    'message': '',
+    'moz_build_date': lambda: datetime.now().strftime("%Y%m%d%H%M%S"),
+    'optimize_target_tasks': True,
+    'owner': 'nobody@mozilla.com',
+    'project': 'mozilla-central',
+    'pushdate': lambda: int(time.time()),
+    'pushlog_id': '0',
+    'release_history': {},
+    'target_tasks_method': 'default',
+    'try_mode': None,
+    'try_options': None,
+    'try_task_config': None,
+}
 
 
 class Parameters(ReadOnlyDict):
     """An immutable dictionary with nicer KeyError messages on failure"""
+
+    def __init__(self, strict=True, **kwargs):
+        self.strict = strict
+
+        if not self.strict:
+            # apply defaults to missing parameters
+            for name, default in PARAMETERS.items():
+                if name not in kwargs:
+                    if callable(default):
+                        default = default()
+                    kwargs[name] = default
+
+        ReadOnlyDict.__init__(self, **kwargs)
+
     def check(self):
         names = set(self)
+        valid = set(PARAMETERS.keys())
         msg = []
 
-        missing = PARAMETER_NAMES - names
+        missing = valid - names
         if missing:
             msg.append("missing parameters: " + ", ".join(missing))
 
-        extra = names - PARAMETER_NAMES
-        if extra:
+        extra = names - valid
+        if extra and self.strict:
             msg.append("extra parameters: " + ", ".join(extra))
 
         if msg:
             raise ParameterMismatch("; ".join(msg))
 
     def __getitem__(self, k):
-        if k not in PARAMETER_NAMES:
+        if k not in PARAMETERS.keys():
             raise KeyError("no such parameter {!r}".format(k))
         try:
             return super(Parameters, self).__getitem__(k)
         except KeyError:
             raise KeyError("taskgraph parameter {!r} not found".format(k))
 
 
-def load_parameters_file(filename):
+def load_parameters_file(filename, strict=True):
     """
     Load parameters from a path, url, decision task-id or project.
 
     Examples:
         task-id=fdtgsD5DQUmAQZEaGMvQ4Q
         project=mozilla-central
     """
     import urllib
     from taskgraph.util.taskcluster import get_artifact_url, find_task_id
 
     if not filename:
-        return Parameters()
+        return Parameters(strict=strict)
 
     try:
         # reading parameters from a local parameters.yml file
         f = open(filename)
     except IOError:
         # fetching parameters.yml using task task-id, project or supplied url
         task_id = None
         if filename.startswith("task-id="):
@@ -92,13 +119,13 @@ def load_parameters_file(filename):
             index = "gecko.v2.{}.latest.firefox.decision".format(filename.split("=")[1])
             task_id = find_task_id(index)
 
         if task_id:
             filename = get_artifact_url(task_id, 'public/parameters.yml')
         f = urllib.urlopen(filename)
 
     if filename.endswith('.yml'):
-        return Parameters(**yaml.safe_load(f))
+        return Parameters(strict=strict, **yaml.safe_load(f))
     elif filename.endswith('.json'):
-        return Parameters(**json.load(f))
+        return Parameters(strict=strict, **json.load(f))
     else:
         raise TypeError("Parameters file `{}` is not JSON or YAML".format(filename))
--- a/taskcluster/taskgraph/test/test_parameters.py
+++ b/taskcluster/taskgraph/test/test_parameters.py
@@ -1,23 +1,28 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 
-from taskgraph.parameters import Parameters, load_parameters_file, PARAMETER_NAMES
+from taskgraph.parameters import (
+    Parameters,
+    ParameterMismatch,
+    load_parameters_file,
+    PARAMETERS,
+)
 from mozunit import main, MockedOpen
 
 
 class TestParameters(unittest.TestCase):
 
-    vals = {n: n for n in PARAMETER_NAMES}
+    vals = {n: n for n in PARAMETERS.keys()}
 
     def test_Parameters_immutable(self):
         p = Parameters(**self.vals)
 
         def assign():
             p['head_ref'] = 20
         self.assertRaises(Exception, assign)
 
@@ -35,21 +40,27 @@ class TestParameters(unittest.TestCase):
         self.assertEqual(p['head_ref'], 10)
 
     def test_Parameters_check(self):
         p = Parameters(**self.vals)
         p.check()  # should not raise
 
     def test_Parameters_check_missing(self):
         p = Parameters()
-        self.assertRaises(Exception, lambda: p.check())
+        self.assertRaises(ParameterMismatch, lambda: p.check())
+
+        p = Parameters(strict=False)
+        p.check()  # should not raise
 
     def test_Parameters_check_extra(self):
         p = Parameters(xyz=10, **self.vals)
-        self.assertRaises(Exception, lambda: p.check())
+        self.assertRaises(ParameterMismatch, lambda: p.check())
+
+        p = Parameters(strict=False, xyz=10, **self.vals)
+        p.check()  # should not raise
 
     def test_load_parameters_file_yaml(self):
         with MockedOpen({"params.yml": "some: data\n"}):
             self.assertEqual(
                     load_parameters_file('params.yml'),
                     {'some': 'data'})
 
     def test_load_parameters_file_json(self):