Bug 1413304 - Update third_party/python/json-e to version 2.3.2, r?bstack draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 31 Oct 2017 15:40:15 -0400
changeset 689623 e73c653e63a3527db3054f496b7bba210bd5674d
parent 689622 a4ed10c530275490da48679c6ca5aa0ed196332a
child 738351 7853a892cca6b980c8dd3761ee34a7ffc9482fd5
push id87060
push userahalberstadt@mozilla.com
push dateTue, 31 Oct 2017 19:40:59 +0000
reviewersbstack
bugs1413304
milestone58.0a1
Bug 1413304 - Update third_party/python/json-e to version 2.3.2, r?bstack MozReview-Commit-ID: 9drZiQRMHiC
third_party/python/json-e/jsone/__init__.py
third_party/python/json-e/jsone/builtins.py
third_party/python/json-e/jsone/interpreter.py
third_party/python/json-e/jsone/prattparser.py
third_party/python/json-e/jsone/render.py
third_party/python/json-e/jsone/shared.py
third_party/python/json-e/jsone/six.py
--- a/third_party/python/json-e/jsone/__init__.py
+++ b/third_party/python/json-e/jsone/__init__.py
@@ -1,22 +1,21 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
-import datetime
 import re
 from .render import renderValue
-from .shared import JSONTemplateError, DeleteMarker, TemplateError
+from .shared import JSONTemplateError, DeleteMarker, TemplateError, fromNow
 from . import builtins
 
 _context_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*$')
 
 
 def render(template, context):
     if not all(_context_re.match(c) for c in context):
         raise TemplateError('top level keys of context must follow '
-                                '/[a-zA-Z_][a-zA-Z0-9_]*/')
-    full_context = {'now': datetime.datetime.utcnow()}
+                            '/[a-zA-Z_][a-zA-Z0-9_]*/')
+    full_context = {'now': fromNow('0 seconds', None)}
     full_context.update(builtins.build(full_context))
     full_context.update(context)
     rv = renderValue(template, full_context)
     if rv is DeleteMarker:
         return None
     return rv
--- a/third_party/python/json-e/jsone/builtins.py
+++ b/third_party/python/json-e/jsone/builtins.py
@@ -9,17 +9,18 @@ class BuiltinError(JSONTemplateError):
 
 
 def build(context):
     builtins = {}
 
     def builtin(name, variadic=None, argument_tests=None, minArgs=None):
         def wrap(fn):
             def bad(reason=None):
-                raise BuiltinError((reason or 'invalid arguments to {}').format(name))
+                raise BuiltinError(
+                    (reason or 'invalid arguments to builtin: {}').format(name))
             if variadic:
                 def invoke(*args):
                     if minArgs:
                         if len(args) < minArgs:
                             bad("too few arguments to {}")
                     for arg in args:
                         if not variadic(arg):
                             bad()
@@ -46,16 +47,19 @@ def build(context):
         return isinstance(v, (int, float)) and not isinstance(v, bool)
 
     def is_string(v):
         return isinstance(v, string)
 
     def is_string_or_array(v):
         return isinstance(v, (string, list))
 
+    def anything_except_array(v):
+        return isinstance(v, (string, int, float, bool)) or v is None
+
     def anything(v):
         return isinstance(v, (string, int, float, list, dict)) or v is None or callable(v)
 
     # ---
 
     builtin('min', variadic=is_number, minArgs=1)(min)
     builtin('max', variadic=is_number, minArgs=1)(max)
     builtin('sqrt', argument_tests=[is_number])(math.sqrt)
@@ -73,17 +77,17 @@ def build(context):
     def lowercase(v):
         return v.lower()
 
     @builtin('uppercase', argument_tests=[is_string])
     def lowercase(v):
         return v.upper()
 
     builtin('len', argument_tests=[is_string_or_array])(len)
-    builtin('str', argument_tests=[anything])(to_str)
+    builtin('str', argument_tests=[anything_except_array])(to_str)
 
     @builtin('strip', argument_tests=[is_string])
     def strip(s):
         return s.strip()
 
     @builtin('rstrip', argument_tests=[is_string])
     def rstrip(s):
         return s.rstrip()
--- a/third_party/python/json-e/jsone/interpreter.py
+++ b/third_party/python/json-e/jsone/interpreter.py
@@ -90,17 +90,18 @@ class ExpressionEvaluator(PrattParser):
             raise expectationError('unary +', 'number')
         return v
 
     @prefix("identifier")
     def identifier(self, token, pc):
         try:
             return self.context[token.value]
         except KeyError:
-            raise TemplateError('no context value named "{}"'.format(token.value))
+            raise TemplateError(
+                'no context value named "{}"'.format(token.value))
 
     @prefix("null")
     def null(self, token, pc):
         return None
 
     @prefix("[")
     def array_bracket(self, token, pc):
         return parseList(pc, ',', ']')
@@ -174,17 +175,18 @@ class ExpressionEvaluator(PrattParser):
     @infix(".")
     def property_dot(self, left, token, pc):
         if not isinstance(left, dict):
             raise expectationError('.', 'object')
         k = pc.require('identifier').value
         try:
             return left[k]
         except KeyError:
-            raise TemplateError('{} not found in {}'.format(k, json.dumps(left)))
+            raise TemplateError(
+                '{} not found in {}'.format(k, json.dumps(left)))
 
     @infix("(")
     def function_call(self, left, token, pc):
         if not callable(left):
             raise TemplateError('function call', 'callable')
         args = parseList(pc, ',', ')')
         return left(*args)
 
@@ -208,17 +210,18 @@ class ExpressionEvaluator(PrattParser):
         right = pc.parse(token.kind)
         if isinstance(right, dict):
             if not isinstance(left, string):
                 raise expectationError('in-object', 'string on left side')
         elif isinstance(right, string):
             if not isinstance(left, string):
                 raise expectationError('in-string', 'string on left side')
         elif not isinstance(right, list):
-            raise expectationError('in', 'Array, string, or object on right side')
+            raise expectationError(
+                'in', 'Array, string, or object on right side')
         try:
             return left in right
         except TypeError:
             raise expectationError('in', 'scalar value, collection')
 
 
 def isNumber(v):
     return isinstance(v, (int, float)) and not isinstance(v, bool)
@@ -278,9 +281,8 @@ def accessProperty(value, a, b, is_inter
         raise expectationError('[..]', 'object, array, or string')
     if not isinstance(a, string):
         raise expectationError('[..]', 'string index')
 
     try:
         return value[a]
     except KeyError:
         return None
-        #raise TemplateError('{} not found in {}'.format(a, json.dumps(value)))
--- a/third_party/python/json-e/jsone/prattparser.py
+++ b/third_party/python/json-e/jsone/prattparser.py
@@ -106,22 +106,24 @@ class PrattParser(with_metaclass(PrattPa
     def _generate_tokens(self, source):
         offset = 0
         while True:
             start = offset
             remainder = source[offset:]
             mo = self.token_re.match(remainder)
             if not mo:
                 if remainder:
-                    raise SyntaxError("Unexpected input: '{}'".format(remainder))
+                    raise SyntaxError(
+                        "Unexpected input: '{}'".format(remainder))
                 break
             offset += mo.end()
 
             # figure out which token matched (note that idx is 0-based)
-            indexes = list(filter(lambda x: x[1] is not None, enumerate(mo.groups())))
+            indexes = list(
+                filter(lambda x: x[1] is not None, enumerate(mo.groups())))
             if indexes:
                 idx = indexes[0][0]
                 yield Token(
                     kind=self.tokens[idx],
                     value=mo.group(idx + 1),  # (mo.group is 1-based)
                     start=start,
                     end=offset)
 
--- a/third_party/python/json-e/jsone/render.py
+++ b/third_party/python/json-e/jsone/render.py
@@ -4,84 +4,102 @@ import re
 import json as json
 from .shared import JSONTemplateError, TemplateError, DeleteMarker, string, to_str
 from . import shared
 from .interpreter import ExpressionEvaluator
 from .six import viewitems
 import functools
 
 operators = {}
+IDENTIFIER_RE = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*$')
 
 
 def operator(name):
     def wrap(fn):
         operators[name] = fn
         return fn
     return wrap
 
 
 def evaluateExpression(expr, context):
     evaluator = ExpressionEvaluator(context)
     return evaluator.parse(expr)
 
 
 _interpolation_start_re = re.compile(r'\$?\${')
+
+
 def interpolate(string, context):
     mo = _interpolation_start_re.search(string)
     if not mo:
         return string
 
     result = []
     evaluator = ExpressionEvaluator(context)
 
     while True:
         result.append(string[:mo.start()])
         if mo.group() != '$${':
             string = string[mo.end():]
             parsed, offset = evaluator.parseUntilTerminator(string, '}')
             if isinstance(parsed, (list, dict)):
                 raise TemplateError(
                     "interpolation of '{}' produced an array or object".format(string[:offset]))
-            result.append(to_str(parsed))
+            if to_str(parsed) == "null":
+                result.append("")
+            else:
+                result.append(to_str(parsed))
             string = string[offset + 1:]
         else:  # found `$${`
             result.append('${')
             string = string[mo.end():]
 
         mo = _interpolation_start_re.search(string)
         if not mo:
             result.append(string)
             break
 
     return ''.join(result)
 
 
+def checkUndefinedProperties(template, allowed):
+    unknownKeys = []
+    combined = "|".join(allowed) + "$"
+    unknownKeys = [key for key in sorted(template)
+                   if not re.match(combined, key)]
+    if unknownKeys:
+        raise TemplateError(allowed[0].replace('\\', '') +
+                            " has undefined properties: " + " ".join(unknownKeys))
+
+
 @operator('$eval')
 def eval(template, context):
     return evaluateExpression(renderValue(template['$eval'], context), context)
 
 
 @operator('$flatten')
 def flatten(template, context):
+    checkUndefinedProperties(template, ['\$flatten'])
     value = renderValue(template['$flatten'], context)
     if not isinstance(value, list):
         raise TemplateError('$flatten value must evaluate to an array')
 
     def gen():
         for e in value:
             if isinstance(e, list):
                 for e2 in e:
                     yield e2
             else:
                 yield e
     return list(gen())
 
 
 @operator('$flattenDeep')
 def flattenDeep(template, context):
+    checkUndefinedProperties(template, ['\$flattenDeep'])
     value = renderValue(template['$flattenDeep'], context)
     if not isinstance(value, list):
         raise TemplateError('$flattenDeep value must evaluate to an array')
 
     def gen(value):
         if isinstance(value, list):
             for e in value:
                 for sub in gen(e):
@@ -89,68 +107,80 @@ def flattenDeep(template, context):
         else:
             yield value
 
     return list(gen(value))
 
 
 @operator('$fromNow')
 def fromNow(template, context):
+    checkUndefinedProperties(template, ['\$fromNow', 'from'])
     offset = renderValue(template['$fromNow'], context)
-    reference = renderValue(template['from'], context) if 'from' in template else context.get('now')
+    reference = renderValue(
+        template['from'], context) if 'from' in template else context.get('now')
 
     if not isinstance(offset, string):
         raise TemplateError("$fromNow expects a string")
     return shared.fromNow(offset, reference)
 
 
 @operator('$if')
 def ifConstruct(template, context):
+    checkUndefinedProperties(template, ['\$if', 'then', 'else'])
     condition = evaluateExpression(template['$if'], context)
     try:
         if condition:
             rv = template['then']
         else:
             rv = template['else']
     except KeyError:
         return DeleteMarker
     return renderValue(rv, context)
 
 
 @operator('$json')
 def jsonConstruct(template, context):
+    checkUndefinedProperties(template, ['\$json'])
     value = renderValue(template['$json'], context)
     return json.dumps(value, separators=(',', ':'))
 
 
 @operator('$let')
 def let(template, context):
+    checkUndefinedProperties(template, ['\$let', 'in'])
     variables = renderValue(template['$let'], context)
     if not isinstance(variables, dict):
         raise TemplateError("$let value must evaluate to an object")
+    else:
+        if not all(IDENTIFIER_RE.match(variableNames) for variableNames in variables.keys()):
+            raise TemplateError('top level keys of $let must follow /[a-zA-Z_][a-zA-Z0-9_]*/')
+
     subcontext = context.copy()
     subcontext.update(variables)
     try:
         in_expression = template['in']
     except KeyError:
         raise TemplateError("$let operator requires an `in` clause")
     return renderValue(in_expression, subcontext)
 
 
 @operator('$map')
 def map(template, context):
+    EACH_RE = 'each\([a-zA-Z_][a-zA-Z0-9_]*\)'
+    checkUndefinedProperties(template, ['\$map', EACH_RE])
     value = renderValue(template['$map'], context)
     if not isinstance(value, list) and not isinstance(value, dict):
         raise TemplateError("$map value must evaluate to an array or object")
 
     is_obj = isinstance(value, dict)
 
     each_keys = [k for k in template if k.startswith('each(')]
     if len(each_keys) != 1:
-        raise TemplateError("$map requires exactly one other property, each(..)")
+        raise TemplateError(
+            "$map requires exactly one other property, each(..)")
     each_key = each_keys[0]
     each_var = each_key[5:-1]
     each_template = template[each_key]
 
     if is_obj:
         value = [{'key': v[0], 'val': v[1]} for v in value.items()]
 
     def gen():
@@ -167,30 +197,35 @@ def map(template, context):
             v.update(e)
         return v
     else:
         return list(gen())
 
 
 @operator('$merge')
 def merge(template, context):
+    checkUndefinedProperties(template, ['\$merge'])
     value = renderValue(template['$merge'], context)
     if not isinstance(value, list) or not all(isinstance(e, dict) for e in value):
-        raise TemplateError("$merge value must evaluate to an array of objects")
+        raise TemplateError(
+            "$merge value must evaluate to an array of objects")
     v = dict()
     for e in value:
         v.update(e)
     return v
 
 
 @operator('$mergeDeep')
 def merge(template, context):
+    checkUndefinedProperties(template, ['\$mergeDeep'])
     value = renderValue(template['$mergeDeep'], context)
     if not isinstance(value, list) or not all(isinstance(e, dict) for e in value):
-        raise TemplateError("$mergeDeep value must evaluate to an array of objects")
+        raise TemplateError(
+            "$mergeDeep value must evaluate to an array of objects")
+
     def merge(l, r):
         if isinstance(l, list) and isinstance(r, list):
             return l + r
         if isinstance(l, dict) and isinstance(r, dict):
             res = l.copy()
             for k, v in viewitems(r):
                 if k in l:
                     res[k] = merge(l[k], v)
@@ -200,24 +235,27 @@ def merge(template, context):
         return r
     if len(value) == 0:
         return {}
     return functools.reduce(merge, value[1:], value[0])
 
 
 @operator('$reverse')
 def reverse(template, context):
+    checkUndefinedProperties(template, ['\$reverse'])
     value = renderValue(template['$reverse'], context)
     if not isinstance(value, list):
         raise TemplateError("$reverse value must evaluate to an array")
     return list(reversed(value))
 
 
 @operator('$sort')
 def sort(template, context):
+    BY_RE = 'by\([a-zA-Z_][a-zA-Z0-9_]*\)'
+    checkUndefinedProperties(template, ['\$sort', BY_RE])
     value = renderValue(template['$sort'], context)
     if not isinstance(value, list):
         raise TemplateError("$sort value must evaluate to an array")
 
     # handle by(..) if given, applying the schwartzian transform
     by_keys = [k for k in template if k.startswith('by(')]
     if len(by_keys) == 1:
         by_key = by_keys[0]
@@ -257,24 +295,28 @@ def renderValue(template, context):
         matches = [k for k in template if k in operators]
         if matches:
             if len(matches) > 1:
                 raise TemplateError("only one operator allowed")
             return operators[matches[0]](template, context)
 
         def updated():
             for k, v in viewitems(template):
-                if k.startswith('$$') and k[1:] in operators:
+                if k.startswith('$$'):
                     k = k[1:]
+                elif k.startswith('$') and IDENTIFIER_RE.match(k[1:]):
+                    raise TemplateError(
+                        '$<identifier> is reserved; ues $$<identifier>')
                 else:
                     k = interpolate(k, context)
+
                 try:
                     v = renderValue(v, context)
                 except JSONTemplateError as e:
-                    if re.match('^[a-zA-Z][a-zA-Z0-9]*$', k):
+                    if IDENTIFIER_RE.match(k):
                         e.add_location('.{}'.format(k))
                     else:
                         e.add_location('[{}]'.format(json.dumps(k)))
                     raise
                 if v is not DeleteMarker:
                     yield k, v
         return dict(updated())
 
--- a/third_party/python/json-e/jsone/shared.py
+++ b/third_party/python/json-e/jsone/shared.py
@@ -80,18 +80,19 @@ def fromNow(offset, reference):
     delta = datetime.timedelta(
         weeks=int(m.group('weeks') or 0),
         days=days,
         hours=hours,
         minutes=minutes,
         seconds=seconds,
     )
 
-    if isinstance(reference, str):
-        reference = datetime.datetime.strptime(reference, '%Y-%m-%dT%H:%M:%S.%fZ')
+    if isinstance(reference, string):
+        reference = datetime.datetime.strptime(
+            reference, '%Y-%m-%dT%H:%M:%S.%fZ')
     elif reference is None:
         reference = datetime.datetime.utcnow()
     return stringDate(reference + delta if future else reference - delta)
 
 
 datefmt_re = re.compile(r'(\.[0-9]{3})[0-9]*(\+00:00)?')
 
 
@@ -107,13 +108,14 @@ def to_str(v):
 
 
 def stringDate(date):
     # Convert to isoFormat
     string = date.isoformat()
     string = datefmt_re.sub(r'\1Z', string)
     return string
 
+
 # the base class for strings, regardless of python version
 try:
     string = basestring
 except NameError:
     string = str
--- a/third_party/python/json-e/jsone/six.py
+++ b/third_party/python/json-e/jsone/six.py
@@ -1,20 +1,23 @@
 import sys
 import operator
 
 # https://github.com/benjaminp/six/blob/2c3492a9f16d294cd5e6b43d6110c5a3a2e58b4c/six.py#L818
+
+
 def with_metaclass(meta, *bases):
     """Create a base class with a metaclass."""
     # This requires a bit of explanation: the basic idea is to make a dummy
     # metaclass for one level of class instantiation that replaces itself with
     # the actual metaclass.
     class metaclass(meta):
 
         def __new__(cls, name, this_bases, d):
             return meta(name, bases, d)
     return type.__new__(metaclass, 'temporary_class', (), {})
 
+
 # https://github.com/benjaminp/six/blob/2c3492a9f16d294cd5e6b43d6110c5a3a2e58b4c/six.py#L578
 if sys.version_info[0] == 3:
     viewitems = operator.methodcaller("items")
 else:
     viewitems = operator.methodcaller("viewitems")