--- a/third_party/python/json-e/jsone/__init__.py
+++ b/third_party/python/json-e/jsone/__init__.py
@@ -1,22 +1,21 @@
from __future__ import absolute_import, print_function, unicode_literals
-import datetime
import re
from .render import renderValue
-from .shared import JSONTemplateError, DeleteMarker, TemplateError
+from .shared import JSONTemplateError, DeleteMarker, TemplateError, fromNow
from . import builtins
_context_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*$')
def render(template, context):
if not all(_context_re.match(c) for c in context):
raise TemplateError('top level keys of context must follow '
- '/[a-zA-Z_][a-zA-Z0-9_]*/')
- full_context = {'now': datetime.datetime.utcnow()}
+ '/[a-zA-Z_][a-zA-Z0-9_]*/')
+ full_context = {'now': fromNow('0 seconds', None)}
full_context.update(builtins.build(full_context))
full_context.update(context)
rv = renderValue(template, full_context)
if rv is DeleteMarker:
return None
return rv
--- a/third_party/python/json-e/jsone/builtins.py
+++ b/third_party/python/json-e/jsone/builtins.py
@@ -9,17 +9,18 @@ class BuiltinError(JSONTemplateError):
def build(context):
builtins = {}
def builtin(name, variadic=None, argument_tests=None, minArgs=None):
def wrap(fn):
def bad(reason=None):
- raise BuiltinError((reason or 'invalid arguments to {}').format(name))
+ raise BuiltinError(
+ (reason or 'invalid arguments to builtin: {}').format(name))
if variadic:
def invoke(*args):
if minArgs:
if len(args) < minArgs:
bad("too few arguments to {}")
for arg in args:
if not variadic(arg):
bad()
@@ -46,16 +47,19 @@ def build(context):
return isinstance(v, (int, float)) and not isinstance(v, bool)
def is_string(v):
return isinstance(v, string)
def is_string_or_array(v):
return isinstance(v, (string, list))
+ def anything_except_array(v):
+ return isinstance(v, (string, int, float, bool)) or v is None
+
def anything(v):
return isinstance(v, (string, int, float, list, dict)) or v is None or callable(v)
# ---
builtin('min', variadic=is_number, minArgs=1)(min)
builtin('max', variadic=is_number, minArgs=1)(max)
builtin('sqrt', argument_tests=[is_number])(math.sqrt)
@@ -73,17 +77,17 @@ def build(context):
def lowercase(v):
return v.lower()
@builtin('uppercase', argument_tests=[is_string])
def lowercase(v):
return v.upper()
builtin('len', argument_tests=[is_string_or_array])(len)
- builtin('str', argument_tests=[anything])(to_str)
+ builtin('str', argument_tests=[anything_except_array])(to_str)
@builtin('strip', argument_tests=[is_string])
def strip(s):
return s.strip()
@builtin('rstrip', argument_tests=[is_string])
def rstrip(s):
return s.rstrip()
--- a/third_party/python/json-e/jsone/interpreter.py
+++ b/third_party/python/json-e/jsone/interpreter.py
@@ -90,17 +90,18 @@ class ExpressionEvaluator(PrattParser):
raise expectationError('unary +', 'number')
return v
@prefix("identifier")
def identifier(self, token, pc):
try:
return self.context[token.value]
except KeyError:
- raise TemplateError('no context value named "{}"'.format(token.value))
+ raise TemplateError(
+ 'no context value named "{}"'.format(token.value))
@prefix("null")
def null(self, token, pc):
return None
@prefix("[")
def array_bracket(self, token, pc):
return parseList(pc, ',', ']')
@@ -174,17 +175,18 @@ class ExpressionEvaluator(PrattParser):
@infix(".")
def property_dot(self, left, token, pc):
if not isinstance(left, dict):
raise expectationError('.', 'object')
k = pc.require('identifier').value
try:
return left[k]
except KeyError:
- raise TemplateError('{} not found in {}'.format(k, json.dumps(left)))
+ raise TemplateError(
+ '{} not found in {}'.format(k, json.dumps(left)))
@infix("(")
def function_call(self, left, token, pc):
if not callable(left):
raise TemplateError('function call', 'callable')
args = parseList(pc, ',', ')')
return left(*args)
@@ -208,17 +210,18 @@ class ExpressionEvaluator(PrattParser):
right = pc.parse(token.kind)
if isinstance(right, dict):
if not isinstance(left, string):
raise expectationError('in-object', 'string on left side')
elif isinstance(right, string):
if not isinstance(left, string):
raise expectationError('in-string', 'string on left side')
elif not isinstance(right, list):
- raise expectationError('in', 'Array, string, or object on right side')
+ raise expectationError(
+ 'in', 'Array, string, or object on right side')
try:
return left in right
except TypeError:
raise expectationError('in', 'scalar value, collection')
def isNumber(v):
return isinstance(v, (int, float)) and not isinstance(v, bool)
@@ -278,9 +281,8 @@ def accessProperty(value, a, b, is_inter
raise expectationError('[..]', 'object, array, or string')
if not isinstance(a, string):
raise expectationError('[..]', 'string index')
try:
return value[a]
except KeyError:
return None
- #raise TemplateError('{} not found in {}'.format(a, json.dumps(value)))
--- a/third_party/python/json-e/jsone/prattparser.py
+++ b/third_party/python/json-e/jsone/prattparser.py
@@ -106,22 +106,24 @@ class PrattParser(with_metaclass(PrattPa
def _generate_tokens(self, source):
offset = 0
while True:
start = offset
remainder = source[offset:]
mo = self.token_re.match(remainder)
if not mo:
if remainder:
- raise SyntaxError("Unexpected input: '{}'".format(remainder))
+ raise SyntaxError(
+ "Unexpected input: '{}'".format(remainder))
break
offset += mo.end()
# figure out which token matched (note that idx is 0-based)
- indexes = list(filter(lambda x: x[1] is not None, enumerate(mo.groups())))
+ indexes = list(
+ filter(lambda x: x[1] is not None, enumerate(mo.groups())))
if indexes:
idx = indexes[0][0]
yield Token(
kind=self.tokens[idx],
value=mo.group(idx + 1), # (mo.group is 1-based)
start=start,
end=offset)
--- a/third_party/python/json-e/jsone/render.py
+++ b/third_party/python/json-e/jsone/render.py
@@ -4,84 +4,102 @@ import re
import json as json
from .shared import JSONTemplateError, TemplateError, DeleteMarker, string, to_str
from . import shared
from .interpreter import ExpressionEvaluator
from .six import viewitems
import functools
operators = {}
+IDENTIFIER_RE = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*$')
def operator(name):
def wrap(fn):
operators[name] = fn
return fn
return wrap
def evaluateExpression(expr, context):
evaluator = ExpressionEvaluator(context)
return evaluator.parse(expr)
_interpolation_start_re = re.compile(r'\$?\${')
+
+
def interpolate(string, context):
mo = _interpolation_start_re.search(string)
if not mo:
return string
result = []
evaluator = ExpressionEvaluator(context)
while True:
result.append(string[:mo.start()])
if mo.group() != '$${':
string = string[mo.end():]
parsed, offset = evaluator.parseUntilTerminator(string, '}')
if isinstance(parsed, (list, dict)):
raise TemplateError(
"interpolation of '{}' produced an array or object".format(string[:offset]))
- result.append(to_str(parsed))
+ if to_str(parsed) == "null":
+ result.append("")
+ else:
+ result.append(to_str(parsed))
string = string[offset + 1:]
else: # found `$${`
result.append('${')
string = string[mo.end():]
mo = _interpolation_start_re.search(string)
if not mo:
result.append(string)
break
return ''.join(result)
+def checkUndefinedProperties(template, allowed):
+ unknownKeys = []
+ combined = "|".join(allowed) + "$"
+ unknownKeys = [key for key in sorted(template)
+ if not re.match(combined, key)]
+ if unknownKeys:
+ raise TemplateError(allowed[0].replace('\\', '') +
+ " has undefined properties: " + " ".join(unknownKeys))
+
+
@operator('$eval')
def eval(template, context):
return evaluateExpression(renderValue(template['$eval'], context), context)
@operator('$flatten')
def flatten(template, context):
+ checkUndefinedProperties(template, ['\$flatten'])
value = renderValue(template['$flatten'], context)
if not isinstance(value, list):
raise TemplateError('$flatten value must evaluate to an array')
def gen():
for e in value:
if isinstance(e, list):
for e2 in e:
yield e2
else:
yield e
return list(gen())
@operator('$flattenDeep')
def flattenDeep(template, context):
+ checkUndefinedProperties(template, ['\$flattenDeep'])
value = renderValue(template['$flattenDeep'], context)
if not isinstance(value, list):
raise TemplateError('$flattenDeep value must evaluate to an array')
def gen(value):
if isinstance(value, list):
for e in value:
for sub in gen(e):
@@ -89,68 +107,80 @@ def flattenDeep(template, context):
else:
yield value
return list(gen(value))
@operator('$fromNow')
def fromNow(template, context):
+ checkUndefinedProperties(template, ['\$fromNow', 'from'])
offset = renderValue(template['$fromNow'], context)
- reference = renderValue(template['from'], context) if 'from' in template else context.get('now')
+ reference = renderValue(
+ template['from'], context) if 'from' in template else context.get('now')
if not isinstance(offset, string):
raise TemplateError("$fromNow expects a string")
return shared.fromNow(offset, reference)
@operator('$if')
def ifConstruct(template, context):
+ checkUndefinedProperties(template, ['\$if', 'then', 'else'])
condition = evaluateExpression(template['$if'], context)
try:
if condition:
rv = template['then']
else:
rv = template['else']
except KeyError:
return DeleteMarker
return renderValue(rv, context)
@operator('$json')
def jsonConstruct(template, context):
+ checkUndefinedProperties(template, ['\$json'])
value = renderValue(template['$json'], context)
return json.dumps(value, separators=(',', ':'))
@operator('$let')
def let(template, context):
+ checkUndefinedProperties(template, ['\$let', 'in'])
variables = renderValue(template['$let'], context)
if not isinstance(variables, dict):
raise TemplateError("$let value must evaluate to an object")
+ else:
+ if not all(IDENTIFIER_RE.match(variableNames) for variableNames in variables.keys()):
+ raise TemplateError('top level keys of $let must follow /[a-zA-Z_][a-zA-Z0-9_]*/')
+
subcontext = context.copy()
subcontext.update(variables)
try:
in_expression = template['in']
except KeyError:
raise TemplateError("$let operator requires an `in` clause")
return renderValue(in_expression, subcontext)
@operator('$map')
def map(template, context):
+ EACH_RE = 'each\([a-zA-Z_][a-zA-Z0-9_]*\)'
+ checkUndefinedProperties(template, ['\$map', EACH_RE])
value = renderValue(template['$map'], context)
if not isinstance(value, list) and not isinstance(value, dict):
raise TemplateError("$map value must evaluate to an array or object")
is_obj = isinstance(value, dict)
each_keys = [k for k in template if k.startswith('each(')]
if len(each_keys) != 1:
- raise TemplateError("$map requires exactly one other property, each(..)")
+ raise TemplateError(
+ "$map requires exactly one other property, each(..)")
each_key = each_keys[0]
each_var = each_key[5:-1]
each_template = template[each_key]
if is_obj:
value = [{'key': v[0], 'val': v[1]} for v in value.items()]
def gen():
@@ -167,30 +197,35 @@ def map(template, context):
v.update(e)
return v
else:
return list(gen())
@operator('$merge')
def merge(template, context):
+ checkUndefinedProperties(template, ['\$merge'])
value = renderValue(template['$merge'], context)
if not isinstance(value, list) or not all(isinstance(e, dict) for e in value):
- raise TemplateError("$merge value must evaluate to an array of objects")
+ raise TemplateError(
+ "$merge value must evaluate to an array of objects")
v = dict()
for e in value:
v.update(e)
return v
@operator('$mergeDeep')
def merge(template, context):
+ checkUndefinedProperties(template, ['\$mergeDeep'])
value = renderValue(template['$mergeDeep'], context)
if not isinstance(value, list) or not all(isinstance(e, dict) for e in value):
- raise TemplateError("$mergeDeep value must evaluate to an array of objects")
+ raise TemplateError(
+ "$mergeDeep value must evaluate to an array of objects")
+
def merge(l, r):
if isinstance(l, list) and isinstance(r, list):
return l + r
if isinstance(l, dict) and isinstance(r, dict):
res = l.copy()
for k, v in viewitems(r):
if k in l:
res[k] = merge(l[k], v)
@@ -200,24 +235,27 @@ def merge(template, context):
return r
if len(value) == 0:
return {}
return functools.reduce(merge, value[1:], value[0])
@operator('$reverse')
def reverse(template, context):
+ checkUndefinedProperties(template, ['\$reverse'])
value = renderValue(template['$reverse'], context)
if not isinstance(value, list):
raise TemplateError("$reverse value must evaluate to an array")
return list(reversed(value))
@operator('$sort')
def sort(template, context):
+ BY_RE = 'by\([a-zA-Z_][a-zA-Z0-9_]*\)'
+ checkUndefinedProperties(template, ['\$sort', BY_RE])
value = renderValue(template['$sort'], context)
if not isinstance(value, list):
raise TemplateError("$sort value must evaluate to an array")
# handle by(..) if given, applying the schwartzian transform
by_keys = [k for k in template if k.startswith('by(')]
if len(by_keys) == 1:
by_key = by_keys[0]
@@ -257,24 +295,28 @@ def renderValue(template, context):
matches = [k for k in template if k in operators]
if matches:
if len(matches) > 1:
raise TemplateError("only one operator allowed")
return operators[matches[0]](template, context)
def updated():
for k, v in viewitems(template):
- if k.startswith('$$') and k[1:] in operators:
+ if k.startswith('$$'):
k = k[1:]
+ elif k.startswith('$') and IDENTIFIER_RE.match(k[1:]):
+ raise TemplateError(
+ '$<identifier> is reserved; ues $$<identifier>')
else:
k = interpolate(k, context)
+
try:
v = renderValue(v, context)
except JSONTemplateError as e:
- if re.match('^[a-zA-Z][a-zA-Z0-9]*$', k):
+ if IDENTIFIER_RE.match(k):
e.add_location('.{}'.format(k))
else:
e.add_location('[{}]'.format(json.dumps(k)))
raise
if v is not DeleteMarker:
yield k, v
return dict(updated())
--- a/third_party/python/json-e/jsone/shared.py
+++ b/third_party/python/json-e/jsone/shared.py
@@ -80,18 +80,19 @@ def fromNow(offset, reference):
delta = datetime.timedelta(
weeks=int(m.group('weeks') or 0),
days=days,
hours=hours,
minutes=minutes,
seconds=seconds,
)
- if isinstance(reference, str):
- reference = datetime.datetime.strptime(reference, '%Y-%m-%dT%H:%M:%S.%fZ')
+ if isinstance(reference, string):
+ reference = datetime.datetime.strptime(
+ reference, '%Y-%m-%dT%H:%M:%S.%fZ')
elif reference is None:
reference = datetime.datetime.utcnow()
return stringDate(reference + delta if future else reference - delta)
datefmt_re = re.compile(r'(\.[0-9]{3})[0-9]*(\+00:00)?')
@@ -107,13 +108,14 @@ def to_str(v):
def stringDate(date):
# Convert to isoFormat
string = date.isoformat()
string = datefmt_re.sub(r'\1Z', string)
return string
+
# the base class for strings, regardless of python version
try:
string = basestring
except NameError:
string = str
--- a/third_party/python/json-e/jsone/six.py
+++ b/third_party/python/json-e/jsone/six.py
@@ -1,20 +1,23 @@
import sys
import operator
# https://github.com/benjaminp/six/blob/2c3492a9f16d294cd5e6b43d6110c5a3a2e58b4c/six.py#L818
+
+
def with_metaclass(meta, *bases):
"""Create a base class with a metaclass."""
# This requires a bit of explanation: the basic idea is to make a dummy
# metaclass for one level of class instantiation that replaces itself with
# the actual metaclass.
class metaclass(meta):
def __new__(cls, name, this_bases, d):
return meta(name, bases, d)
return type.__new__(metaclass, 'temporary_class', (), {})
+
# https://github.com/benjaminp/six/blob/2c3492a9f16d294cd5e6b43d6110c5a3a2e58b4c/six.py#L578
if sys.version_info[0] == 3:
viewitems = operator.methodcaller("items")
else:
viewitems = operator.methodcaller("viewitems")