diff --git a/.gitignore b/.gitignore index b4f6fe8..fbb629b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ build dist/ MANIFEST *.egg-info +.tox diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7b699d7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ + Copyright (c) 2018 JSONBender authors. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. diff --git a/README.md b/README.md index 705020c..6ce11e8 100644 --- a/README.md +++ b/README.md @@ -4,22 +4,39 @@ About JSONBender is an embedded Python [DSL](https://en.wikipedia.org/wiki/Domain-specific_language) for transforming dicts. It's name is inspired by Nickelodeon's cartoon series [Avatar: The Last Airbender](https://en.wikipedia.org/wiki/Avatar:_The_Last_Airbender). -![aang](http://cdn-static.denofgeek.com/sites/denofgeek/files/9/21//the-last-airbender-aang-the-avatar.jpg) +![aang](aang.png) + + +License +--- + +JSONBender is licensed under the [MIT license](https://choosealicense.com/licenses/mit/). See the LICENSE file for more details. Installing --- ```bash -git clone git@github.com:Onyo/jsonbender.git -python setup.py install +pip install JSONBender +``` + +Contributing +--- + +If you want to contribute to JSONBender (thanks!), here's how to do it: + +1. Fork the repository. +2. Make sure the tests are all fine. Since we support both python 2 and 3, run the tests with [tox](https://github.com/tox-dev/tox): +```bash +tox tests ``` +3. Open the pull request! Usage --- -JSONBender works by calling the `bend()` function with a mapping and the source `dict` as arguments. It raises a `BendingException` if anyting bad happens during the transformation fase. +JSONBender works by calling the `bend()` function with a mapping and the source `dict` as arguments. It raises a `BendingException` if anyting bad happens during the transformation phase. The mapping itself is a dict whose values are benders, i.e. objects that represent the transformations to be done to the source dict. Ex: @@ -30,7 +47,9 @@ from jsonbender import bend, K, S MAPPING = { - 'fullName': S('customer', 'first_name') + K(' ') + S('customer', 'last_name'), + 'fullName': (S('customer', 'first_name') + + K(' ') + + S('customer', 'last_name')), 'city': S('address', 'city'), } @@ -53,36 +72,356 @@ print(json.dumps(result)) {"city": "Sicily", "fullName": "Inigo Montoya"} ``` -###Benders -####K +### Benders + + +#### Selectors -`K()` is a selector for constant values: It takes any value as a parameter and always returns that value regardless of the source dict. +##### K +`K()` is a selector for constant values: +It takes any value as a parameter and always returns that value regardless of the input. -####S + +##### S `S()` is a selector for accessing keys and indices: It takes a variable number of keys / indices and returns the corresponding value on the source dict: ```python +from jsonbender import bend, S + MAPPING = {'val': S('a', 'deeply', 'nested', 0, 'value')} ret = bend(MAPPING, {'a': {'deeply': {'nested': [{'value': 42}]}}}) assert ret == {'val': 42} ``` -####F -`F()` takes a function and optional args, and applies that function at bending time. It is useful for performing complex operations for which actual python code is necessary. F-benders can be composed with other benders using `<<` and `>>` to make them receive arbitrary values at bending time. Ex: +If any of keys may not exist, `S()` can be "annotated" by calling the `.optional(default)` method, which returns an instance of `OptionalS`. +`.optional()` takes a single parameter which is passed as the `default` value of `OptionalS`; it defaults to `None`. + +##### OptionalS + +`OptionalS` is like `S()` but does not raise errors when any of the keys is not found. Instead, it returns `None` or the `default` value that is passed on its construction. + +```python +from jsonbender import bend, OptionalS + +source = {'does': {'exist': 23}} + +MAPPING_1 = {'val': OptionalS('does', 'not', 'exist')} +ret = bend(MAPPING_1, source) +assert ret == {'val': None} + +MAPPING_2 = {'val': OptionalS('does', 'not', 'exist', default=27)} +ret = bend(MAPPING_2, source) +assert ret == {'val': 27} +``` + +For readability and reusability, prefer using `S().optional()` instead. + + +##### F +`F()` lifts a python callable into a Bender, so it can be called at bending time. +It is useful for performing more complex operations for which actual python code is necessary. + +The extra optional args and kwargs are passed to the function at +bending time after the given value. ```python +from jsonbender import bend, F, S + MAPPING = { 'total_number_of_keys': F(len), - 'number_of_str_keys': F(lambda source: len([k for k in source.iterkeys() + 'number_of_str_keys': F(lambda source: len([k for k in source.keys() if isinstance(k, str)])), - 'price_floor': S('price_as_str') >> F(float) >> F(int), + 'price_truncated': S('price_as_str') >> F(float) >> F(int), } ret = bend(MAPPING, {'price_as_str': '42.2', 'k1': 'v', 1: 'a'}) -assert ret == {'price_floor': 42, +assert ret == {'price_truncated': 42, 'total_number_of_keys': 3, 'number_of_str_keys': 2} ``` +If the function can't take certain values, you can protect it by calling the `.protect()` method. + +```python +import math +from jsonbender import bend, F, S + +MAPPING_1 = {'sqrt': S('val') >> F(math.sqrt).protect()} +assert bend(MAPPING_1, {'val': 4}) == {'sqrt': 2} +assert bend(MAPPING_1, {'val': None}) == {'sqrt': None} + +MAPPING_2 = {'sqrt': S('val') >> F(math.sqrt).protect(-1)} +assert bend(MAPPING_2, {'val': -1}) == {'sqrt': -1} +``` + + +#### Operators + +Benders implement most of python's binary operators. + +##### Arithmetic + +For the arithmetic `+`, `-`, `*`, `/`, +the behavior is to apply the operator to the bended values of each operand. + +```python +from jsonbender import bend, K, S + +a = S('a') +b = S('b') +MAPPING = {'add': a + b, 'sub': a - b, 'mul': a * b, 'div': a / b} +ret = bend(MAPPING, {'a': 10, 'b': 5}) +assert ret == {'add': 15, 'sub': 5, 'mul': 50, 'div': 2} + +ret = bend({'full_name': S('first_name') + K(' ') + S('last_name')}, + {'first_name': 'John', 'last_name': 'Doe'}) +assert ret == {'full_name': 'John Doe'} +``` + +##### Bitwise + +The bitwise operators are not yet implemented, except for the lshift (`<<`) and rshift (`>>`). +See "Composition" below. + + +#### List ops + +There are 4 benders for working with lists, inspired by the common functional programming operations. + +##### Reduce + +Similar to Python's `reduce()`. +Reduces an iterable into a single value by repeatedly applying the given +function to the elements. +The function must accept two parameters: the first is the accumulator (the +value returned from the last call), which defaults to the first element of +the iterable (it must be nonempty); the second is the next value from the +iterable. + + +```python +from jsonbender import bend, Reduce, S + +MAPPING = {'sum': S('ints') >> Reduce(lambda acc, i: acc + i)} +ret = bend(MAPPING, {'ints': [1, 4, 7, 9]}) +assert ret == {'sum': 21} +``` + +##### Filter + +Similar to Python's `filter()`. +Builds a new list with the elements of the iterable for which the given +function returns True. + +```python +from jsonbender import bend, Filter, S + +MAPPING = {'even': S('ints') >> Filter(lambda i: i % 2 == 0)} +ret = bend(MAPPING, {'ints': range(5)}) +assert ret == {'even': [0, 2, 4]} +``` + +##### Forall + +Similar to Python's `map()`. +Builds a new list by applying the given function to each element of the +iterable. + + +```python +from jsonbender import bend, Forall, S + +MAPPING = {'doubles': S('ints') >> Forall(lambda i: i * 2)} +ret = bend(MAPPING, {'ints': range(5)}) +assert ret == {'doubles': [0, 2, 4, 6, 8]} +``` + +For the common case of applying a JSONBender mapping to each element of a list, +the `.bend()` *class method* is provided, which returns a `ForallBend` instance +. `.bend()` takes the mapping and the context (optional) which are then passed +to `ForallBend`. + + +##### ForallBend +Bends each element of the list with given mapping and context. + +If no context is passed, it "inherits" at bend-time the context passed to the outer `bend()` call. + + +```python +from jsonbender import bend, S +from jsonbender.list_ops import ForallBend + +MAPPING = {'list_of_bs': S('list_of_as') >> ForallBend({'b': S('a')})} +source = {'list_of_as': [{'a': 23}, {'a': 27}]} +ret = bend(MAPPING, source) +assert ret == {'list_of_bs': [{'b': 23}, {'b': 27}]} +``` + +##### FlatForall + +Similar to Forall, but the given function must return an iterable for each +element of the iterable, which are than "flattened" into a single +list. + +```python +from jsonbender import bend, S +from jsonbender.list_ops import FlatForall + +MAPPING = {'doubles_triples': S('ints') >> FlatForall(lambda x: [x * 2, x * 3])} +source = {'ints': [2, 15, 50]} +ret = bend(MAPPING, source) +assert ret == {'doubles_triples': [4, 6, 30, 45, 100, 150]} +``` + +#### Control Flow + +Sometimes what bender to use must be decided at bending time, +so JSONBender provides 3 control flow structures: + + +##### Alternation + +Take any number of benders, and return the value of the first one that +doesn't raise a LookupError (KeyError, IndexError etc.). + +If all benders raise LookupError, re-raise the last raised exception. + +```python +from jsonbender import S +from jsonbender.control_flow import Alternation + +b = Alternation(S(1), S(0), S('key1')) + +b(['a', 'b']) # -> 'b' +b(['a']) # -> 'a' +try: + b([]) # -> TypeError +except TypeError: + pass + +try: + b({}) # -> KeyError +except KeyError: + pass + +b({'key1': 23}) # -> 23 +``` + +##### If + +Takes a condition bender, and two benders (both default to K(None)). +If the condition bender evaluates to true, return the value of the first +bender. If it evaluates to false, return the value of the second bender. + +```python +from jsonbender import K, S +from jsonbender.control_flow import If + +if_ = If(S('country') == K('China'), S('first_name'), S('last_name')) +if_({'country': 'China', + 'first_name': 'Li', + 'last_name': 'Na'}) # -> 'Li' + +if_({'country': 'Brazil', + 'first_name': 'Gustavo', + 'last_name': 'Kuerten'}) # -> 'Kuerten' +``` + +##### Switch + +Take a key bender, a 'case' container of benders and a default bender +(optional). + +The value returned by the key bender is used to get a bender from the +case container, which then returns the result. + +If the key is not in the case container, the default is used. + +If it's unavailable, raise the original LookupError. + +```python +from jsonbender import K, S +from jsonbender.control_flow import Switch + +b = Switch(S('service'), + {'twitter': S('handle'), + 'mastodon': S('handle') + K('@') + S('server')}, + default=S('email')) + +b({'service': 'twitter', 'handle': 'etandel'}) # -> 'etandel' +b({'service': 'mastodon', 'handle': 'etandel', + 'server': 'mastodon.social'}) # -> 'etandel@mastodon.social' +b({'service': 'facebook', + 'email': 'email@whatever.com'}) # -> 'email@whatever.com' +``` + +#### String ops + +JSONBender currently provides only one string-related bender. + +##### Format + +Return a formatted string just like `str.format()`. +Where the values to be formatted are given by benders as positional or +named parameters. + +It uses the same syntax as `str.format()` + +```python +from jsonbender import bend, Format, S + +MAPPING = {'formatted': Format('{} {} {last}', + S('first'), + S('second'), + last=S('last'))} +source = {'first': 'Edsger', 'second': 'W.', 'last': 'Dijkstra'} +ret = bend(MAPPING, source) +assert ret == {'formatted': 'Edsger W. Dijkstra'} +``` + + +### Composition + +All JSONBenders can be composed with other benders using `<<` and `>>` +to make them receive previously bended values. + +```python +from jsonbender import bend, F, S +from jsonbender.list_ops import Forall + +MAPPING = { + 'name': S('name'), + 'pythonista': S('prog_langs') >> Forall(str.lower) >> F(lambda ls: 'python' in ls), +} +source = { + 'name': 'Mary', + 'prog_langs': ['C', 'Python', 'Lua'], +} +ret = bend(MAPPING, source) +assert ret == {'name': 'Mary', 'pythonista': True} +``` + +### Context + +Sometimes it's necessary to use values at bending time that are not on the +source json and are not known at mapping time. +For these cases there is the optional `context` argument to `bend()` function. +Whatever you pass for the argument is can be used at bending time by the +`Context()` bender. + + +```python +from jsonbender import bend, Context, S + +MAPPING = { + 'name': S('name'), + 'age': (Context() >> S('year')) - S('birthyear'), +} +source = {'name': 'Mary', 'birthyear': 1990} +ret = bend(MAPPING, source, context={'year': 2016}) +assert ret == {'name': 'Mary', 'age': 26} +``` + diff --git a/aang.png b/aang.png new file mode 100644 index 0000000..f957c4f Binary files /dev/null and b/aang.png differ diff --git a/jsonbender/__init__.py b/jsonbender/__init__.py index afff7f2..1589dfb 100644 --- a/jsonbender/__init__.py +++ b/jsonbender/__init__.py @@ -1,8 +1,9 @@ -from jsonbender.core import Bender, bend, BendingException +from jsonbender.core import Bender, Context, bend, BendingException from jsonbender.list_ops import FlatForall, Forall, Filter, Reduce from jsonbender.string_ops import Format -from jsonbender.selectors import F, K, S +from jsonbender.selectors import F, K, S, OptionalS +from jsonbender.control_flow import Alternation, If, Switch -__version__ = '0.3' +__version__ = '0.9.3' diff --git a/jsonbender/_compat.py b/jsonbender/_compat.py new file mode 100644 index 0000000..5c4082d --- /dev/null +++ b/jsonbender/_compat.py @@ -0,0 +1,8 @@ +import sys + +PY2 = sys.version_info[0] == 2 + +if not PY2: + iteritems = lambda d: iter(d.items()) +else: + iteritems = lambda d: d.iteritems() diff --git a/jsonbender/control_flow.py b/jsonbender/control_flow.py new file mode 100644 index 0000000..3525006 --- /dev/null +++ b/jsonbender/control_flow.py @@ -0,0 +1,108 @@ +from jsonbender.core import Bender +from jsonbender.selectors import K + + +class If(Bender): + """ + Takes a condition bender, and two benders (both default to K(None)). + If the condition bender evaluates to true, return the value of the first + bender. If it evaluates to false, return the value of the second bender. + + Example: + ``` + if_ = If(S('country') == K('China'), S('first_name'), S('last_name')) + if_({'country': 'China', + 'first_name': 'Li', + 'last_name': 'Na'}) # -> 'Li' + + if_({'country': 'Brazil', + 'first_name': 'Gustavo', + 'last_name': 'Kuerten'}) # -> 'Kuerten' + ``` + """ + + def __init__(self, condition, when_true=K(None), when_false=K(None)): + self.condition = condition + self.when_true = when_true + self.when_false = when_false + + def execute(self, val): + return (self.when_true(val) + if self.condition(val) + else self.when_false(val)) + + +class Alternation(Bender): + """ + Take any number of benders, and return the value of the first one that + doesn't raise a LookupError (KeyError, IndexError etc.). + If all benders raise LookupError, re-raise the last raised exception. + + Example: + ``` + b = Alternation(S(1), S(0), S('key1')) + b(['a', 'b']) # -> 'b' + b(['a']) # -> 'a' + b([]) # -> KeyError + b({}) # -> KeyError + b({'key1': 23}) # -> 23 + ``` + """ + + def __init__(self, *benders): + self.benders = benders + + def execute(self, source): + exc = ValueError() + for bender in self.benders: + try: + result = bender(source) + except LookupError as e: + exc = e + else: + return result + else: + raise exc + + +class Switch(Bender): + """ + Take a key bender, a 'case' container of benders and a default bender + (optional). + The value returned by the key bender is used to get a bender from the + case container, which then returns the result. + If the key is not in the case container, the default is used. + If it's unavailable, raise the original LookupError. + + Example: + ``` + b = Switch(S('service'), + {'twitter': S('handle'), + 'mastodon': S('handle') + K('@') + S('server')}, + default=S('email')) + + b({'service': 'twitter', 'handle': 'etandel'}) # -> 'etandel' + b({'service': 'mastodon', 'handle': 'etandel', + 'server': 'mastodon.social'}) # -> 'etandel@mastodon.social' + b({'service': 'facebook', + 'email': 'email@whatever.com'}) # -> 'email@whatever.com' + ``` + """ + + def __init__(self, key_bender, cases, default=None): + self.key_bender = key_bender + self.cases = cases + self.default = default + + def execute(self, source): + key = self.key_bender(source) + try: + bender = self.cases[key] + except LookupError: + if self.default: + bender = self.default + else: + raise + + return bender(source) + diff --git a/jsonbender/core.py b/jsonbender/core.py index 867018e..50030f0 100644 --- a/jsonbender/core.py +++ b/jsonbender/core.py @@ -1,4 +1,4 @@ -from functools import partial +from jsonbender._compat import iteritems class Bender(object): @@ -15,14 +15,33 @@ class Bender(object): """ def __init__(self, *args, **kwargs): - raise NotImplementedError() + pass def __call__(self, source): - return self.execute(source) + return self.raw_execute(source).value + + def raw_execute(self, source): + transport = Transport.from_source(source) + return Transport(self.execute(transport.value), transport.context) def execute(self, source): raise NotImplementedError() + def __eq__(self, other): + return Eq(self, other) + + def __ne__(self, other): + return Ne(self, other) + + def __and__(self, other): + return And(self, other) + + def __or__(self, other): + return Or(self, other) + + def __invert__(self): + return Invert(self) + def __add__(self, other): return Add(self, other) @@ -35,6 +54,15 @@ def __mul__(self, other): def __div__(self, other): return Div(self, other) + def __neg__(self): + return Neg(self) + + def __truediv__(self, other): + return Div(self, other) + + def __floordiv__(self, other): + return Div(self, other) + def __rshift__(self, other): return Compose(self, other) @@ -58,12 +86,45 @@ def __init__(self, first, second): self._first = first self._second = second - def execute(self, source): - return self._second(self._first(source)) + def raw_execute(self, source): + return self._second.raw_execute(self._first.raw_execute(source)) -class BinaryOperator(Bender): +class UnaryOperator(Bender): + """ + Base class for unary bending operators. Should not be directly + instantiated. + Whenever a unary op is activated, the op() method is called with the + *value* (that is, the bender is implicitly activated). + + Subclasses must implement the op() method, which takes one value and + should return the desired result. + """ + + def __init__(self, bender): + self.bender = bender + + def op(self, v): + raise NotImplementedError() + + def raw_execute(self, source): + source = Transport.from_source(source) + val = self.op(self.bender(source)) + return Transport(val, source.context) + + +class Neg(UnaryOperator): + def op(self, v): + return -v + + +class Invert(UnaryOperator): + def op(self, v): + return not v + + +class BinaryOperator(Bender): """ Base class for binary bending operators. Should not be directly instantiated. @@ -82,8 +143,11 @@ def __init__(self, bender1, bender2): def op(self, v1, v2): raise NotImplementedError() - def execute(self, source): - return self.op(self._bender1(source), self._bender2(source)) + def raw_execute(self, source): + source = Transport.from_source(source) + val = self.op(self._bender1(source), + self._bender2(source)) + return Transport(val, source.context) class Add(BinaryOperator): @@ -106,11 +170,50 @@ def op(self, v1, v2): return float(v1) / float(v2) +class Eq(BinaryOperator): + def op(self, v1, v2): + return v1 == v2 + + +class Ne(BinaryOperator): + def op(self, v1, v2): + return v1 != v2 + + +class And(BinaryOperator): + def op(self, v1, v2): + return v1 and v2 + + +class Or(BinaryOperator): + def op(self, v1, v2): + return v1 or v2 + + +class Context(Bender): + def raw_execute(self, source): + transport = Transport.from_source(source) + return Transport(transport.context, transport.context) + + class BendingException(Exception): pass -def bend(mapping, source): +class Transport(object): + def __init__(self, value, context): + self.value = value + self.context = context + + @classmethod + def from_source(cls, source): + if isinstance(source, cls): + return source + else: + return cls(source, {}) + + +def bend(mapping, source, context=None): """ The main bending function. @@ -119,18 +222,28 @@ def bend(mapping, source): returns a new dict according to the provided map. """ - res = {} - for k, value in mapping.iteritems(): - if isinstance(value, Bender): + context = {} if context is None else context + transport = Transport(source, context) + return _bend(mapping, transport) + + +def _bend(mapping, transport): + if isinstance(mapping, list): + return [_bend(v, transport) for v in mapping] + + elif isinstance(mapping, dict): + res = {} + for k, v in iteritems(mapping): try: - newv = value(source) + res[k] = _bend(v, transport) except Exception as e: m = 'Error for key {}: {}'.format(k, str(e)) raise BendingException(m) - elif isinstance(value, list): - newv = map(lambda v: bend(v, source), value) - else: - newv = bend(value, source) - res[k] = newv - return res + return res + + elif isinstance(mapping, Bender): + return mapping(transport) + + else: + return mapping diff --git a/jsonbender/list_ops.py b/jsonbender/list_ops.py index 286eb41..6647582 100644 --- a/jsonbender/list_ops.py +++ b/jsonbender/list_ops.py @@ -1,69 +1,158 @@ +from functools import reduce from itertools import chain +from warnings import warn -from jsonbender.core import Bender +from jsonbender.core import Bender, bend, Transport class ListOp(Bender): """ Base class for operations on lists. Subclasses must implement the op() method, which takes the function passed - to the operator's __init__(), a list of *values* and should return the + to the operator's __init__(), an iterable, and should return the desired result. """ - def __init__(self, bender, function): - self._func = function - self._bender = bender + def __init__(self, *args): + if len(args) == 1: + self._func = args[0] + self._bender = None + # TODO: this is here for compatibility reasons. + elif len(args) == 2: + self._bender, self._func = args + msg = ('Passing a bender to {0} is deprecated.' + 'Please use {0} in a composition chain ' + '(see docs for more details).' + .format(type(self).__name__)) + warn(DeprecationWarning(msg)) + else: + msg = ('{} constructor only takes one parameter, {} given' + .format(type(self).__name__, len(args))) + raise TypeError(msg) def op(self, func, vals): raise NotImplementedError() def execute(self, source): - return self.op(self._func, self._bender(source)) + # TODO: this is here for compatibility reasons + if self._bender: + source = self._bender(source) + return self.op(self._func, source) class Forall(ListOp): """ + Similar to Python's map(). Builds a new list by applying the given function to each element of the - selected list. Similar to Python's map(). + iterable. + + Example: + ``` + Forall(lambda i: i * 2)(range(5)) # -> [0, 2, 4, 6, 8] + ``` """ - op = map + + def op(self, func, vals): + if vals is None: + return None + return list(map(func, vals)) + + @classmethod + def bend(cls, mapping, context=None): + """ + Return a ForallBend instance that bends each element of the list with the + given mapping. + + mapping: a JSONBender mapping as passed to the `bend()` function. + context: optional. the context that will be passed to `bend()`. + Note that if context is not passed, it defaults at bend-time + to the one passed to the outer mapping. + + Example: + ``` + source = [{'a': 23}, {'a': 27}] + bender = Forall.bend({'b': S('a')}) + bender(source) # -> [{'b': 23}, {'b': 27}] + ``` + + """ + return ForallBend(mapping, context) + + +class ForallBend(Forall): + """ + Bends each element of the list with given mapping and context. + + mapping: a JSONBender mapping as passed to the `bend()` function. + context: optional. the context that will be passed to `bend()`. + Note that if context is not passed, it defaults at bend-time + to the one passed to the outer mapping. + """ + + def __init__(self, mapping, context=None): + self._mapping = mapping + self._context = context + # TODO this is here for retrocompatibility reasons. + # remove this when ListOp also breaks retrocompatibility + self._bender = None + + def raw_execute(self, source): + transport = Transport.from_source(source) + context = self._context or transport.context + # ListOp.execute assumes the func is saved on self._func + self._func = lambda v: bend(self._mapping, v, context) + return Transport(self.execute(transport.value), transport.context) class Reduce(ListOp): """ - Reduces a list into a single value by repeatedly applying the given + Similar to Python's reduce(). + Reduces an iterable into a single value by repeatedly applying the given function to the elements. The function must accept two parameters: the first is the accumulator (the value returned from the last call), which defaults to the first element of - the list (hence, the list must be nonempty); the second is the next value fro the list. + the iterable (it must be nonempty); the second is the next value from the + iterable. Example: To sum a given list, - Reduce(K([1, 4, 6], lambda acc, i: acc+i) -> 11 + ``` + Reduce(lambda acc, i: acc + i)([1, 4, 6]) # -> 11 + ``` """ def op(self, func, vals): try: return reduce(func, vals) except TypeError as e: # empty list with no initial value - raise ValueError(e.message) + raise ValueError(e.args[0]) class Filter(ListOp): """ - Builds a new list with the elements of the selected list for which the - given function returns True. Similar to Python's filter(). + Similar to Python's filter(). + Builds a new list with the elements of the iterable for which the given + function returns True. + + Example: + ``` + Filter(lambda i: i % 2 == 0)(range(5)) # -> [0, 2, 4] + ``` """ - op = filter + + def op(self, func, vals): + return list(filter(func, vals)) class FlatForall(ListOp): """ - Similar to Forall, but the given function must return a list for each - element of the selected list, which are than "flattened" into a single + Similar to Forall, but the given function must return an iterable for each + element of the iterable, which are than "flattened" into a single list. - Example: FlatForall(K([1, 10, 100]), lambda x: [x-1, x+1]) -> - [[0, 2], [9, 11], [99, 101]] -> - [0, 1, 9, 11, 99, 101] + Example: + ``` + FlatForall(lambda x: [x-1, x+1])([1, 10, 100]) -> + [[0, 2], [9, 11], [99, 101]] -> + [0, 1, 9, 11, 99, 101] + ``` """ def op(self, func, vals): return list(chain.from_iterable(map(func, vals))) diff --git a/jsonbender/selectors.py b/jsonbender/selectors.py index 3bc6fdf..fd297cc 100644 --- a/jsonbender/selectors.py +++ b/jsonbender/selectors.py @@ -28,8 +28,50 @@ def execute(self, source): source = source[key] return source + def optional(self, default=None): + """ + Return an OptionalS with the same path and with the given `default`. + """ + return OptionalS(*self._path, default=default) + + +class OptionalS(S): + """ + Similar to S. However, if any of the keys doesn't exist, returns the + `default` value. + + `default` defaults to None. + Example: + OptionalS('a', 0, 'b', default=23).execute({'a': []}) -> 23 + """ + + def __init__(self, *path, **kwargs): + self.default = kwargs.get('default') + super(OptionalS, self).__init__(*path) + + def execute(self, source): + try: + ret = super(OptionalS, self).execute(source) + except LookupError: + return self.default + else: + return ret + class F(Bender): + """ + Lifts a python callable into a Bender, so it can be composed. + The extra positional and named parameters are passed to the function at + bending time after the given value. + + `func` is a callable + + Example: + ``` + f = F(sorted, key=lambda d: d['id']) + K([{'id': 3}, {'id': 1}]) >> f # -> [{'id': 1}, {'id': 3}] + ``` + """ def __init__(self, func, *args, **kwargs): self._func = func self._args = args @@ -38,3 +80,39 @@ def __init__(self, func, *args, **kwargs): def execute(self, value): return self._func(value, *self._args, **self._kwargs) + def protect(self, protect_against=None): + """ + Return a ProtectedF with the same parameters and with the given + `protect_against`. + """ + return ProtectedF(self._func, + *self._args, + protect_against=protect_against, + **self._kwargs) + + +class ProtectedF(F): + """ + Similar to F. + However, if the passing value equals the `protect_against` parameter, + don't execute the function and return the passed value. + + `protect_against` defaults to None. + Example: + ``` + f = ProtectedF(lambda i: 1.0 / i, protect_against=0.0) + f.execute(0) # -> 0 + ``` + + """ + def __init__(self, func, *args, **kwargs): + self._protect_against = kwargs.pop('protect_against', None) + super(ProtectedF, self).__init__(func, *args, **kwargs) + + def execute(self, value): + if value == self._protect_against: + return value + else: + return super(ProtectedF, self).execute(value) + + diff --git a/jsonbender/string_ops.py b/jsonbender/string_ops.py index 563b719..16ddc63 100644 --- a/jsonbender/string_ops.py +++ b/jsonbender/string_ops.py @@ -1,15 +1,56 @@ -from jsonbender.core import Bender +from jsonbender.core import Bender, Transport +from jsonbender._compat import iteritems class Format(Bender): + """ + Return a formatted string just like `str.format()`. + However, the values to be formatted are given by benders as positional or + named parameters. + + `format_string` is a template with the same syntax as `str.format()` + + Example: + ``` + fmt = Format('{} {} {last}', S('first'), S('second'), last=S('last')) + source = {'first': 'Edsger', 'second': 'W.', 'last': 'Dijkstra'} + fmt.execute(source) # -> 'Edsger W. Dijkstra' + ``` + """ def __init__(self, format_string, *args, **kwargs): self._format_str = format_string self._positional_benders = args self._named_benders = kwargs - def execute(self, source): + def raw_execute(self, source): + transport = Transport.from_source(source) args = [bender(source) for bender in self._positional_benders] kwargs = {k: bender(source) - for k, bender in self._named_benders.iteritems()} - return self._format_str.format(*args, **kwargs) + for k, bender in iteritems(self._named_benders)} + value = self._format_str.format(*args, **kwargs) + return Transport(value, transport.context) + + +class ProtectedFormat(Format): + """ + Returns a formatted String, like Python's built-in format. + If one of the arguments is None, it evaluates to None + Examples: + fmt = Format('{} {} {last}', S('first'), S('second'), last=S('last')) + source = {'first': 'Edsger', 'second': 'W.', 'last': 'Dijkstra'} + fmt.execute(source) # -> 'Edsger W. Dijkstra' + fmt = Format('{} {}', S('first'), S('second')) + source = {'first': 'Edsger'} + fmt.execute(source) # -> None + """ + def raw_execute(self, source): + # if any of the args to print are None, return None + if any( + [bender(source) is None for bender in self._positional_benders] + + [bender(source) is None for bender in self._named_benders.values()] + ): + # create an object with property value=None so it can be processed + return type(str('none_obj'), (object,), dict(value=None)) + # else just behave normally + return super(ProtectedFormat, self).raw_execute(source) diff --git a/jsonbender/test.py b/jsonbender/test.py new file mode 100644 index 0000000..757bf12 --- /dev/null +++ b/jsonbender/test.py @@ -0,0 +1,10 @@ +from jsonbender.core import Transport + + +class BenderTestMixin(object): + def assert_bender(self, bender, source, expected_value, + context=None, msg=None): + context = context or {} + got = bender(Transport(source, context)) + self.assertEqual(got, expected_value, msg) + diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..224a779 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md \ No newline at end of file diff --git a/setup.py b/setup.py index 8630b51..f022646 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,18 @@ setup( name='JSONBender', version=__version__, - description='Library for transforming dicts.', - packages=find_packages(), + description='Library for transforming JSON data between different formats.', + author='Elias Tandel', + author_email='backend@onyo.com', + url='https://github.com/Onyo/jsonbender', + download_url='https://codeload.github.com/Onyo/jsonbender/tar.gz/' + __version__, + keywords=['dsl', 'edsl', 'json'], + packages=['jsonbender'], + classifiers=[ + 'Intended Audience :: Developers', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.5', + ], ) diff --git a/tests/test_control_flow.py b/tests/test_control_flow.py new file mode 100644 index 0000000..c64e619 --- /dev/null +++ b/tests/test_control_flow.py @@ -0,0 +1,82 @@ +from operator import add +import unittest + +from jsonbender import Context, K, S, bend +from jsonbender.control_flow import If, Alternation, Switch +from jsonbender.test import BenderTestMixin + + +class TestIf(BenderTestMixin, unittest.TestCase): + def setUp(self): + self.na_li = {'country': 'China', + 'first_name': 'Li', + 'last_name': 'Na'} + self.guga = {'country': 'Brazil', + 'first_name': 'Gustavo', + 'last_name': 'Kuerten'} + + def test_if_true(self): + if_ = If(S('country') == K('China'), S('first_name'), S('last_name')) + self.assert_bender(if_, self.na_li, 'Li') + + def test_if_false(self): + if_ = If(S('country') == K('China'), S('first_name'), S('last_name')) + self.assert_bender(if_, self.guga, 'Kuerten') + + def test_if_true_default(self): + if_ = If(S('country') == K('China'), when_false=S('last_name')) + self.assert_bender(if_, self.na_li, None) + + def test_if_false_default(self): + if_ = If(S('country') == K('China'), S('first_name')) + self.assert_bender(if_, self.guga, None) + + +class TestAlternation(BenderTestMixin, unittest.TestCase): + def test_empty_benders(self): + self.assertRaises(ValueError, Alternation(), {}) + + def test_matches(self): + bender = Alternation(S(1), S(0), S('key1')) + self.assert_bender(bender, ['a', 'b'], 'b') + self.assert_bender(bender, ['a'], 'a') + self.assert_bender(bender, {'key1': 23}, 23) + + def test_no_match(self): + self.assertRaises(IndexError, Alternation(S(1)), []) + self.assertRaises(KeyError, Alternation(S(1)), {}) + + +class TestSwitch(BenderTestMixin, unittest.TestCase): + def test_match(self): + bender = Switch(S('service'), + {'twitter': S('handle'), + 'mastodon': S('handle') + K('@') + S('server')}, + default=S('email')) + + self.assert_bender(bender, + {'service': 'twitter', 'handle': 'etandel'}, + 'etandel') + self.assert_bender(bender, + {'service': 'mastodon', + 'handle': 'etandel', + 'server': 'mastodon.social'}, + 'etandel@mastodon.social') + + def test__no_match_with_default(self): + bender = Switch(S('service'), + {'twitter': S('handle'), + 'mastodon': S('handle') + K('@') + S('server')}, + default=S('email')) + self.assert_bender(bender, + {'service': 'facebook', + 'email': 'email@whatever.com'}, + 'email@whatever.com') + + def test__no_match_without_default(self): + self.assertRaises(KeyError, Switch(S('key'), {}), {'key': None}) + + +if __name__ == '__main__': + unittest.main() + diff --git a/tests/test_core.py b/tests/test_core.py index 8d98a7b..a421504 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,6 +1,10 @@ import unittest -from jsonbender import bend, BendingException, S, K +import sys + +from jsonbender import S, K +from jsonbender.core import bend, BendingException, Context +from jsonbender.test import BenderTestMixin class TestBend(unittest.TestCase): @@ -57,30 +61,91 @@ def test_nested_mapping_with_lists(self): } self.assertDictEqual(bend(mapping, source), expected) + def test_list_with_non_dict_elements(self): + mapping = {'k': ['foo1', S('bar1')]} + source = {'bar1': 'val 1'} + expected = {'k': ['foo1', 'val 1']} + self.assertDictEqual(bend(mapping, source), expected) + def test_bending_exception_is_raised_when_something_bad_happens(self): mapping = {'a': S('nonexistant')} source = {} self.assertRaises(BendingException, bend, mapping, source) + def test_constants_without_K(self): + mapping = {'a': 'a const value', 'b': 123} + self.assertDictEqual(bend(mapping, {}), + {'a': 'a const value', 'b': 123}) + + def test_context_shallow(self): + mapping = {'a': Context() >> S('b')} + res = bend(mapping, {}, context={'b': 23}) + self.assertDictEqual(res, {'a': 23}) -class TestOperators(unittest.TestCase): + def test_context_deep(self): + mapping = {'a': [{'a': Context() >> S('b')}]} + res = bend(mapping, {}, context={'b': 23}) + self.assertDictEqual(res, {'a': [{'a': 23}]}) + + +class TestOperators(unittest.TestCase, BenderTestMixin): def test_add(self): - self.assertEqual((S('v1') + K(2))({'v1': 5}), 7) + self.assert_bender(K(5) + K(2), None, 7) def test_sub(self): - self.assertEqual((S('v1') - K(2))({'v1': 5}), 3) + self.assert_bender(K(5) - K(2), None, 3) def test_mul(self): - self.assertEqual((S('v1') * K(2))({'v1': 5}), 10) + self.assert_bender(K(5) * K(2), None, 10) def test_div(self): - self.assertAlmostEqual((S('v1') / K(2))({'v1': 5}), 2.5, 2) - - -class TestGetItem(unittest.TestCase): + self.assert_bender(K(4) / K(2), None, 2) + self.assertAlmostEqual((K(5) / K(2))(None), 2.5, 2) + + def test_neg(self): + self.assert_bender(-K(1), None, -1) + self.assert_bender(-K(-1), None, 1) + + def test_op_with_context(self): + mapping = {'res': (Context() >> S('b')) - S('a')} + in_ = {'a': 23} + context = {'b': 27} + res = bend(mapping, in_, context=context) + self.assertEqual(res, {'res': 4}) + + def test_eq(self): + self.assert_bender(K(42) == K(42), None, True) + self.assert_bender(K(42) == K(27), None, False) + + def test_ne(self): + self.assert_bender(K(42) != K(42), None, False) + self.assert_bender(K(42) != K(27), None, True) + + def test_and(self): + self.assert_bender(K(True) & K(True), None, True) + self.assert_bender(K(True) & K(False), None, False) + self.assert_bender(K(False) & K(True), None, False) + self.assert_bender(K(False) & K(False), None, False) + + def test_or(self): + self.assert_bender(K(True) | K(True), None, True) + self.assert_bender(K(True) | K(False), None, True) + self.assert_bender(K(False) | K(True), None, True) + self.assert_bender(K(False) | K(False), None, False) + + def test_invert(self): + self.assert_bender(~K(True), None, False) + self.assert_bender(~K(False), None, True) + + +class TestGetItem(unittest.TestCase, BenderTestMixin): def test_getitem(self): bender = S('val')[2:8:2] - self.assertEqual(bender({'val': range(10)}), [2, 4, 6]) + if sys.version_info.major == 2: + val = range(10) + else: + val = list(range(10)) + self.assert_bender(bender, {'val': val}, [2, 4, 6]) if __name__ == '__main__': diff --git a/tests/test_list_ops.py b/tests/test_list_ops.py index 3f73f3a..2c667c4 100644 --- a/tests/test_list_ops.py +++ b/tests/test_list_ops.py @@ -1,15 +1,16 @@ +from operator import add import unittest -from jsonbender import K +from jsonbender import Context, K, S, bend from jsonbender.list_ops import Forall, FlatForall, Filter, ListOp, Reduce +from jsonbender.test import BenderTestMixin -class ListOpTestCase(unittest.TestCase): +class ListOpTestCase(unittest.TestCase, BenderTestMixin): cls = ListOp def assert_list_op(self, the_list, func, expected_value): - bender = self.cls(K(the_list), func) - self.assertEqual(bender({}), expected_value) + self.assert_bender(self.cls(func), the_list, expected_value) class TestForall(ListOpTestCase): @@ -18,19 +19,57 @@ class TestForall(ListOpTestCase): def test_empty_list(self): self.assert_list_op([], lambda i: i*2, []) + def test_none_list(self): + self.assert_list_op(None, lambda i: i*2, None) + def test_nonempty_list(self): self.assert_list_op(range(1, 5), lambda i: i*2, [2, 4, 6, 8]) + def test_compatibility(self): + # TODO: remove this when compatibility is broken + bender = self.cls(K([1]), lambda i: i) + self.assert_bender(bender, {}, [1]) + + def test_bend(self): + self.assert_bender(self.cls.bend({'b': S('a')}), + [{'a': 23}, {'a': 27}], + [{'b': 23}, {'b': 27}]) + + def test_bend_none(self): + self.assert_bender(self.cls.bend({'b': S('a')}), + None, + None) + + def test_bend_with_context(self): + mapping = {'b': Context() >> S('c')} + context = {'c': 42} + self.assert_bender(self.cls.bend(mapping, context), + [{}, {}], + [{'b': 42}, {'b': 42}]) + + def test_bend_inherits_outer_context_by_default(self): + inner_mapping = {'val': Context()} + outer_mapping = {'a': S('items') >> Forall.bend(inner_mapping)} + source = {'items': range(3)} + got = bend(outer_mapping, source, context=27) + expected = {'a': [{'val': 27}, {'val': 27}, {'val': 27}]} + self.assertEqual(got, expected) + class TestReduce(ListOpTestCase): cls = Reduce def test_empty_list(self): - bender = Reduce(K([]), lambda acc, i: acc + i) - self.assertRaises(ValueError, bender, {}) + bender = Reduce(add) + self.assertRaises(ValueError, bender, []) def test_nonempty_list(self): - self.assert_list_op(range(1, 5), lambda acc, i: acc + i, 10) + self.assert_list_op(range(1, 5), add, 10) + + def test_compatibility(self): + # TODO: remove this when compatibility is broken + bender = self.cls(K([1, 2]), add) + self.assert_bender(bender, {}, 3) class TestFilter(ListOpTestCase): @@ -48,6 +87,11 @@ def test_nonempty_list(self): expected = [{'id': 2, 'ignore': False}, {'id': 3, 'ignore': False}] self.assert_list_op(the_list, lambda d: not d['ignore'], expected) + def test_compatibility(self): + # TODO: remove this on next release + bender = self.cls(K([1]), lambda i: True) + self.assert_bender(bender, {}, [1]) + class TestFlatForall(ListOpTestCase): cls = FlatForall @@ -60,6 +104,11 @@ def test_nonempty_list(self): lambda d: d['b'], [1, 2, -2, -1]) + def test_compatibility(self): + # TODO: remove this on next release + bender = self.cls(K([1]), lambda i: [i]) + self.assert_bender(bender, {}, [1]) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_selectors.py b/tests/test_selectors.py index c42ae4f..f32c751 100644 --- a/tests/test_selectors.py +++ b/tests/test_selectors.py @@ -1,41 +1,93 @@ import unittest -from jsonbender import F, K, S +from jsonbender.selectors import F, ProtectedF, K, S, OptionalS +from jsonbender.test import BenderTestMixin -class TestK(unittest.TestCase): +class TestK(unittest.TestCase, BenderTestMixin): + selector_cls = K + def test_k(self): - self.assertEqual(K(1)({}), 1) - self.assertEqual(K('string')({}), 'string') + self.assert_bender(K(1), {}, 1) + self.assert_bender(K('string'), {}, 'string') -class TestS(unittest.TestCase): +class STestsMixin(BenderTestMixin): def test_no_selector_raises_value_error(self): - self.assertRaises(ValueError, S) + self.assertRaises(ValueError, self.selector_cls) - def test_single_existing_field(self): - self.assertEqual(S('a')({'a': 'val'}), 'val') + def test_shallow_existing_field(self): + source = {'a': 'val'} + self.assert_bender(self.selector_cls('a'), source, 'val') def test_deep_existing_path(self): source = {'a': [{}, {'b': 'ok!'}]} - self.assertEqual(S('a', 1, 'b')(source), 'ok!') + self.assert_bender(self.selector_cls('a', 1, 'b'), source, 'ok!') + + +class TestS(unittest.TestCase, STestsMixin): + selector_cls = S + + def test_shallow_missing_field(self): + self.assertRaises(KeyError, self.selector_cls('k'), {}) + + def test_deep_missing_field(self): + self.assertRaises(KeyError, self.selector_cls('k', 'k2'), {'k': {}}) + + +class TestOptionalS(unittest.TestCase, STestsMixin): + selector_cls = OptionalS + + def test_opts_without_default(self): + bender = OptionalS('key', 'missing') + self.assert_bender(bender, {'key': {}}, None) + self.assert_bender(bender, {}, None) + def test_opts_with_default(self): + default = 27 + bender = OptionalS('key', 'missing', default=default) + self.assert_bender(bender, {'key': {}}, default) + self.assert_bender(bender, {}, default) -class TestF(unittest.TestCase): + def test_activate_on_IndexError(self): + self.assert_bender(OptionalS(0), [], None) + + +class FTestsMixin(BenderTestMixin): def test_f(self): - self.assertEqual(F(len)(range(5)), 5) + self.assert_bender(self.selector_cls(len), range(5), 5) def test_curry_kwargs(self): - f = F(sorted, key=lambda d: d['v']) + f = self.selector_cls(sorted, key=lambda d: d['v']) source = [{'v': 2}, {'v': 3}, {'v': 1}] - self.assertEqual(f(source), [{'v': 1}, {'v': 2}, {'v': 3}]) + self.assert_bender(f, source, [{'v': 1}, {'v': 2}, {'v': 3}]) + def test_protect(self): + protected = self.selector_cls(int).protect(protect_against='bad') + self.assertIsInstance(protected, ProtectedF) + self.assert_bender(protected, '123', 123) + self.assert_bender(protected, 'bad', 'bad') + + # TODO: move this to a more general Bender test def test_composition(self): s = S('val') - f = F(len) + f = self.selector_cls(len) source = {'val': 'hello'} - self.assertEqual((f << s)(source), 5) - self.assertEqual((s >> f)(source), 5) + self.assert_bender((f << s), source, 5) + self.assert_bender((s >> f), source, 5) + + +class TestF(unittest.TestCase, FTestsMixin): + selector_cls = F + + +class TestProtectedF(unittest.TestCase, FTestsMixin): + selector_cls = ProtectedF + + def test_protectedf(self): + protected = ProtectedF(int) + self.assert_bender(protected, '123', 123) + self.assert_bender(protected, None, None) if __name__ == '__main__': diff --git a/tests/test_string_ops.py b/tests/test_string_ops.py index af3282f..3d1dd34 100644 --- a/tests/test_string_ops.py +++ b/tests/test_string_ops.py @@ -1,15 +1,21 @@ import unittest -from jsonbender import K +from jsonbender import Context, K, S from jsonbender.string_ops import Format +from jsonbender.test import BenderTestMixin -class TestFormat(unittest.TestCase): +class TestFormat(unittest.TestCase, BenderTestMixin): def test_format(self): bender = Format('{} {} {} {noun}.', K('This'), K('is'), K('a'), noun=K('test')) - self.assertEqual(bender({}), 'This is a test.') + self.assert_bender(bender, None, 'This is a test.') + + def test_with_context(self): + bender = Format('value: {}', Context() >> S('b')) + self.assert_bender(bender, None, 'value: 23', + context={'b': 23}) if __name__ == '__main__': diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..2df5825 --- /dev/null +++ b/tox.ini @@ -0,0 +1,9 @@ +[testenv] +deps= + unittest2 + discover +commands=unit2 discover tests -v + +[tox] +envlist = + py{27,37}-app