diff --git a/.gitignore b/.gitignore index 12994bc..d629a4f 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,8 @@ parser.out /jsonpath_rw/VERSION .idea + +venv +.eggs +coverage.xml +htmlcov diff --git a/README.rst b/README.rst index a6ef686..71fb7c3 100644 --- a/README.rst +++ b/README.rst @@ -38,6 +38,14 @@ Then: >>> [match.value for match in jsonpath_expr.find({'foo': [{'baz': 1}, {'baz': 2}]})] [1, 2] + # Update values + >>> jsonpath_expr.update({'foo': [{'baz': 1}, {'baz': 2}]}, 3) + {'foo': [{'baz': 3}, {'baz': 3}]} + + # Exclude values + >>> jsonpath_expr.exclude({'foo': [{'baz': 1}, {'baz': 2}]}) + {'foo': []} + # Matches remember where they came from >>> [str(match.full_path) for match in jsonpath_expr.find({'foo': [{'baz': 1}, {'baz': 2}]})] ['foo.[0].baz', 'foo.[1].baz'] diff --git a/jsonpath_rw/jsonpath.py b/jsonpath_rw/jsonpath.py index 146a960..1d10e69 100644 --- a/jsonpath_rw/jsonpath.py +++ b/jsonpath_rw/jsonpath.py @@ -46,6 +46,20 @@ def child(self, child): else: return Child(self, child) + def exclude(self, data): + """ + Returns `data` without the specified path + """ + raise NotImplementedError() + + def include(self, data): + """ + Returns `data` with the specified path + :param data: + :return: + """ + raise NotImplementedError() + def make_datum(self, value): if isinstance(value, DatumInContext): return value @@ -181,6 +195,12 @@ def find(self, data): def update(self, data, val): return val + def exclude(self, data): + return None + + def include(self, data): + return data + def __str__(self): return '$' @@ -201,6 +221,12 @@ def find(self, datum): def update(self, data, val): return val + def exclude(self, data): + return None + + def include(self, data): + return data + def __str__(self): return '`this`' @@ -236,6 +262,16 @@ def update(self, data, val): self.right.update(datum.value, val) return data + def exclude(self, data): + for datum in self.left.find(data): + self.right.exclude(datum.value) + return data + + def include(self, data): + for datum in self.left.find(data): + self.right.include(datum.value) + return data + def __eq__(self, other): return isinstance(other, Child) and self.left == other.left and self.right == other.right @@ -288,6 +324,12 @@ def update(self, data, val): datum.path.update(data, val) return data + def exclude(self, data): + for path in reversed([datum.path for datum in self.find(data)]): + path.exclude(data) + + return data + def __str__(self): return '%s where %s' % (self.left, self.right) @@ -340,36 +382,45 @@ def match_recursively(datum): for left_match in left_matches for submatch in match_recursively(left_match)] - def is_singular(): + def is_singular(self): return False - def update(self, data, val): + def _modify(self, data, val = None, exclude = False): # Get all left matches into a list left_matches = self.left.find(data) if not isinstance(left_matches, list): left_matches = [left_matches] - def update_recursively(data): + def modify_recursively(data): # Update only mutable values corresponding to JSON types if not (isinstance(data, list) or isinstance(data, dict)): return - self.right.update(data, val) + if exclude: + self.right.exclude(data) + else: + self.right.update(data, val) # Manually do the * or [*] to avoid coercion and recurse just the right-hand pattern if isinstance(data, list): - for i in range(0, len(data)): - update_recursively(data[i]) + for i in reversed(range(0, len(data))): + modify_recursively(data[i]) elif isinstance(data, dict): for field in data.keys(): - update_recursively(data[field]) + modify_recursively(data[field]) for submatch in left_matches: - update_recursively(submatch.value) + modify_recursively(submatch.value) return data + def update(self, data, val): + return self._modify(data, val, exclude = False) + + def exclude(self, data): + return self._modify(data, None, exclude = True) + def __str__(self): return '%s..%s' % (self.left, self.right) @@ -396,6 +447,16 @@ def is_singular(self): def find(self, data): return self.left.find(data) + self.right.find(data) + def update(self, data, val): + self.left.update(data, val) + self.right.update(data, val) + return data + + def exclude(self, data): + self.left.exclude(data) + self.right.exclude(data) + return data + class Intersect(JSONPath): """ JSONPath for bits that match *both* patterns. @@ -462,6 +523,29 @@ def update(self, data, val): data[field] = val return data + def exclude(self, data): + for field in self.reified_fields(DatumInContext.wrap(data)): + if data and field in data: + del data[field] + return data + + def include(self, data): + datum = DatumInContext.wrap(data) + + try: + all_fields = tuple(datum.value.keys()) + except AttributeError: + all_fields = () + + path_fields = self.reified_fields(datum) + remove_fields = set(all_fields) - set(path_fields) + + for field in remove_fields: + if field in data: + del data[field] + + return data + def __str__(self): return ','.join(map(str, self.fields)) @@ -497,6 +581,20 @@ def update(self, data, val): data[self.index] = val return data + def exclude(self, data): + if data is not None and len(data) > self.index: + del data[self.index] + return data + + def include(self, data): + if data is None: + return None + + if len(data) > self.index: + return [data[self.index]] + + return [] + def __eq__(self, other): return isinstance(other, Index) and self.index == other.index @@ -552,6 +650,24 @@ def update(self, data, val): datum.path.update(data, val) return data + def exclude(self, data): + for path in reversed([datum.path for datum in self.find(data)]): + path.exclude(data) + + return data + + def include(self, data): + + if not data: + return data + + ret = [] + for datum in self.find(data): + ret.append(datum.value) + + data = ret + return data + def __str__(self): if self.start == None and self.end == None and self.step == None: return '[*]' diff --git a/tests/test_jsonpath.py b/tests/test_jsonpath.py index 33c0ea6..e3014cb 100644 --- a/tests/test_jsonpath.py +++ b/tests/test_jsonpath.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals, print_function, absolute_import, division, generators, nested_scopes import unittest +import json from jsonpath_rw import jsonpath # For setting the global auto_id_field flag @@ -71,8 +72,6 @@ def test_DatumInContext_in_context(self): # assert AutoIdForDatum(DatumInContext(value=3, path=Fields('foo')), # id_field='id', # context=DatumInContext(value={'id': 'bizzle'}, path=This())).pseudopath == Fields('bizzle').child(Fields('foo')) - - class TestJsonPath(unittest.TestCase): """ @@ -81,7 +80,8 @@ class TestJsonPath(unittest.TestCase): @classmethod def setup_class(cls): - logging.basicConfig() + logging.basicConfig(format = '%(levelname)s:%(funcName)s:%(message)s', + level = logging.DEBUG) # # Check that the data value returned is good @@ -91,7 +91,7 @@ def check_cases(self, test_cases): # Also, we coerce iterables, etc, into the desired target type for string, data, target in test_cases: - print('parse("%s").find(%s) =?= %s' % (string, data, target)) + logging.debug('parse("%s").find(%s) =?= %s' % (string, data, target)) result = parse(string).find(data) if isinstance(target, list): assert [r.value for r in result] == target @@ -102,10 +102,12 @@ def check_cases(self, test_cases): def test_fields_value(self): jsonpath.auto_id_field = None - self.check_cases([ ('foo', {'foo': 'baz'}, ['baz']), - ('foo,baz', {'foo': 1, 'baz': 2}, [1, 2]), - ('@foo', {'@foo': 1}, [1]), - ('*', {'foo': 1, 'baz': 2}, set([1, 2])) ]) + self.check_cases([ + ('foo', {'foo': 'baz'}, ['baz']), + ('foo,baz', {'foo': 1, 'baz': 2}, [1, 2]), + ('@foo', {'@foo': 1}, [1]), + ('*', {'foo': 1, 'baz': 2}, set([1, 2])) + ]) jsonpath.auto_id_field = 'id' self.check_cases([ ('*', {'foo': 1, 'baz': 2}, set([1, 2, '`this`'])) ]) @@ -159,6 +161,11 @@ def test_descendants_value(self): ('foo..baz', {'foo': [{'baz': 1}, {'baz': 2}]}, [1, 2] ), ]) + def test_union_value(self): + self.check_cases([ + ('foo | bar', {'foo': 1, 'bar': 2}, [1, 2]) + ]) + def test_parent_value(self): self.check_cases([('foo.baz.`parent`', {'foo': {'baz': 3}}, [{'baz': 3}]), ('foo.`parent`.foo.baz.`parent`.baz.bizzle', {'foo': {'baz': {'bizzle': 5}}}, [5])]) @@ -182,7 +189,7 @@ def check_paths(self, test_cases): # Also, we coerce iterables, etc, into the desired target type for string, data, target in test_cases: - print('parse("%s").find(%s).paths =?= %s' % (string, data, target)) + logging.debug('parse("%s").find(%s).paths =?= %s' % (string, data, target)) result = parse(string).find(data) if isinstance(target, list): assert [str(r.full_path) for r in result] == target @@ -294,7 +301,7 @@ def test_descendants_auto_id(self): def check_update_cases(self, test_cases): for original, expr_str, value, expected in test_cases: - print('parse(%r).update(%r, %r) =?= %r' + logger.debug('parse(%r).update(%r, %r) =?= %r' % (expr_str, original, value, expected)) expr = parse(expr_str) actual = expr.update(original, value) @@ -307,7 +314,10 @@ def test_update_root(self): def test_update_this(self): self.check_update_cases([ - ('foo', '`this`', 'bar', 'bar') + ('foo', '`this`', 'bar', 'bar'), + # TODO: fixme + # ({'foo': 'bar'}, 'foo.`this`', 'baz', {'foo': 'baz'}), + ({'foo': {'bar': 'baz'}}, 'foo.`this`.bar', 'foo', {'foo': {'bar': 'foo'}}) ]) def test_update_fields(self): @@ -322,6 +332,11 @@ def test_update_child(self): ({'foo': {'bar': 1}}, 'foo.bar', 'baz', {'foo': {'bar': 'baz'}}) ]) + def test_update_union(self): + self.check_update_cases([ + ({'foo': 1, 'bar': 2}, 'foo | bar', 3, {'foo': 3, 'bar': 3}) + ]) + def test_update_where(self): self.check_update_cases([ ({'foo': {'bar': {'baz': 1}}, 'bar': {'baz': 2}}, @@ -353,3 +368,200 @@ def test_update_slice(self): self.check_update_cases([ (['foo', 'bar', 'baz'], '[0:2]', 'test', ['test', 'test', 'baz']) ]) + + def check_exclude_cases(self, test_cases): + for original, string, expected in test_cases: + logging.debug('parse("%s").exclude(%s) =?= %s' % (string, original, expected)) + actual = parse(string).exclude(original) + assert actual == expected + + def test_exclude_fields(self): + jsonpath.auto_id_field = None + self.check_exclude_cases([ + ({'foo': 'baz'}, 'foo', {}), + ({'foo': 1, 'baz': 2}, 'foo', {'baz': 2}), + ({'foo': 1, 'baz': 2}, 'foo,baz', {}), + ({'@foo': 1}, '@foo', {}), + ({'@foo': 1, 'baz': 2}, '@foo', {'baz': 2}), + ({'foo': 1, 'baz': 2}, '*', {}) + ]) + + def test_exclude_root(self): + self.check_exclude_cases([ + ('foo', '$', None), + ]) + + def test_exclude_this(self): + self.check_exclude_cases([ + ('foo', '`this`', None), + ({}, '`this`', None), + ({'foo': 1}, '`this`', None), + # TODO: fixme + #({'foo': 1}, 'foo.`this`', {}), + ({'foo': {'bar': 1}}, 'foo.`this`.bar', {'foo': {}}), + ({'foo': {'bar': 1, 'baz': 2}}, 'foo.`this`.bar', {'foo': {'baz': 2}}) + ]) + + def test_exclude_child(self): + self.check_exclude_cases([ + ({'foo': 'bar'}, '$.foo', {}), + ({'foo': 'bar'}, 'foo', {}), + ({'foo': {'bar': 1}}, 'foo.bar', {'foo': {}}), + ({'foo': {'bar': 1}}, 'foo.$.foo.bar', {'foo': {}}) + ]) + + def test_exclude_where(self): + self.check_exclude_cases([ + ({'foo': {'bar': {'baz': 1}}, 'bar': {'baz': 2}}, + '*.bar where none', {'foo': {'bar': {'baz': 1}}, 'bar': {'baz': 2}}), + + ({'foo': {'bar': {'baz': 1}}, 'bar': {'baz': 2}}, + '*.bar where baz', {'foo': {}, 'bar': {'baz': 2}}) + ]) + + def test_exclude_descendants(self): + self.check_exclude_cases([ + ({'somefield': 1}, '$..somefield', {}), + ({'outer': {'nestedfield': 1}}, '$..nestedfield', {'outer': {}}), + ({'outs': {'bar': 1, 'ins': {'bar': 9}}, 'outs2': {'bar': 2}}, + '$..bar', + {'outs': {'ins': {}}, 'outs2': {}}) + ]) + + def test_exclude_descendants_where(self): + self.check_exclude_cases([ + ({'foo': {'bar': 1, 'flag': 1}, 'baz': {'bar': 2}}, + '(* where flag) .. bar', + {'foo': {'flag': 1}, 'baz': {'bar': 2}}) + ]) + + def test_exclude_union(self): + self.check_exclude_cases([ + ({'foo': 1, 'bar': 2}, 'foo | bar', {}), + ({'foo': 1, 'bar': 2, 'baz': 3}, 'foo | bar', {'baz': 3}), + ]) + + def test_exclude_index(self): + self.check_exclude_cases([ + ([42], '[0]', []), + ([42], '[5]', [42]), + ([34, 65, 29, 59], '[2]', [34, 65, 59]), + (None, '[0]', None), + ([], '[0]', []), + (['foo', 'bar', 'baz'], '[0]', ['bar', 'baz']), + ]) + + def test_exclude_slice(self): + self.check_exclude_cases([ + (['foo', 'bar', 'baz'], '[0:2]', ['baz']), + (['foo', 'bar', 'baz'], '[0:1]', ['bar', 'baz']), + (['foo', 'bar', 'baz'], '[0:]', []), + (['foo', 'bar', 'baz'], '[:2]', ['baz']), + (['foo', 'bar', 'baz'], '[:3]', []) + ]) + + def check_include_cases(self, test_cases): + for original, string, expected in test_cases: + logging.debug('parse("%s").include(%s) =?= %s' % (string, original, expected)) + actual = parse(string).include(original) + assert actual == expected + + def test_include_fields(self): + self.check_include_cases([ + ({'foo': 'baz'}, 'foo', {'foo': 'baz'}), + ({'foo': 1, 'baz': 2}, 'foo', {'foo': 1}), + ({'foo': 1, 'baz': 2}, 'foo,baz', {'foo': 1, 'baz': 2}), + ({'@foo': 1}, '@foo', {'@foo': 1}), + ({'@foo': 1, 'baz': 2}, '@foo', {'@foo': 1}), + ({'foo': 1, 'baz': 2}, '*', {'foo': 1, 'baz': 2}), + ]) + + def test_include_index(self): + self.check_include_cases([ + ([42], '[0]', [42]), + ([42], '[5]', []), + ([34, 65, 29, 59], '[2]', [29]), + (None, '[0]', None), + ([], '[0]', []), + (['foo', 'bar', 'baz'], '[0]', ['foo']), + ]) + + def test_include_slice(self): + self.check_include_cases([ + (['foo', 'bar', 'baz'], '[0:2]', ['foo', 'bar']), + (['foo', 'bar', 'baz'], '[0:1]', ['foo']), + (['foo', 'bar', 'baz'], '[0:]', ['foo', 'bar', 'baz']), + (['foo', 'bar', 'baz'], '[:2]', ['foo', 'bar']), + (['foo', 'bar', 'baz'], '[:3]', ['foo', 'bar', 'baz']), + (['foo', 'bar', 'baz'], '[0:0]', []), + ]) + + def test_include_root(self): + self.check_include_cases([ + ('foo', '$', 'foo'), + ({}, '$', {}), + ({'foo': 1}, '$', {'foo': 1}) + ]) + + def test_include_this(self): + self.check_include_cases([ + ('foo', '`this`', 'foo'), + ({}, '`this`', {}), + ({'foo': 1}, '`this`', {'foo': 1}), + # TODO: fixme + #({'foo': 1}, 'foo.`this`', {}), + ({'foo': {'bar': 1}}, 'foo.`this`.bar', {'foo': {'bar': 1}}), + ({'foo': {'bar': 1, 'baz': 2}}, 'foo.`this`.bar', {'foo': {'bar': 1}}) + ]) + + def test_include_child(self): + self.check_include_cases([ + ({'foo': 'bar'}, '$.foo', {'foo': 'bar'}), + ({'foo': 'bar'}, 'foo', {'foo': 'bar'}), + ({'foo': {'bar': 1}}, 'foo.bar', {'foo': {'bar': 1}}), + ({'foo': {'bar': 1}}, 'foo.$.foo.bar', {'foo': {'bar': 1}}), + ({'foo': {'bar': 1, 'baz': 2}}, 'foo.$.foo.bar', {'foo': {'bar': 1}}), + ({'foo': {'bar': 1, 'baz': 2}}, '*', {'foo': {'bar': 1, 'baz': 2}}), + ({'foo': {'bar': 1, 'baz': 2}}, 'non', {}), + ]) + + def test_exclude_not_exists(self): + self.check_exclude_cases([ + ( + { + 'foo': [ + {'bar': 'bar'}, + {'baz': None} + ] + }, + 'foo.[*].baz.not_exist_key', + { + 'foo': [ + {'bar': 'bar'}, + {'baz': None} + ] + }, + ), + ]) + + """ + def test_include_where(self): + self.check_include_cases([ + #({'foo': {'bar': {'baz': 1}}, 'bar': {'baz': 2}}, + # '*.bar where none', {}), + + ({'foo': {'bar': {'baz': 1}}, 'bar': {'baz': 2}}, + '*.bar where baz', {'foo': {'bar': {'baz': 1}}}) + ]) + """ + + """ + def test_include_descendants(self): + self.check_include_cases([ + ({'somefield': 1}, '$..somefield', {'somefield': 1}), + ({'outer': {'nestedfield': 1}}, '$..nestedfield', {'outer': {'nestedfield': 1}}), + ({'outs': {'bar': 1, 'ins': {'bar': 9}}, 'outs2': {'bar': 2}}, + '$..bar', + {'outs': {'bar': 1, 'ins': {'bar': 9}}, 'outs2': {'bar': 2}}) + ]) + """ diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 9d9fe38..e252add 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -23,13 +23,14 @@ def assert_lex_equiv(self, s, stream2): stream2 = list(stream2) assert len(stream1) == len(stream2) for token1, token2 in zip(stream1, stream2): - print(token1, token2) + logging.debug(token1, token2) assert token1.type == token2.type assert token1.value == token2.value @classmethod def setup_class(cls): - logging.basicConfig() + logging.basicConfig(format = '%(levelname)s:%(funcName)s:%(message)s', + level = logging.DEBUG) def test_simple_inputs(self): self.assert_lex_equiv('$', [self.token('$', '$')]) @@ -51,6 +52,9 @@ def test_simple_inputs(self): self.assert_lex_equiv('&', [self.token('&', '&')]) self.assert_lex_equiv('@', [self.token('@', 'ID')]) self.assert_lex_equiv('`this`', [self.token('this', 'NAMED_OPERATOR')]) + self.assert_lex_equiv('fuzz.`this`', [self.token('fuzz', 'ID'), + self.token('.', '.'), + self.token('this', 'NAMED_OPERATOR')]) self.assert_lex_equiv('|', [self.token('|', '|')]) self.assert_lex_equiv('where', [self.token('where', 'WHERE')]) diff --git a/tests/test_parser.py b/tests/test_parser.py index fd1e121..b3c8ed4 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -10,13 +10,14 @@ class TestParser(unittest.TestCase): @classmethod def setup_class(cls): - logging.basicConfig() + logging.basicConfig(format = '%(levelname)s:%(funcName)s:%(message)s', + level = logging.DEBUG) def check_parse_cases(self, test_cases): parser = JsonPathParser(debug=True, lexer_class=lambda:JsonPathLexer(debug=False)) # Note that just manually passing token streams avoids this dep, but that sucks for string, parsed in test_cases: - print(string, '=?=', parsed) # pytest captures this and we see it only on a failure, for debugging + logging.debug(string, '=?=', parsed) # pytest captures this and we see it only on a failure, for debugging assert parser.parse(string) == parsed def test_atomic(self): @@ -36,5 +37,7 @@ def test_nested(self): self.check_parse_cases([('foo.baz', Child(Fields('foo'), Fields('baz'))), ('foo.baz,bizzle', Child(Fields('foo'), Fields('baz', 'bizzle'))), ('foo where baz', Where(Fields('foo'), Fields('baz'))), + ('`this`', This()), + ('foo.`this`', Child(Fields('foo'), This())), ('foo..baz', Descendants(Fields('foo'), Fields('baz'))), ('foo..baz.bing', Descendants(Fields('foo'), Child(Fields('baz'), Fields('bing'))))])