diff --git a/.travis.yml b/.travis.yml index 94f1e7b..70f48e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,14 @@ python: - "2.7" - "3.2" - "3.3" - - "3.4" + - "3.4" + - "3.5" - "pypy" - "pypy3" install: - travis_retry pip install -r requirements.txt + - if [ "$TRAVIS_PYTHON_VERSION" == "3.2" ]; then travis_retry pip install 'coverage<4'; fi - travis_retry pip install coveralls script: @@ -17,3 +19,5 @@ script: after_script: - coveralls + +sudo: false diff --git a/ext_tests.py b/ext_tests.py index 5e5ded8..0e1404c 100755 --- a/ext_tests.py +++ b/ext_tests.py @@ -60,12 +60,15 @@ def _test(self, test): ) else: - res = jsonpatch.apply_patch(test['doc'], test['patch']) + try: + res = jsonpatch.apply_patch(test['doc'], test['patch']) + except jsonpatch.JsonPatchException as jpe: + raise Exception(test.get('comment', '')) from jpe # if there is no 'expected' we only verify that applying the patch # does not raies an exception if 'expected' in test: - self.assertEquals(res, test['expected']) + self.assertEquals(res, test['expected'], test.get('comment', '')) def make_test_case(tests): diff --git a/jsondiff.py b/jsondiff.py new file mode 100644 index 0000000..cb19639 --- /dev/null +++ b/jsondiff.py @@ -0,0 +1,303 @@ +''' +The MIT License (MIT) + +Copyright (c) 2014 Ilya Volkov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +''' + +import sys + +__all__ = ["make",] + +if sys.version_info[0] >= 3: + _range = range + _str = str + _viewkeys = dict.keys +else: + _range = xrange + _str = unicode + if sys.version_info[1] >= 7: + _viewkeys = dict.viewkeys + else: + _viewkeys = lambda x: set(dict.keys(x)) + +_ST_ADD = 0 +_ST_REMOVE = 1 + +class _compare_info(object): + + def __init__(self): + self.index_storage = [{}, {}] + self.index_storage2 = [[], []] + self.__root = root = [] + root[:] = [root, root, None] + + def store_index(self, value, index, st): + try: + storage = self.index_storage[st] + stored = storage.get(value) + if stored == None: + storage[value] = [index] + else: + storage[value].append(index) + except TypeError: + self.index_storage2[st].append((value, index)) + + def take_index(self, value, st): + try: + stored = self.index_storage[st].get(value) + if stored: + return stored.pop() + except TypeError: + storage = self.index_storage2[st] + for i in range(len(storage)-1, -1, -1): + if storage[i][0] == value: + return storage.pop(i)[1] + + def insert(self, op): + root = self.__root + last = root[0] + last[1] = root[0] = [last, root, op] + return root[0] + + def remove(self, index): + link_prev, link_next, _ = index + link_prev[1] = link_next + link_next[0] = link_prev + index[:] = [] + + def iter_from(self, start): + root = self.__root + curr = start[1] + while curr is not root: + yield curr[2] + curr = curr[1] + + def __iter__(self): + root = self.__root + curr = root[1] + while curr is not root: + yield curr[2] + curr = curr[1] + + def execute(self): + root = self.__root + curr = root[1] + while curr is not root: + if curr[1] is not root: + op_first, op_second = curr[2], curr[1][2] + if op_first.key == op_second.key and \ + op_first.path == op_second.path and \ + type(op_first) == _op_remove and \ + type(op_second) == _op_add: + info = _compare_info() + _compare_values(op_second.path, op_second.key, info, op_first.value, op_second.value) + for i in info.execute(): + yield i + curr = curr[1][1] + continue + if type(curr[2]) != _op_move or curr[2].oldpath != curr[2].path or curr[2].oldkey != curr[2].key: + yield curr[2].get() + curr = curr[1] + +class _op_base(object): + def __init__(self, path, key, value): + self.path = path + self.key = key + self.value = value + + def __repr__(self): + return _str(self.get()) + +class _op_add(_op_base): + def _on_undo_remove(self, path, key): + if self.path == path: + if self.key > key: + self.key += 1 + else: + key += 1 + return key + + def _on_undo_add(self, path, key): + if self.path == path: + if self.key > key: + self.key -= 1 + else: + key += 1 + return key + + def get(self): + return {'op': 'add', 'path': _path_join(self.path, self.key), 'value': self.value} + +class _op_remove(_op_base): + def _on_undo_remove(self, path, key): + if self.path == path: + if self.key >= key: + self.key += 1 + else: + key -= 1 + return key + + def _on_undo_add(self, path, key): + if self.path == path: + if self.key > key: + self.key -= 1 + else: + key -= 1 + return key + + def get(self): + return {'op': 'remove', 'path': _path_join(self.path, self.key)} + +class _op_replace(_op_base): + def _on_undo_remove(self, path, key): + return key + + def _on_undo_add(self, path, key): + return key + + def get(self): + return {'op': 'replace', 'path': _path_join(self.path, self.key), 'value': self.value} + +class _op_move(object): + def __init__(self, oldpath, oldkey, path, key): + self.oldpath = oldpath + self.oldkey = oldkey + self.path = path + self.key = key + + def _on_undo_remove(self, path, key): + if self.oldpath == path: + if self.oldkey >= key: + self.oldkey += 1 + else: + key -= 1 + if self.path == path: + if self.key > key: + self.key += 1 + else: + key += 1 + return key + + def _on_undo_add(self, path, key): + if self.oldpath == path: + if self.oldkey > key: + self.oldkey -= 1 + else: + key -= 1 + if self.path == path: + if self.key > key: + self.key -= 1 + else: + key += 1 + return key + + def get(self): + return {'op': 'move', 'path': _path_join(self.path, self.key), 'from': _path_join(self.oldpath, self.oldkey)} + + def __repr__(self): + return _str(self.get()) + +def _path_join(path, key): + if key != None: + return path + '/' + _str(key).replace('~', '~0').replace('/', '~1') + return path + +def _item_added(path, key, info, item): + index = info.take_index(item, _ST_REMOVE) + if index != None: + op = index[2] + if type(op.key) == int: + for v in info.iter_from(index): + op.key = v._on_undo_remove(op.path, op.key) + info.remove(index) + if op.path != path or op.key != key: + new_op = _op_move(op.path, op.key, path, key) + info.insert(new_op) + else: + new_op = _op_add(path, key, item) + new_index = info.insert(new_op) + info.store_index(item, new_index, _ST_ADD) + +def _item_removed(path, key, info, item): + new_op = _op_remove(path, key, item) + index = info.take_index(item, _ST_ADD) + new_index = info.insert(new_op) + if index != None: + op = index[2] + if type(op.key) == int: + for v in info.iter_from(index): + op.key = v._on_undo_add(op.path, op.key) + info.remove(index) + if new_op.path != op.path or new_op.key != op.key: + new_op = _op_move(new_op.path, new_op.key, op.path, op.key) + new_index[2] = new_op + else: + info.remove(new_index) + else: + info.store_index(item, new_index, _ST_REMOVE) + +def _item_replaced(path, key, info, item): + info.insert(_op_replace(path, key, item)) + +def _compare_dicts(path, info, src, dst): + src_keys = _viewkeys(src) + dst_keys = _viewkeys(dst) + added_keys = dst_keys - src_keys + removed_keys = src_keys - dst_keys + for key in removed_keys: + _item_removed(path, _str(key), info, src[key]) + for key in added_keys: + _item_added(path, _str(key), info, dst[key]) + for key in src_keys & dst_keys: + _compare_values(path, key, info, src[key], dst[key]) + +def _compare_lists(path, info, src, dst): + len_src, len_dst = len(src), len(dst) + max_len = max(len_src, len_dst) + min_len = min(len_src, len_dst) + for key in _range(max_len): + if key < min_len: + old, new = src[key], dst[key] + if old == new: + continue + _item_removed(path, key, info, old) + _item_added(path, key, info, new) + elif len_src > len_dst: + _item_removed(path, len_dst, info, src[key]) + else: + _item_added(path, key, info, dst[key]) + +def _compare_values(path, key, info, src, dst): + if src == dst: + return + elif isinstance(src, dict) and \ + isinstance(dst, dict): + _compare_dicts(_path_join(path, key), info, src, dst) + elif isinstance(src, list) and \ + isinstance(dst, list): + _compare_lists(_path_join(path, key), info, src, dst) + else: + _item_replaced(path, key, info, dst) + +def make(src, dst, **kwargs): + info = _compare_info() + _compare_values('', None, info, src, dst) + return [op for op in info.execute()] diff --git a/jsonpatch.py b/jsonpatch.py index 6318821..80b4aa6 100644 --- a/jsonpatch.py +++ b/jsonpatch.py @@ -37,11 +37,12 @@ import collections import copy import functools -import inspect import itertools import json import sys +import jsondiff + try: from collections.abc import MutableMapping, MutableSequence except ImportError: @@ -51,7 +52,7 @@ # Will be parsed by setup.py to determine package metadata __author__ = 'Stefan Kögl ' -__version__ = '1.9' +__version__ = '1.14+jsondiff.unicode.replacefix.0' __website__ = 'https://github.com/stefankoegl/python-json-patch' __license__ = 'Modified BSD License' @@ -97,20 +98,11 @@ def multidict(ordered_pairs): def get_loadjson(): - """ adds the object_pairs_hook parameter to json.load when possible - - The "object_pairs_hook" parameter is used to handle duplicate keys when - loading a JSON object. This parameter does not exist in Python 2.6. This - methods returns an unmodified json.load for Python 2.6 and a partial - function with object_pairs_hook set to multidict for Python versions that - support the parameter. """ - - argspec = inspect.getargspec(json.load) - if 'object_pairs_hook' not in argspec.args: - return json.load + """ adds the object_pairs_hook parameter to json.load """ return functools.partial(json.load, object_pairs_hook=multidict) + json.load = get_loadjson() @@ -168,7 +160,7 @@ def make_patch(src, dst): >>> new == dst True """ - return JsonPatch.from_diff(src, dst) + return JsonPatch(jsondiff.make(src, dst)) class JsonPatch(object): @@ -484,6 +476,10 @@ def apply(self, obj): except (KeyError, IndexError) as ex: raise JsonPatchConflict(str(ex)) + # If source and target are equal, this is a no-op + if self.pointer == from_ptr: + return obj + if isinstance(subobj, MutableMapping) and \ self.pointer.contains(from_ptr): raise JsonPatchConflict('Cannot move values into its own children') @@ -641,7 +637,7 @@ def _split_by_common_seq(src, dst, bx=(0, -1), by=(0, -1)): (by[0], by[0] + y[0])), _split_by_common_seq(src[x[1]:], dst[y[1]:], (bx[0] + x[1], bx[0] + len(src)), - (bx[0] + y[1], bx[0] + len(dst)))] + (by[0] + y[1], by[0] + len(dst)))] def _compare(path, src, dst, left, right): @@ -690,7 +686,8 @@ def _compare_left(path, src, left, shift): # yes, there should be any value field, but we'll use it # to apply `move` optimization a bit later and will remove # it in _optimize function. - 'value': src[idx - shift], + #'value': src[idx - shift], + 'value': src[idx - shift] if shift>0 else src[idx], 'path': ptr.path, }, shift - 1 @@ -726,12 +723,15 @@ def _optimize(operations): result = [] ops_by_path = {} ops_by_value = {} + ops_to_replace = {} add_remove = set(['add', 'remove']) for item in operations: # could we apply "move" optimization for dict values? hashable_value = not isinstance(item['value'], (MutableMapping, MutableSequence)) if item['path'] in ops_by_path: + if item['op'] == 'add': + ops_to_replace[item['path']] = (ops_to_replace[item['path']][0], item['value']) _optimize_using_replace(ops_by_path[item['path']], item) continue if hashable_value and item['value'] in ops_by_value: @@ -745,6 +745,17 @@ def _optimize(operations): ops_by_path[item['path']] = item if hashable_value: ops_by_value[item['value']] = item + if item['op'] == 'remove': + ops_to_replace[item['path']] = (item['value'], None) + + for i in result: + if i['path'] in ops_to_replace and ops_to_replace[i['path']][1]: + patch = make_patch(*ops_to_replace[i['path']]).patch + for x in patch: + x['path'] = i['path'] + x['path'] + x['value'] = x.get('value', '') + index = result.index(i) + result[index:index+1] = patch # cleanup ops_by_path.clear() @@ -756,11 +767,18 @@ def _optimize(operations): def _optimize_using_replace(prev, cur): - """Optimises JSON patch by using ``replace`` operation instead of - ``remove`` and ``add`` against the same path.""" + """Optimises by replacing ``add``/``remove`` with ``replace`` on same path + + For nested strucures, tries to recurse replacement, see #36 """ prev['op'] = 'replace' if cur['op'] == 'add': - prev['value'] = cur['value'] + # make recursive patch + patch = make_patch(prev['value'], cur['value']) + if len(patch.patch) == 1: + prev['path'] = prev['path'] + patch.patch[0]['path'] + prev['value'] = patch.patch[0]['value'] + else: + prev['value'] = cur['value'] def _optimize_using_move(prev_item, item): diff --git a/requirements-dev.txt b/requirements-dev.txt index fd0fd6c..21daf9a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ wheel -pandoc==1.0.0-alpha.3 +pypandoc diff --git a/requirements.txt b/requirements.txt index 7b26233..cd4b892 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -jsonpointer>=1.5 +jsonpointer>=1.9 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..2a9acf1 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/setup.py b/setup.py index d31b990..363c0bf 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- import sys import io @@ -19,6 +20,7 @@ MODULES = ( 'jsonpatch', + 'jsondiff', ) REQUIREMENTS = list(open('requirements.txt')) diff --git a/tests.py b/tests.py index 5b0d9e9..f3f6460 100755 --- a/tests.py +++ b/tests.py @@ -299,15 +299,52 @@ def test_add_nested(self): } self.assertEqual(expected, res) + def test_use_replace_instead_of_remove_add_nested(self): + src = {'foo': [{'bar': 1, 'baz': 2}, {'bar': 2, 'baz': 3}]} + dst = {'foo': [{'bar': 1}, {'bar': 2, 'baz': 3}]} + patch = list(jsonpatch.make_patch(src, dst)) + self.assertEqual(len(patch), 1) + self.assertEqual(patch[0]['op'], 'remove') + res = jsonpatch.apply_patch(src, patch) + self.assertEqual(res, dst) + def test_should_just_add_new_item_not_rebuild_all_list(self): src = {'foo': [1, 2, 3]} - dst = {'foo': [3, 1, 2, 3]} + dst = {'foo': [4, 1, 2, 3]} patch = list(jsonpatch.make_patch(src, dst)) self.assertEqual(len(patch), 1) self.assertEqual(patch[0]['op'], 'add') res = jsonpatch.apply_patch(src, patch) self.assertEqual(res, dst) + def test_should_add_instead_of_move_add(self): + src = [1, 2] + dst = [2, 1, 2] + patch = list(jsonpatch.make_patch(src, dst)) + self.assertEqual(len(patch), 1) + self.assertEqual(patch[0]['op'], 'add') + res = jsonpatch.apply_patch(src, patch) + self.assertEqual(res, dst) + + def test_should_add_instead_of_move_add2(self): + src = [2, 4, 5] + dst = [1, 2, 3, 4, 5] + patch = list(jsonpatch.make_patch(src, dst)) + self.assertEqual(len(patch), 2) + self.assertEqual(patch[0]['op'], 'add') + res = jsonpatch.apply_patch(src, patch) + self.assertEqual(res, dst) + + def test_should_remove_instead_of_remove_move(self): + src = [1, 2, 3, 4, 5] + dst = [2, 4, 5] + patch = list(jsonpatch.make_patch(src, dst)) + self.assertEqual(len(patch), 2) + self.assertEqual(patch[0]['op'], 'remove') + self.assertEqual(patch[1]['op'], 'remove') + res = jsonpatch.apply_patch(src, patch) + self.assertEqual(res, dst) + def test_use_replace_instead_of_remove_add(self): src = {'foo': [1, 2, 3]} dst = {'foo': [3, 2, 3]} @@ -327,13 +364,18 @@ def test_use_move_instead_of_remove_add(self): self.assertEqual(res, dst) def test_use_move_instead_of_add_remove(self): - src = {'foo': [1, 2, 3]} - dst = {'foo': [3, 1, 2]} - patch = list(jsonpatch.make_patch(src, dst)) - self.assertEqual(len(patch), 1) - self.assertEqual(patch[0]['op'], 'move') - res = jsonpatch.apply_patch(src, patch) - self.assertEqual(res, dst) + def fn(_src, _dst): + patch = list(jsonpatch.make_patch(_src, _dst)) + # Check if there are only 'move' operations + for p in patch: + self.assertEqual(p['op'], 'move') + res = jsonpatch.apply_patch(_src, patch) + self.assertEqual(res, _dst) + + fn({'foo': [1, 2, 3]}, {'foo': [3, 1, 2]}) + fn({'foo': [1, 2, 3]}, {'foo': [3, 2, 1]}) + fn([1, 2, 3], [3, 1, 2]) + fn([1, 2, 3], [3, 2, 1]) def test_escape(self): src = {"x/y": 1} @@ -351,6 +393,72 @@ def test_root_list(self): res = patch.apply(src) self.assertEqual(res, dst) + def test_make_patch_unicode(self): + """ Test if unicode keys and values are handled correctly """ + src = {} + dst = {'\xee': '\xee'} + patch = jsonpatch.make_patch(src, dst) + res = patch.apply(src) + self.assertEqual(res, dst) + + def test_issue40(self): + """ Tests an issue in _split_by_common_seq reported in #40 """ + + src = [8, 7, 2, 1, 0, 9, 4, 3, 5, 6] + dest = [7, 2, 1, 0, 9, 4, 3, 6, 5, 8] + patch = jsonpatch.make_patch(src, dest) + + def test_minimal_patch(self): + """ Test whether a minimal patch is created, see #36 """ + src = [{"foo": 1, "bar": 2}] + dst = [{"foo": 2, "bar": 2}] + patch = jsonpatch.make_patch(src, dst) + + exp = [ + { + "path": "/0/foo", + "value": 2, + "op": "replace" + } + ] + + self.assertEqual(patch.patch, exp) + + +class ListTests(unittest.TestCase): + + def test_fail_prone_list_1(self): + """ Test making and applying a patch of the root is a list """ + src = ['a', 'r', 'b'] + dst = ['b', 'o'] + patch = jsonpatch.make_patch(src, dst) + res = patch.apply(src) + self.assertEqual(res, dst) + + def test_fail_prone_list_2(self): + """ Test making and applying a patch of the root is a list """ + src = ['a', 'r', 'b', 'x', 'm', 'n'] + dst = ['b', 'o', 'm', 'n'] + patch = jsonpatch.make_patch(src, dst) + res = patch.apply(src) + self.assertEqual(res, dst) + + def test_fail_prone_list_3(self): + """ Test making and applying a patch of the root is a list """ + src = ['boo1', 'bar', 'foo1', 'qux'] + dst = ['qux', 'bar'] + patch = jsonpatch.make_patch(src, dst) + res = patch.apply(src) + self.assertEqual(res, dst) + + def test_fail_prone_list_4(self): + """ Test making and applying a patch of the root is a list """ + src = ['bar1', 59, 'foo1', 'foo'] + dst = ['foo', 'bar', 'foo1'] + patch = jsonpatch.make_patch(src, dst) + res = patch.apply(src) + self.assertEqual(res, dst) + class InvalidInputTests(unittest.TestCase): @@ -406,30 +514,31 @@ def test_replace_missing(self): self.assertRaises(jsonpatch.JsonPatchConflict, jsonpatch.apply_patch, src, patch_obj) - -modules = ['jsonpatch'] +if __name__ == '__main__': + modules = ['jsonpatch'] -def get_suite(): - suite = unittest.TestSuite() - suite.addTest(doctest.DocTestSuite(jsonpatch)) - suite.addTest(unittest.makeSuite(ApplyPatchTestCase)) - suite.addTest(unittest.makeSuite(EqualityTestCase)) - suite.addTest(unittest.makeSuite(MakePatchTestCase)) - suite.addTest(unittest.makeSuite(InvalidInputTests)) - suite.addTest(unittest.makeSuite(ConflictTests)) - return suite + def get_suite(): + suite = unittest.TestSuite() + suite.addTest(doctest.DocTestSuite(jsonpatch)) + suite.addTest(unittest.makeSuite(ApplyPatchTestCase)) + suite.addTest(unittest.makeSuite(EqualityTestCase)) + suite.addTest(unittest.makeSuite(MakePatchTestCase)) + suite.addTest(unittest.makeSuite(ListTests)) + suite.addTest(unittest.makeSuite(InvalidInputTests)) + suite.addTest(unittest.makeSuite(ConflictTests)) + return suite -suite = get_suite() + suite = get_suite() -for module in modules: - m = __import__(module, fromlist=[module]) - suite.addTest(doctest.DocTestSuite(m)) + for module in modules: + m = __import__(module, fromlist=[module]) + suite.addTest(doctest.DocTestSuite(m)) -runner = unittest.TextTestRunner(verbosity=1) + runner = unittest.TextTestRunner(verbosity=1) -result = runner.run(suite) + result = runner.run(suite) -if not result.wasSuccessful(): - sys.exit(1) + if not result.wasSuccessful(): + sys.exit(1)