jsonpatch.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908
  1. # -*- coding: utf-8 -*-
  2. #
  3. # python-json-patch - An implementation of the JSON Patch format
  4. # https://github.com/stefankoegl/python-json-patch
  5. #
  6. # Copyright (c) 2011 Stefan Kögl <stefan@skoegl.net>
  7. # All rights reserved.
  8. #
  9. # Redistribution and use in source and binary forms, with or without
  10. # modification, are permitted provided that the following conditions
  11. # are met:
  12. #
  13. # 1. Redistributions of source code must retain the above copyright
  14. # notice, this list of conditions and the following disclaimer.
  15. # 2. Redistributions in binary form must reproduce the above copyright
  16. # notice, this list of conditions and the following disclaimer in the
  17. # documentation and/or other materials provided with the distribution.
  18. # 3. The name of the author may not be used to endorse or promote products
  19. # derived from this software without specific prior written permission.
  20. #
  21. # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  22. # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  23. # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  24. # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  25. # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  26. # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  30. # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31. #
  32. """ Apply JSON-Patches (RFC 6902) """
  33. from __future__ import unicode_literals
  34. import collections
  35. import copy
  36. import functools
  37. import json
  38. import sys
  39. try:
  40. from types import MappingProxyType
  41. except ImportError:
  42. # Python < 3.3
  43. MappingProxyType = dict
  44. from jsonpointer import JsonPointer, JsonPointerException
  45. _ST_ADD = 0
  46. _ST_REMOVE = 1
  47. try:
  48. from collections.abc import MutableMapping, MutableSequence
  49. except ImportError:
  50. from collections import MutableMapping, MutableSequence
  51. str = unicode
  52. # Will be parsed by setup.py to determine package metadata
  53. __author__ = 'Stefan Kögl <stefan@skoegl.net>'
  54. __version__ = '1.32'
  55. __website__ = 'https://github.com/stefankoegl/python-json-patch'
  56. __license__ = 'Modified BSD License'
  57. # pylint: disable=E0611,W0404
  58. if sys.version_info >= (3, 0):
  59. basestring = (bytes, str) # pylint: disable=C0103,W0622
  60. class JsonPatchException(Exception):
  61. """Base Json Patch exception"""
  62. class InvalidJsonPatch(JsonPatchException):
  63. """ Raised if an invalid JSON Patch is created """
  64. class JsonPatchConflict(JsonPatchException):
  65. """Raised if patch could not be applied due to conflict situation such as:
  66. - attempt to add object key when it already exists;
  67. - attempt to operate with nonexistence object key;
  68. - attempt to insert value to array at position beyond its size;
  69. - etc.
  70. """
  71. class JsonPatchTestFailed(JsonPatchException, AssertionError):
  72. """ A Test operation failed """
  73. def multidict(ordered_pairs):
  74. """Convert duplicate keys values to lists."""
  75. # read all values into lists
  76. mdict = collections.defaultdict(list)
  77. for key, value in ordered_pairs:
  78. mdict[key].append(value)
  79. return dict(
  80. # unpack lists that have only 1 item
  81. (key, values[0] if len(values) == 1 else values)
  82. for key, values in mdict.items()
  83. )
  84. # The "object_pairs_hook" parameter is used to handle duplicate keys when
  85. # loading a JSON object.
  86. _jsonloads = functools.partial(json.loads, object_pairs_hook=multidict)
  87. def apply_patch(doc, patch, in_place=False, pointer_cls=JsonPointer):
  88. """Apply list of patches to specified json document.
  89. :param doc: Document object.
  90. :type doc: dict
  91. :param patch: JSON patch as list of dicts or raw JSON-encoded string.
  92. :type patch: list or str
  93. :param in_place: While :const:`True` patch will modify target document.
  94. By default patch will be applied to document copy.
  95. :type in_place: bool
  96. :param pointer_cls: JSON pointer class to use.
  97. :type pointer_cls: Type[JsonPointer]
  98. :return: Patched document object.
  99. :rtype: dict
  100. >>> doc = {'foo': 'bar'}
  101. >>> patch = [{'op': 'add', 'path': '/baz', 'value': 'qux'}]
  102. >>> other = apply_patch(doc, patch)
  103. >>> doc is not other
  104. True
  105. >>> other == {'foo': 'bar', 'baz': 'qux'}
  106. True
  107. >>> patch = [{'op': 'add', 'path': '/baz', 'value': 'qux'}]
  108. >>> apply_patch(doc, patch, in_place=True) == {'foo': 'bar', 'baz': 'qux'}
  109. True
  110. >>> doc == other
  111. True
  112. """
  113. if isinstance(patch, basestring):
  114. patch = JsonPatch.from_string(patch, pointer_cls=pointer_cls)
  115. else:
  116. patch = JsonPatch(patch, pointer_cls=pointer_cls)
  117. return patch.apply(doc, in_place)
  118. def make_patch(src, dst, pointer_cls=JsonPointer):
  119. """Generates patch by comparing two document objects. Actually is
  120. a proxy to :meth:`JsonPatch.from_diff` method.
  121. :param src: Data source document object.
  122. :type src: dict
  123. :param dst: Data source document object.
  124. :type dst: dict
  125. :param pointer_cls: JSON pointer class to use.
  126. :type pointer_cls: Type[JsonPointer]
  127. >>> src = {'foo': 'bar', 'numbers': [1, 3, 4, 8]}
  128. >>> dst = {'baz': 'qux', 'numbers': [1, 4, 7]}
  129. >>> patch = make_patch(src, dst)
  130. >>> new = patch.apply(src)
  131. >>> new == dst
  132. True
  133. """
  134. return JsonPatch.from_diff(src, dst, pointer_cls=pointer_cls)
  135. class PatchOperation(object):
  136. """A single operation inside a JSON Patch."""
  137. def __init__(self, operation, pointer_cls=JsonPointer):
  138. self.pointer_cls = pointer_cls
  139. if not operation.__contains__('path'):
  140. raise InvalidJsonPatch("Operation must have a 'path' member")
  141. if isinstance(operation['path'], self.pointer_cls):
  142. self.location = operation['path'].path
  143. self.pointer = operation['path']
  144. else:
  145. self.location = operation['path']
  146. try:
  147. self.pointer = self.pointer_cls(self.location)
  148. except TypeError as ex:
  149. raise InvalidJsonPatch("Invalid 'path'")
  150. self.operation = operation
  151. def apply(self, obj):
  152. """Abstract method that applies a patch operation to the specified object."""
  153. raise NotImplementedError('should implement the patch operation.')
  154. def __hash__(self):
  155. return hash(frozenset(self.operation.items()))
  156. def __eq__(self, other):
  157. if not isinstance(other, PatchOperation):
  158. return False
  159. return self.operation == other.operation
  160. def __ne__(self, other):
  161. return not(self == other)
  162. @property
  163. def path(self):
  164. return '/'.join(self.pointer.parts[:-1])
  165. @property
  166. def key(self):
  167. try:
  168. return int(self.pointer.parts[-1])
  169. except ValueError:
  170. return self.pointer.parts[-1]
  171. @key.setter
  172. def key(self, value):
  173. self.pointer.parts[-1] = str(value)
  174. self.location = self.pointer.path
  175. self.operation['path'] = self.location
  176. class RemoveOperation(PatchOperation):
  177. """Removes an object property or an array element."""
  178. def apply(self, obj):
  179. subobj, part = self.pointer.to_last(obj)
  180. try:
  181. del subobj[part]
  182. except (KeyError, IndexError) as ex:
  183. msg = "can't remove a non-existent object '{0}'".format(part)
  184. raise JsonPatchConflict(msg)
  185. return obj
  186. def _on_undo_remove(self, path, key):
  187. if self.path == path:
  188. if self.key >= key:
  189. self.key += 1
  190. else:
  191. key -= 1
  192. return key
  193. def _on_undo_add(self, path, key):
  194. if self.path == path:
  195. if self.key > key:
  196. self.key -= 1
  197. else:
  198. key -= 1
  199. return key
  200. class AddOperation(PatchOperation):
  201. """Adds an object property or an array element."""
  202. def apply(self, obj):
  203. try:
  204. value = self.operation["value"]
  205. except KeyError as ex:
  206. raise InvalidJsonPatch(
  207. "The operation does not contain a 'value' member")
  208. subobj, part = self.pointer.to_last(obj)
  209. if isinstance(subobj, MutableSequence):
  210. if part == '-':
  211. subobj.append(value) # pylint: disable=E1103
  212. elif part > len(subobj) or part < 0:
  213. raise JsonPatchConflict("can't insert outside of list")
  214. else:
  215. subobj.insert(part, value) # pylint: disable=E1103
  216. elif isinstance(subobj, MutableMapping):
  217. if part is None:
  218. obj = value # we're replacing the root
  219. else:
  220. subobj[part] = value
  221. else:
  222. if part is None:
  223. raise TypeError("invalid document type {0}".format(type(subobj)))
  224. else:
  225. raise JsonPatchConflict("unable to fully resolve json pointer {0}, part {1}".format(self.location, part))
  226. return obj
  227. def _on_undo_remove(self, path, key):
  228. if self.path == path:
  229. if self.key > key:
  230. self.key += 1
  231. else:
  232. key += 1
  233. return key
  234. def _on_undo_add(self, path, key):
  235. if self.path == path:
  236. if self.key > key:
  237. self.key -= 1
  238. else:
  239. key += 1
  240. return key
  241. class ReplaceOperation(PatchOperation):
  242. """Replaces an object property or an array element by a new value."""
  243. def apply(self, obj):
  244. try:
  245. value = self.operation["value"]
  246. except KeyError as ex:
  247. raise InvalidJsonPatch(
  248. "The operation does not contain a 'value' member")
  249. subobj, part = self.pointer.to_last(obj)
  250. if part is None:
  251. return value
  252. if part == "-":
  253. raise InvalidJsonPatch("'path' with '-' can't be applied to 'replace' operation")
  254. if isinstance(subobj, MutableSequence):
  255. if part >= len(subobj) or part < 0:
  256. raise JsonPatchConflict("can't replace outside of list")
  257. elif isinstance(subobj, MutableMapping):
  258. if part not in subobj:
  259. msg = "can't replace a non-existent object '{0}'".format(part)
  260. raise JsonPatchConflict(msg)
  261. else:
  262. if part is None:
  263. raise TypeError("invalid document type {0}".format(type(subobj)))
  264. else:
  265. raise JsonPatchConflict("unable to fully resolve json pointer {0}, part {1}".format(self.location, part))
  266. subobj[part] = value
  267. return obj
  268. def _on_undo_remove(self, path, key):
  269. return key
  270. def _on_undo_add(self, path, key):
  271. return key
  272. class MoveOperation(PatchOperation):
  273. """Moves an object property or an array element to a new location."""
  274. def apply(self, obj):
  275. try:
  276. if isinstance(self.operation['from'], self.pointer_cls):
  277. from_ptr = self.operation['from']
  278. else:
  279. from_ptr = self.pointer_cls(self.operation['from'])
  280. except KeyError as ex:
  281. raise InvalidJsonPatch(
  282. "The operation does not contain a 'from' member")
  283. subobj, part = from_ptr.to_last(obj)
  284. try:
  285. value = subobj[part]
  286. except (KeyError, IndexError) as ex:
  287. raise JsonPatchConflict(str(ex))
  288. # If source and target are equal, this is a no-op
  289. if self.pointer == from_ptr:
  290. return obj
  291. if isinstance(subobj, MutableMapping) and \
  292. self.pointer.contains(from_ptr):
  293. raise JsonPatchConflict('Cannot move values into their own children')
  294. obj = RemoveOperation({
  295. 'op': 'remove',
  296. 'path': self.operation['from']
  297. }, pointer_cls=self.pointer_cls).apply(obj)
  298. obj = AddOperation({
  299. 'op': 'add',
  300. 'path': self.location,
  301. 'value': value
  302. }, pointer_cls=self.pointer_cls).apply(obj)
  303. return obj
  304. @property
  305. def from_path(self):
  306. from_ptr = self.pointer_cls(self.operation['from'])
  307. return '/'.join(from_ptr.parts[:-1])
  308. @property
  309. def from_key(self):
  310. from_ptr = self.pointer_cls(self.operation['from'])
  311. try:
  312. return int(from_ptr.parts[-1])
  313. except TypeError:
  314. return from_ptr.parts[-1]
  315. @from_key.setter
  316. def from_key(self, value):
  317. from_ptr = self.pointer_cls(self.operation['from'])
  318. from_ptr.parts[-1] = str(value)
  319. self.operation['from'] = from_ptr.path
  320. def _on_undo_remove(self, path, key):
  321. if self.from_path == path:
  322. if self.from_key >= key:
  323. self.from_key += 1
  324. else:
  325. key -= 1
  326. if self.path == path:
  327. if self.key > key:
  328. self.key += 1
  329. else:
  330. key += 1
  331. return key
  332. def _on_undo_add(self, path, key):
  333. if self.from_path == path:
  334. if self.from_key > key:
  335. self.from_key -= 1
  336. else:
  337. key -= 1
  338. if self.path == path:
  339. if self.key > key:
  340. self.key -= 1
  341. else:
  342. key += 1
  343. return key
  344. class TestOperation(PatchOperation):
  345. """Test value by specified location."""
  346. def apply(self, obj):
  347. try:
  348. subobj, part = self.pointer.to_last(obj)
  349. if part is None:
  350. val = subobj
  351. else:
  352. val = self.pointer.walk(subobj, part)
  353. except JsonPointerException as ex:
  354. raise JsonPatchTestFailed(str(ex))
  355. try:
  356. value = self.operation['value']
  357. except KeyError as ex:
  358. raise InvalidJsonPatch(
  359. "The operation does not contain a 'value' member")
  360. if val != value:
  361. msg = '{0} ({1}) is not equal to tested value {2} ({3})'
  362. raise JsonPatchTestFailed(msg.format(val, type(val),
  363. value, type(value)))
  364. return obj
  365. class CopyOperation(PatchOperation):
  366. """ Copies an object property or an array element to a new location """
  367. def apply(self, obj):
  368. try:
  369. from_ptr = self.pointer_cls(self.operation['from'])
  370. except KeyError as ex:
  371. raise InvalidJsonPatch(
  372. "The operation does not contain a 'from' member")
  373. subobj, part = from_ptr.to_last(obj)
  374. try:
  375. value = copy.deepcopy(subobj[part])
  376. except (KeyError, IndexError) as ex:
  377. raise JsonPatchConflict(str(ex))
  378. obj = AddOperation({
  379. 'op': 'add',
  380. 'path': self.location,
  381. 'value': value
  382. }, pointer_cls=self.pointer_cls).apply(obj)
  383. return obj
  384. class JsonPatch(object):
  385. json_dumper = staticmethod(json.dumps)
  386. json_loader = staticmethod(_jsonloads)
  387. operations = MappingProxyType({
  388. 'remove': RemoveOperation,
  389. 'add': AddOperation,
  390. 'replace': ReplaceOperation,
  391. 'move': MoveOperation,
  392. 'test': TestOperation,
  393. 'copy': CopyOperation,
  394. })
  395. """A JSON Patch is a list of Patch Operations.
  396. >>> patch = JsonPatch([
  397. ... {'op': 'add', 'path': '/foo', 'value': 'bar'},
  398. ... {'op': 'add', 'path': '/baz', 'value': [1, 2, 3]},
  399. ... {'op': 'remove', 'path': '/baz/1'},
  400. ... {'op': 'test', 'path': '/baz', 'value': [1, 3]},
  401. ... {'op': 'replace', 'path': '/baz/0', 'value': 42},
  402. ... {'op': 'remove', 'path': '/baz/1'},
  403. ... ])
  404. >>> doc = {}
  405. >>> result = patch.apply(doc)
  406. >>> expected = {'foo': 'bar', 'baz': [42]}
  407. >>> result == expected
  408. True
  409. JsonPatch object is iterable, so you can easily access each patch
  410. statement in a loop:
  411. >>> lpatch = list(patch)
  412. >>> expected = {'op': 'add', 'path': '/foo', 'value': 'bar'}
  413. >>> lpatch[0] == expected
  414. True
  415. >>> lpatch == patch.patch
  416. True
  417. Also JsonPatch could be converted directly to :class:`bool` if it contains
  418. any operation statements:
  419. >>> bool(patch)
  420. True
  421. >>> bool(JsonPatch([]))
  422. False
  423. This behavior is very handy with :func:`make_patch` to write more readable
  424. code:
  425. >>> old = {'foo': 'bar', 'numbers': [1, 3, 4, 8]}
  426. >>> new = {'baz': 'qux', 'numbers': [1, 4, 7]}
  427. >>> patch = make_patch(old, new)
  428. >>> if patch:
  429. ... # document have changed, do something useful
  430. ... patch.apply(old) #doctest: +ELLIPSIS
  431. {...}
  432. """
  433. def __init__(self, patch, pointer_cls=JsonPointer):
  434. self.patch = patch
  435. self.pointer_cls = pointer_cls
  436. # Verify that the structure of the patch document
  437. # is correct by retrieving each patch element.
  438. # Much of the validation is done in the initializer
  439. # though some is delayed until the patch is applied.
  440. for op in self.patch:
  441. self._get_operation(op)
  442. def __str__(self):
  443. """str(self) -> self.to_string()"""
  444. return self.to_string()
  445. def __bool__(self):
  446. return bool(self.patch)
  447. __nonzero__ = __bool__
  448. def __iter__(self):
  449. return iter(self.patch)
  450. def __hash__(self):
  451. return hash(tuple(self._ops))
  452. def __eq__(self, other):
  453. if not isinstance(other, JsonPatch):
  454. return False
  455. return self._ops == other._ops
  456. def __ne__(self, other):
  457. return not(self == other)
  458. @classmethod
  459. def from_string(cls, patch_str, loads=None, pointer_cls=JsonPointer):
  460. """Creates JsonPatch instance from string source.
  461. :param patch_str: JSON patch as raw string.
  462. :type patch_str: str
  463. :param loads: A function of one argument that loads a serialized
  464. JSON string.
  465. :type loads: function
  466. :param pointer_cls: JSON pointer class to use.
  467. :type pointer_cls: Type[JsonPointer]
  468. :return: :class:`JsonPatch` instance.
  469. """
  470. json_loader = loads or cls.json_loader
  471. patch = json_loader(patch_str)
  472. return cls(patch, pointer_cls=pointer_cls)
  473. @classmethod
  474. def from_diff(
  475. cls, src, dst, optimization=True, dumps=None,
  476. pointer_cls=JsonPointer,
  477. ):
  478. """Creates JsonPatch instance based on comparison of two document
  479. objects. Json patch would be created for `src` argument against `dst`
  480. one.
  481. :param src: Data source document object.
  482. :type src: dict
  483. :param dst: Data source document object.
  484. :type dst: dict
  485. :param dumps: A function of one argument that produces a serialized
  486. JSON string.
  487. :type dumps: function
  488. :param pointer_cls: JSON pointer class to use.
  489. :type pointer_cls: Type[JsonPointer]
  490. :return: :class:`JsonPatch` instance.
  491. >>> src = {'foo': 'bar', 'numbers': [1, 3, 4, 8]}
  492. >>> dst = {'baz': 'qux', 'numbers': [1, 4, 7]}
  493. >>> patch = JsonPatch.from_diff(src, dst)
  494. >>> new = patch.apply(src)
  495. >>> new == dst
  496. True
  497. """
  498. json_dumper = dumps or cls.json_dumper
  499. builder = DiffBuilder(src, dst, json_dumper, pointer_cls=pointer_cls)
  500. builder._compare_values('', None, src, dst)
  501. ops = list(builder.execute())
  502. return cls(ops, pointer_cls=pointer_cls)
  503. def to_string(self, dumps=None):
  504. """Returns patch set as JSON string."""
  505. json_dumper = dumps or self.json_dumper
  506. return json_dumper(self.patch)
  507. @property
  508. def _ops(self):
  509. return tuple(map(self._get_operation, self.patch))
  510. def apply(self, obj, in_place=False):
  511. """Applies the patch to a given object.
  512. :param obj: Document object.
  513. :type obj: dict
  514. :param in_place: Tweaks the way how patch would be applied - directly to
  515. specified `obj` or to its copy.
  516. :type in_place: bool
  517. :return: Modified `obj`.
  518. """
  519. if not in_place:
  520. obj = copy.deepcopy(obj)
  521. for operation in self._ops:
  522. obj = operation.apply(obj)
  523. return obj
  524. def _get_operation(self, operation):
  525. if 'op' not in operation:
  526. raise InvalidJsonPatch("Operation does not contain 'op' member")
  527. op = operation['op']
  528. if not isinstance(op, basestring):
  529. raise InvalidJsonPatch("Operation must be a string")
  530. if op not in self.operations:
  531. raise InvalidJsonPatch("Unknown operation {0!r}".format(op))
  532. cls = self.operations[op]
  533. return cls(operation, pointer_cls=self.pointer_cls)
  534. class DiffBuilder(object):
  535. def __init__(self, src_doc, dst_doc, dumps=json.dumps, pointer_cls=JsonPointer):
  536. self.dumps = dumps
  537. self.pointer_cls = pointer_cls
  538. self.index_storage = [{}, {}]
  539. self.index_storage2 = [[], []]
  540. self.__root = root = []
  541. self.src_doc = src_doc
  542. self.dst_doc = dst_doc
  543. root[:] = [root, root, None]
  544. def store_index(self, value, index, st):
  545. typed_key = (value, type(value))
  546. try:
  547. storage = self.index_storage[st]
  548. stored = storage.get(typed_key)
  549. if stored is None:
  550. storage[typed_key] = [index]
  551. else:
  552. storage[typed_key].append(index)
  553. except TypeError:
  554. self.index_storage2[st].append((typed_key, index))
  555. def take_index(self, value, st):
  556. typed_key = (value, type(value))
  557. try:
  558. stored = self.index_storage[st].get(typed_key)
  559. if stored:
  560. return stored.pop()
  561. except TypeError:
  562. storage = self.index_storage2[st]
  563. for i in range(len(storage)-1, -1, -1):
  564. if storage[i][0] == typed_key:
  565. return storage.pop(i)[1]
  566. def insert(self, op):
  567. root = self.__root
  568. last = root[0]
  569. last[1] = root[0] = [last, root, op]
  570. return root[0]
  571. def remove(self, index):
  572. link_prev, link_next, _ = index
  573. link_prev[1] = link_next
  574. link_next[0] = link_prev
  575. index[:] = []
  576. def iter_from(self, start):
  577. root = self.__root
  578. curr = start[1]
  579. while curr is not root:
  580. yield curr[2]
  581. curr = curr[1]
  582. def __iter__(self):
  583. root = self.__root
  584. curr = root[1]
  585. while curr is not root:
  586. yield curr[2]
  587. curr = curr[1]
  588. def execute(self):
  589. root = self.__root
  590. curr = root[1]
  591. while curr is not root:
  592. if curr[1] is not root:
  593. op_first, op_second = curr[2], curr[1][2]
  594. if op_first.location == op_second.location and \
  595. type(op_first) == RemoveOperation and \
  596. type(op_second) == AddOperation:
  597. yield ReplaceOperation({
  598. 'op': 'replace',
  599. 'path': op_second.location,
  600. 'value': op_second.operation['value'],
  601. }, pointer_cls=self.pointer_cls).operation
  602. curr = curr[1][1]
  603. continue
  604. yield curr[2].operation
  605. curr = curr[1]
  606. def _item_added(self, path, key, item):
  607. index = self.take_index(item, _ST_REMOVE)
  608. if index is not None:
  609. op = index[2]
  610. if type(op.key) == int and type(key) == int:
  611. for v in self.iter_from(index):
  612. op.key = v._on_undo_remove(op.path, op.key)
  613. self.remove(index)
  614. if op.location != _path_join(path, key):
  615. new_op = MoveOperation({
  616. 'op': 'move',
  617. 'from': op.location,
  618. 'path': _path_join(path, key),
  619. }, pointer_cls=self.pointer_cls)
  620. self.insert(new_op)
  621. else:
  622. new_op = AddOperation({
  623. 'op': 'add',
  624. 'path': _path_join(path, key),
  625. 'value': item,
  626. }, pointer_cls=self.pointer_cls)
  627. new_index = self.insert(new_op)
  628. self.store_index(item, new_index, _ST_ADD)
  629. def _item_removed(self, path, key, item):
  630. new_op = RemoveOperation({
  631. 'op': 'remove',
  632. 'path': _path_join(path, key),
  633. }, pointer_cls=self.pointer_cls)
  634. index = self.take_index(item, _ST_ADD)
  635. new_index = self.insert(new_op)
  636. if index is not None:
  637. op = index[2]
  638. # We can't rely on the op.key type since PatchOperation casts
  639. # the .key property to int and this path wrongly ends up being taken
  640. # for numeric string dict keys while the intention is to only handle lists.
  641. # So we do an explicit check on the item affected by the op instead.
  642. added_item = op.pointer.to_last(self.dst_doc)[0]
  643. if type(added_item) == list:
  644. for v in self.iter_from(index):
  645. op.key = v._on_undo_add(op.path, op.key)
  646. self.remove(index)
  647. if new_op.location != op.location:
  648. new_op = MoveOperation({
  649. 'op': 'move',
  650. 'from': new_op.location,
  651. 'path': op.location,
  652. }, pointer_cls=self.pointer_cls)
  653. new_index[2] = new_op
  654. else:
  655. self.remove(new_index)
  656. else:
  657. self.store_index(item, new_index, _ST_REMOVE)
  658. def _item_replaced(self, path, key, item):
  659. self.insert(ReplaceOperation({
  660. 'op': 'replace',
  661. 'path': _path_join(path, key),
  662. 'value': item,
  663. }, pointer_cls=self.pointer_cls))
  664. def _compare_dicts(self, path, src, dst):
  665. src_keys = set(src.keys())
  666. dst_keys = set(dst.keys())
  667. added_keys = dst_keys - src_keys
  668. removed_keys = src_keys - dst_keys
  669. for key in removed_keys:
  670. self._item_removed(path, str(key), src[key])
  671. for key in added_keys:
  672. self._item_added(path, str(key), dst[key])
  673. for key in src_keys & dst_keys:
  674. self._compare_values(path, key, src[key], dst[key])
  675. def _compare_lists(self, path, src, dst):
  676. len_src, len_dst = len(src), len(dst)
  677. max_len = max(len_src, len_dst)
  678. min_len = min(len_src, len_dst)
  679. for key in range(max_len):
  680. if key < min_len:
  681. old, new = src[key], dst[key]
  682. if old == new:
  683. continue
  684. elif isinstance(old, MutableMapping) and \
  685. isinstance(new, MutableMapping):
  686. self._compare_dicts(_path_join(path, key), old, new)
  687. elif isinstance(old, MutableSequence) and \
  688. isinstance(new, MutableSequence):
  689. self._compare_lists(_path_join(path, key), old, new)
  690. else:
  691. self._item_removed(path, key, old)
  692. self._item_added(path, key, new)
  693. elif len_src > len_dst:
  694. self._item_removed(path, len_dst, src[key])
  695. else:
  696. self._item_added(path, key, dst[key])
  697. def _compare_values(self, path, key, src, dst):
  698. if isinstance(src, MutableMapping) and \
  699. isinstance(dst, MutableMapping):
  700. self._compare_dicts(_path_join(path, key), src, dst)
  701. elif isinstance(src, MutableSequence) and \
  702. isinstance(dst, MutableSequence):
  703. self._compare_lists(_path_join(path, key), src, dst)
  704. # To ensure we catch changes to JSON, we can't rely on a simple
  705. # src == dst, because it would not recognize the difference between
  706. # 1 and True, among other things. Using json.dumps is the most
  707. # fool-proof way to ensure we catch type changes that matter to JSON
  708. # and ignore those that don't. The performance of this could be
  709. # improved by doing more direct type checks, but we'd need to be
  710. # careful to accept type changes that don't matter when JSONified.
  711. elif self.dumps(src) == self.dumps(dst):
  712. return
  713. else:
  714. self._item_replaced(path, key, dst)
  715. def _path_join(path, key):
  716. if key is None:
  717. return path
  718. return path + '/' + str(key).replace('~', '~0').replace('/', '~1')