From 54ebdb5a719d68010e20c4f95666ae1eabc97b39 Mon Sep 17 00:00:00 2001 From: Seperman Date: Mon, 8 Apr 2024 12:17:27 -0700 Subject: [PATCH 1/4] Include type info and change the "unknown" value for flat rows to something that is friendly for Postgres enums --- deepdiff/delta.py | 38 ++++++++++++++---- deepdiff/helper.py | 2 +- docs/serialization.rst | 28 +++++++------- tests/test_delta.py | 87 +++++++++++++++++++++++------------------- 4 files changed, 93 insertions(+), 62 deletions(-) diff --git a/deepdiff/delta.py b/deepdiff/delta.py index 62068dd..39f7d36 100644 --- a/deepdiff/delta.py +++ b/deepdiff/delta.py @@ -878,7 +878,7 @@ def to_dict(self): return dict(self.diff) @staticmethod - def _get_flat_row(action, info, _parse_path, keys_and_funcs): + def _get_flat_row(action, info, _parse_path, keys_and_funcs, report_type_changes=True): for path, details in info.items(): row = {'path': _parse_path(path), 'action': action} for key, new_key, func in keys_and_funcs: @@ -887,6 +887,11 @@ def _get_flat_row(action, info, _parse_path, keys_and_funcs): row[new_key] = func(details[key]) else: row[new_key] = details[key] + if report_type_changes: + if 'value' in row and 'type' not in row: + row['type'] = type(row['value']) + if 'old_value' in row and 'old_type' not in row: + row['old_type'] = type(row['old_value']) yield FlatDeltaRow(**row) @staticmethod @@ -1060,6 +1065,9 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) - 'iterable_items_removed_at_indexes': 'unordered_iterable_item_removed', } for action, info in self.diff.items(): + if action == '_iterable_opcodes': + result.extend(self._flatten_iterable_opcodes()) + continue if action.startswith('_'): continue if action in FLATTENING_NEW_ACTION_MAP: @@ -1072,12 +1080,20 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) - path2.append((index, 'GET')) else: path2.append(index) - result.append(FlatDeltaRow(path=path2, value=value, action=new_action)) + if report_type_changes: + row = FlatDeltaRow(path=path2, value=value, action=new_action, type=type(value)) + else: + row = FlatDeltaRow(path=path2, value=value, action=new_action) + result.append(row) elif action in {'set_item_added', 'set_item_removed'}: for path, values in info.items(): path = _parse_path(path) for value in values: - result.append(FlatDeltaRow(path=path, value=value, action=action)) + if report_type_changes: + row = FlatDeltaRow(path=path, value=value, action=action, type=type(value)) + else: + row = FlatDeltaRow(path=path, value=value, action=action) + result.append(row) elif action == 'dictionary_item_added': for path, value in info.items(): path = _parse_path(path) @@ -1092,14 +1108,22 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) - elif isinstance(value, set) and len(value) == 1: value = value.pop() action = 'set_item_added' - result.append(FlatDeltaRow(path=path, value=value, action=action)) + if report_type_changes: + row = FlatDeltaRow(path=path, value=value, action=action, type=type(value)) + else: + row = FlatDeltaRow(path=path, value=value, action=action) + result.append(row) elif action in { 'dictionary_item_removed', 'iterable_item_added', 'iterable_item_removed', 'attribute_removed', 'attribute_added' }: for path, value in info.items(): path = _parse_path(path) - result.append(FlatDeltaRow(path=path, value=value, action=action)) + if report_type_changes: + row = FlatDeltaRow(path=path, value=value, action=action, type=type(value)) + else: + row = FlatDeltaRow(path=path, value=value, action=action) + result.append(row) elif action == 'type_changes': if not report_type_changes: action = 'values_changed' @@ -1109,16 +1133,16 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) - info=info, _parse_path=_parse_path, keys_and_funcs=keys_and_funcs, + report_type_changes=report_type_changes, ): result.append(row) - elif action == '_iterable_opcodes': - result.extend(self._flatten_iterable_opcodes()) else: for row in self._get_flat_row( action=action, info=info, _parse_path=_parse_path, keys_and_funcs=keys_and_funcs, + report_type_changes=report_type_changes, ): result.append(row) return result diff --git a/deepdiff/helper.py b/deepdiff/helper.py index cdf34ca..22846f1 100644 --- a/deepdiff/helper.py +++ b/deepdiff/helper.py @@ -771,7 +771,7 @@ class FlatDataAction(str, enum.Enum): unordered_iterable_item_removed = 'unordered_iterable_item_removed' -UnkownValueCode = '*-UNKNOWN-*' +UnkownValueCode = 'unknown___' class FlatDeltaRow(NamedTuple): diff --git a/docs/serialization.rst b/docs/serialization.rst index 0f63428..92ef757 100644 --- a/docs/serialization.rst +++ b/docs/serialization.rst @@ -181,7 +181,7 @@ Flat Row Specs: unordered_iterable_item_removed = 'unordered_iterable_item_removed' - UnkownValueCode = '*-UNKNOWN-*' + UnkownValueCode = 'unknown___' class FlatDeltaRow(NamedTuple): @@ -205,7 +205,7 @@ Delta Serialize To Flat Dictionaries Sometimes, it is desired to serialize a :ref:`delta_label` object to a list of flat dictionaries. For example, to store them in relation databases. In that case, you can use the Delta.to_flat_dicts to achieve the desired outcome. -Since None is a valid value, we use a special hard-coded string to signify "unkown": '*-UNKNOWN-*' +Since None is a valid value, we use a special hard-coded string to signify "unkown": 'unknown___' .. note:: Many new keys are added to the flat dicts in DeepDiff 7.0.0 @@ -226,25 +226,25 @@ For example: >>> pprint(flat_dicts, indent=2) [ { 'action': 'dictionary_item_added', 'new_path': None, - 'old_type': '*-UNKNOWN-*', - 'old_value': '*-UNKNOWN-*', + 'old_type': 'unknown___', + 'old_value': 'unknown___', 'path': ['field2', 'key2'], 't1_from_index': None, 't1_to_index': None, 't2_from_index': None, 't2_to_index': None, - 'type': '*-UNKNOWN-*', + 'type': 'unknown___', 'value': 'value2'}, { 'action': 'dictionary_item_removed', 'new_path': None, - 'old_type': '*-UNKNOWN-*', - 'old_value': '*-UNKNOWN-*', + 'old_type': 'unknown___', + 'old_value': 'unknown___', 'path': ['key1'], 't1_from_index': None, 't1_to_index': None, 't2_from_index': None, 't2_to_index': None, - 'type': '*-UNKNOWN-*', + 'type': 'unknown___', 'value': 'value1'}] @@ -261,25 +261,25 @@ Example 2: >>> pprint(flat_dicts, indent=2) [ { 'action': 'iterable_item_added', 'new_path': None, - 'old_type': '*-UNKNOWN-*', - 'old_value': '*-UNKNOWN-*', + 'old_type': 'unknown___', + 'old_value': 'unknown___', 'path': [2], 't1_from_index': None, 't1_to_index': None, 't2_from_index': None, 't2_to_index': None, - 'type': '*-UNKNOWN-*', + 'type': 'unknown___', 'value': 'C'}, { 'action': 'iterable_item_added', 'new_path': None, - 'old_type': '*-UNKNOWN-*', - 'old_value': '*-UNKNOWN-*', + 'old_type': 'unknown___', + 'old_value': 'unknown___', 'path': [3], 't1_from_index': None, 't1_to_index': None, 't2_from_index': None, 't2_to_index': None, - 'type': '*-UNKNOWN-*', + 'type': 'unknown___', 'value': 'D'}] diff --git a/tests/test_delta.py b/tests/test_delta.py index b03b9e6..72386e7 100644 --- a/tests/test_delta.py +++ b/tests/test_delta.py @@ -73,8 +73,8 @@ def test_list_difference_add_delta(self): flat_result1 = delta.to_flat_rows() flat_expected1 = [ - FlatDeltaRow(path=[3], value=5, action='iterable_item_added'), - FlatDeltaRow(path=[2], value=3, action='iterable_item_added'), + FlatDeltaRow(path=[3], value=5, action='iterable_item_added', type=int), + FlatDeltaRow(path=[2], value=3, action='iterable_item_added', type=int), ] assert flat_expected1 == flat_result1 @@ -291,9 +291,9 @@ def test_list_difference3_delta(self): flat_result1 = delta.to_flat_rows() flat_expected1 = [ - FlatDeltaRow(path=[4, 'b', 2], action='values_changed', value=2, old_value=5), - FlatDeltaRow(path=[4, 'b', 1], action='values_changed', value=3, old_value=2), - FlatDeltaRow(path=[4, 'b', 3], value=5, action='iterable_item_added'), + FlatDeltaRow(path=[4, 'b', 2], action='values_changed', value=2, old_value=5, type=int, old_type=int), + FlatDeltaRow(path=[4, 'b', 1], action='values_changed', value=3, old_value=2, type=int, old_type=int), + FlatDeltaRow(path=[4, 'b', 3], value=5, action='iterable_item_added', type=int), ] assert flat_expected1 == flat_result1 @@ -332,9 +332,9 @@ def test_list_difference_delta_raises_error_if_prev_value_does_not_match(self): flat_result2 = delta2.to_flat_rows() flat_expected2 = [ - FlatDeltaRow(path=[2], action='values_changed', value=2, old_value=5), - FlatDeltaRow(path=[1], action='values_changed', value=3, old_value=2), - FlatDeltaRow(path=[3], value=5, action='iterable_item_added'), + FlatDeltaRow(path=[2], action='values_changed', value=2, old_value=5, type=int, old_type=int), + FlatDeltaRow(path=[1], action='values_changed', value=3, old_value=2, type=int, old_type=int), + FlatDeltaRow(path=[3], value=5, action='iterable_item_added', type=int), ] assert flat_expected2 == flat_result2 @@ -363,8 +363,8 @@ def test_list_difference_delta1(self): flat_result = delta.to_flat_rows() flat_expected = [ - FlatDeltaRow(path=[4, 'b', 2], value='to_be_removed', action='iterable_item_removed'), - FlatDeltaRow(path=[4, 'b', 3], value='to_be_removed2', action='iterable_item_removed'), + FlatDeltaRow(path=[4, 'b', 2], value='to_be_removed', action='iterable_item_removed', type=str), + FlatDeltaRow(path=[4, 'b', 3], value='to_be_removed2', action='iterable_item_removed', type=str), ] assert flat_expected == flat_result @@ -567,7 +567,8 @@ def compare_func(item1, item2, level=None): 'professionalDesignation': '', 'suffix': 'SR', 'nameIdentifier': '00003'}, - action='unordered_iterable_item_added'), + action='unordered_iterable_item_added', + type=dict), FlatDeltaRow(path=['individualNames', 1], value={'firstName': 'John', 'lastName': 'Doe', @@ -577,7 +578,9 @@ def compare_func(item1, item2, level=None): 'professionalDesignation': '', 'suffix': 'SR', 'nameIdentifier': '00002'}, - action='unordered_iterable_item_removed')] + action='unordered_iterable_item_removed', + type=dict), + ] preserved_flat_dict_list = copy.deepcopy(flat_rows_list) # Use this later for assert comparison @@ -1405,13 +1408,13 @@ def test_list_ignore_order_various_deltas2(self): flat_result1 = delta1.to_flat_rows() flat_expected1 = [ - {'path': [0], 'value': 7, 'action': 'unordered_iterable_item_added'}, - {'path': [6], 'value': 8, 'action': 'unordered_iterable_item_added'}, - {'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added'}, - {'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added'}, - {'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added'}, - {'path': [6], 'value': 6, 'action': 'unordered_iterable_item_removed'}, - {'path': [0], 'value': 5, 'action': 'unordered_iterable_item_removed'}, + {'path': [0], 'value': 7, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [6], 'value': 8, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [6], 'value': 6, 'action': 'unordered_iterable_item_removed', 'type': int}, + {'path': [0], 'value': 5, 'action': 'unordered_iterable_item_removed', 'type': int}, ] flat_expected1 = [FlatDeltaRow(**i) for i in flat_expected1] assert flat_expected1 == flat_result1 @@ -1422,11 +1425,11 @@ def test_list_ignore_order_various_deltas2(self): flat_result2 = delta2.to_flat_rows() flat_expected2 = [ - {'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added'}, - {'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added'}, - {'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added'}, - {'path': [6], 'action': 'values_changed', 'value': 7}, - {'path': [0], 'action': 'values_changed', 'value': 8}, + {'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int}, + {'path': [6], 'action': 'values_changed', 'value': 7, 'type': int}, + {'path': [0], 'action': 'values_changed', 'value': 8, 'type': int}, ] flat_expected2 = [FlatDeltaRow(**i) for i in flat_expected2] assert flat_expected2 == flat_result2 @@ -1565,7 +1568,7 @@ def test_apply_delta_to_incompatible_object6_value_change(self): assert [] == t4 flat_result2 = delta2.to_flat_rows() - flat_expected2 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5}] + flat_expected2 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5, 'type': int}] flat_expected2 = [FlatDeltaRow(**i) for i in flat_expected2] assert flat_expected2 == flat_result2 @@ -1575,7 +1578,7 @@ def test_apply_delta_to_incompatible_object6_value_change(self): delta3 = Delta(diff, raise_errors=False, bidirectional=True) flat_result3 = delta3.to_flat_rows() - flat_expected3 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5, 'old_value': 4}] + flat_expected3 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5, 'old_value': 4, 'type': int, 'old_type': int}] flat_expected3 = [FlatDeltaRow(**i) for i in flat_expected3] assert flat_expected3 == flat_result3 @@ -1685,7 +1688,7 @@ def test_delta_to_dict(self): assert expected == result flat_result = delta.to_flat_rows() - flat_expected = [{'action': 'unordered_iterable_item_removed', 'path': [2], 'value': 'B'}] + flat_expected = [{'action': 'unordered_iterable_item_removed', 'path': [2], 'value': 'B', 'type': str}] flat_expected = [FlatDeltaRow(**i) for i in flat_expected] assert flat_expected == flat_result @@ -1766,10 +1769,10 @@ def test_delta_set_in_objects(self): delta = Delta(DeepDiff(t1, t2)) flat_result = delta.to_flat_rows() flat_expected = [ - {'path': [0, 1], 'value': 10, 'action': 'set_item_added'}, - {'path': [0, 0], 'action': 'values_changed', 'value': 2}, - {'path': [0, 1], 'value': 'A', 'action': 'set_item_removed'}, - {'path': [0, 1], 'value': 'C', 'action': 'set_item_added'}, + {'path': [0, 1], 'value': 10, 'action': 'set_item_added', 'type': int}, + {'path': [0, 0], 'action': 'values_changed', 'value': 2, 'type': int}, + {'path': [0, 1], 'value': 'A', 'action': 'set_item_removed', 'type': str}, + {'path': [0, 1], 'value': 'C', 'action': 'set_item_added', 'type': str}, ] flat_expected = [FlatDeltaRow(**i) for i in flat_expected] @@ -1885,11 +1888,11 @@ def test_compare_func_with_duplicates_removed(self): flat_result = delta.to_flat_rows() flat_expected = [ - {'path': [2], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed'}, - {'path': [0], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed'}, - {'path': [3], 'value': {'id': 3, 'val': 3}, 'action': 'iterable_item_removed'}, - {'path': [0], 'action': 'iterable_item_moved', 'value': {'id': 1, 'val': 3}, 'new_path': [2]}, - {'path': [3], 'action': 'iterable_item_moved', 'value': {'id': 3, 'val': 3}, 'new_path': [0]}, + {'path': [2], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed', 'type': dict}, + {'path': [0], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed', 'type': dict}, + {'path': [3], 'value': {'id': 3, 'val': 3}, 'action': 'iterable_item_removed', 'type': dict}, + {'path': [0], 'action': 'iterable_item_moved', 'value': {'id': 1, 'val': 3}, 'new_path': [2], 'type': dict}, + {'path': [3], 'action': 'iterable_item_moved', 'value': {'id': 3, 'val': 3}, 'new_path': [0], 'type': dict}, ] flat_expected = [FlatDeltaRow(**i) for i in flat_expected] @@ -2289,11 +2292,13 @@ def test_subtract_delta_made_from_flat_dicts1(self): expected_flat_dicts = [{ 'path': ['field_name1', 0], 'value': 'xxx', - 'action': 'iterable_item_removed' + 'action': 'iterable_item_removed', + 'type': str, }, { 'path': ['field_name1', 1], 'value': 'yyy', - 'action': 'iterable_item_removed' + 'action': 'iterable_item_removed', + 'type': str, }] expected_flat_dicts = [FlatDeltaRow(**i) for i in expected_flat_dicts] @@ -2318,11 +2323,13 @@ def test_subtract_delta_made_from_flat_dicts2(self): expected_flat_dicts = [{ 'path': ['field_name1', 0], 'value': 'xxx', - 'action': 'iterable_item_added' + 'action': 'iterable_item_added', + 'type': str, }, { 'path': ['field_name1', 1], 'value': 'yyy', - 'action': 'iterable_item_added' + 'action': 'iterable_item_added', + 'type': str, }] expected_flat_dicts = [FlatDeltaRow(**i) for i in expected_flat_dicts] From 759bb8217eaf2ebc7cd168458d601222be61164b Mon Sep 17 00:00:00 2001 From: Seperman Date: Mon, 8 Apr 2024 14:55:14 -0700 Subject: [PATCH 2/4] op_codes conversion to flat dicts --- deepdiff/delta.py | 82 ++++++++++++++++++++++++++++++++++++++++----- deepdiff/helper.py | 30 ++++++++++++++++- deepdiff/path.py | 3 +- tests/test_delta.py | 25 +++++++++++++- 4 files changed, 128 insertions(+), 12 deletions(-) diff --git a/deepdiff/delta.py b/deepdiff/delta.py index 39f7d36..b679d50 100644 --- a/deepdiff/delta.py +++ b/deepdiff/delta.py @@ -11,7 +11,9 @@ strings, short_repr, numbers, np_ndarray, np_array_factory, numpy_dtypes, get_doc, not_found, numpy_dtype_string_to_type, dict_, - Opcode, FlatDeltaRow, UnkownValueCode, + Opcode, FlatDeltaRow, UnkownValueCode, FlatDataAction, + OPCODE_TAG_TO_FLAT_DATA_ACTION, + FLAT_DATA_ACTION_TO_OPCODE_TAG, ) from deepdiff.path import ( _path_to_elements, _get_nested_obj, _get_nested_obj_and_force, @@ -877,6 +879,31 @@ def dumps(self): def to_dict(self): return dict(self.diff) + def _flatten_iterable_opcodes(self, _parse_path): + """ + Converts op_codes to FlatDeltaRows + """ + result = [] + for path, op_codes in self.diff['_iterable_opcodes'].items(): + for op_code in op_codes: + result.append( + FlatDeltaRow( + path=_parse_path(path), + action=OPCODE_TAG_TO_FLAT_DATA_ACTION[op_code.tag], + value=op_code.new_values, + old_value=op_code.old_values, + type=type(op_code.new_values), + old_type=type(op_code.old_values), + new_path=None, + t1_from_index=op_code.t1_from_index, + t1_to_index=op_code.t1_to_index, + t2_from_index=op_code.t2_from_index, + t2_to_index=op_code.t2_to_index, + + ) + ) + return result + @staticmethod def _get_flat_row(action, info, _parse_path, keys_and_funcs, report_type_changes=True): for path, details in info.items(): @@ -923,28 +950,44 @@ def _from_flat_dicts(flat_dict_list): if action in FLATTENING_NEW_ACTION_MAP: action = FLATTENING_NEW_ACTION_MAP[action] index = path.pop() - if action in {'attribute_added', 'attribute_removed'}: + if action in { + FlatDataAction.attribute_added, + FlatDataAction.attribute_removed, + }: root_element = ('root', GETATTR) else: root_element = ('root', GET) - path_str = stringify_path(path, root_element=root_element) # We need the string path + if isinstance(path, str): + path_str = path + else: + path_str = stringify_path(path, root_element=root_element) # We need the string path if new_path and new_path != path: new_path = stringify_path(new_path, root_element=root_element) else: new_path = None if action not in result: result[action] = {} - if action in {'iterable_items_added_at_indexes', 'iterable_items_removed_at_indexes'}: + if action in { + 'iterable_items_added_at_indexes', + 'iterable_items_removed_at_indexes', + }: if path_str not in result[action]: result[action][path_str] = {} result[action][path_str][index] = value - elif action in {'set_item_added', 'set_item_removed'}: + elif action in { + FlatDataAction.set_item_added, + FlatDataAction.set_item_removed + }: if path_str not in result[action]: result[action][path_str] = set() result[action][path_str].add(value) elif action in { - 'dictionary_item_added', 'dictionary_item_removed', - 'attribute_removed', 'attribute_added', 'iterable_item_added', 'iterable_item_removed', + FlatDataAction.dictionary_item_added, + FlatDataAction.dictionary_item_removed, + FlatDataAction.attribute_removed, + FlatDataAction.attribute_added, + FlatDataAction.iterable_item_added, + FlatDataAction.iterable_item_removed, }: result[action][path_str] = value elif action == 'values_changed': @@ -964,8 +1007,29 @@ def _from_flat_dicts(flat_dict_list): ]: if elem_value != UnkownValueCode: result[action][path_str][elem] = elem_value - elif action == 'iterable_item_moved': + elif action == FlatDataAction.iterable_item_moved: result[action][path_str] = {'value': value} + elif action in { + FlatDataAction.iterable_items_inserted, + FlatDataAction.iterable_items_deleted, + FlatDataAction.iterable_items_replaced, + FlatDataAction.iterable_items_equal, + }: + if '_iterable_opcodes' not in result: + result['_iterable_opcodes'] = {} + if path_str not in result['_iterable_opcodes']: + result['_iterable_opcodes'][path_str] = [] + result['_iterable_opcodes'][path_str].append( + Opcode( + tag=FLAT_DATA_ACTION_TO_OPCODE_TAG[action], + t1_from_index=flat_dict.get('t1_from_index'), + t1_to_index=flat_dict.get('t1_to_index'), + t2_from_index=flat_dict.get('t2_from_index'), + t2_to_index=flat_dict.get('t2_to_index'), + new_values=flat_dict.get('value'), + old_values=flat_dict.get('old_value'), + ) + ) if new_path: result[action][path_str]['new_path'] = new_path @@ -1066,7 +1130,7 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) - } for action, info in self.diff.items(): if action == '_iterable_opcodes': - result.extend(self._flatten_iterable_opcodes()) + result.extend(self._flatten_iterable_opcodes(_parse_path=_parse_path)) continue if action.startswith('_'): continue diff --git a/deepdiff/helper.py b/deepdiff/helper.py index 22846f1..431bd58 100644 --- a/deepdiff/helper.py +++ b/deepdiff/helper.py @@ -110,6 +110,17 @@ class pydantic_base_model_type: NUMERICS = frozenset(string.digits) +class EnumBase(str, enum.Enum): + def __repr__(self): + """ + We need to add a single quotes so we can easily copy the value when we do ipdb. + """ + return f"'{self.name}'" + + def __str__(self): + return self.name + + def _int_or_zero(value): """ Tries to extract some number from a string. @@ -739,6 +750,13 @@ def named_tuple_repr(self): return f"{self.__class__.__name__}({', '.join(fields)})" +class OpcodeTag(EnumBase): + insert = 'insert' + delete = 'delete' + equal = 'equal' + replace = 'replace' + + class Opcode(NamedTuple): tag: str t1_from_index: int @@ -751,7 +769,7 @@ class Opcode(NamedTuple): __repr__ = __str__ = named_tuple_repr -class FlatDataAction(str, enum.Enum): +class FlatDataAction(EnumBase): values_changed = 'values_changed' type_changes = 'type_changes' set_item_added = 'set_item_added' @@ -771,6 +789,16 @@ class FlatDataAction(str, enum.Enum): unordered_iterable_item_removed = 'unordered_iterable_item_removed' +OPCODE_TAG_TO_FLAT_DATA_ACTION = { + OpcodeTag.insert: FlatDataAction.iterable_items_inserted, + OpcodeTag.delete: FlatDataAction.iterable_items_deleted, + OpcodeTag.replace: FlatDataAction.iterable_items_replaced, + OpcodeTag.equal: FlatDataAction.iterable_items_equal, +} + +FLAT_DATA_ACTION_TO_OPCODE_TAG = {v: i for i, v in OPCODE_TAG_TO_FLAT_DATA_ACTION.items()} + + UnkownValueCode = 'unknown___' diff --git a/deepdiff/path.py b/deepdiff/path.py index dd74144..faf7b51 100644 --- a/deepdiff/path.py +++ b/deepdiff/path.py @@ -261,7 +261,8 @@ def parse_path(path, root_element=DEFAULT_FIRST_ELEMENT, include_actions=False): result = _path_to_elements(path, root_element=root_element) result = iter(result) - next(result) # We don't want the root item + if root_element: + next(result) # We don't want the root item if include_actions is False: return [i[0] for i in result] return [{'element': i[0], 'action': i[1]} for i in result] diff --git a/tests/test_delta.py b/tests/test_delta.py index 72386e7..e60d675 100644 --- a/tests/test_delta.py +++ b/tests/test_delta.py @@ -9,7 +9,7 @@ from unittest import mock from ordered_set import OrderedSet from deepdiff import Delta, DeepDiff -from deepdiff.helper import np, number_to_string, TEXT_VIEW, DELTA_VIEW, CannotCompare, FlatDeltaRow +from deepdiff.helper import np, number_to_string, TEXT_VIEW, DELTA_VIEW, CannotCompare, FlatDeltaRow, FlatDataAction from deepdiff.path import GETATTR, GET from deepdiff.delta import ( ELEM_NOT_FOUND_TO_ADD_MSG, @@ -2397,6 +2397,29 @@ def test_list_of_alphabet_and_its_delta(self): assert l2 == l1 + delta4 assert l1 == l2 - delta4 + flat_rows = delta2.to_flat_rows() + + expected_flat_rows = [ + FlatDeltaRow(path=[3], action='values_changed', value='X', old_value='D', type=str, old_type=str, new_path=[2]), + FlatDeltaRow(path=[6], action='values_changed', value='Z', old_value='G', type=str, old_type=str), + FlatDeltaRow(path=[5], action='values_changed', value='Y', old_value='F', type=str, old_type=str), + FlatDeltaRow(path=[], action=FlatDataAction.iterable_items_deleted, value=[], old_value=['A'], type=list, old_type=list, t1_from_index=0, t1_to_index=1, t2_from_index=0, t2_to_index=0), + FlatDeltaRow(path=[], action=FlatDataAction.iterable_items_equal, value=None, old_value=None, type=type(None), old_type=type(None), t1_from_index=1, t1_to_index=3, t2_from_index=0, t2_to_index=2), + FlatDeltaRow(path=[], action=FlatDataAction.iterable_items_replaced, value=['X'], old_value=['D', 'E', 'F', 'G'], type=list, old_type=list, t1_from_index=3, t1_to_index=7, t2_from_index=2, t2_to_index=3), + FlatDeltaRow(path=[], action=FlatDataAction.iterable_items_equal, value=None, old_value=None, type=type(None), old_type=type(None), t1_from_index=7, t1_to_index=9, t2_from_index=3, t2_to_index=5), + FlatDeltaRow(path=[], action=FlatDataAction.iterable_items_inserted, value=['Y', 'Z'], old_value=[], type=list, old_type=list, t1_from_index=9, t1_to_index=9, t2_from_index=5, t2_to_index=7) + ] + + # The order of the first 3 items is not deterministic + assert not DeepDiff(expected_flat_rows[:3], flat_rows[:3], ignore_order=True) + assert expected_flat_rows[3:] == flat_rows[3:] + + delta5 = Delta(flat_rows_list=flat_rows, bidirectional=True, force=True) + + + assert l2 == l1 + delta5 + assert l1 == l2 - delta5 + def test_delta_flat_rows(self): t1 = {"key1": "value1"} t2 = {"field2": {"key2": "value2"}} From 2a4f963a1d4d3afab35bfc1eb134dee7c08f326c Mon Sep 17 00:00:00 2001 From: Seperman Date: Mon, 8 Apr 2024 15:55:44 -0700 Subject: [PATCH 3/4] updating docs --- CHANGELOG.md | 3 +++ README.md | 4 ++++ docs/changelog.rst | 5 +++++ 3 files changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6769077..d5629e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # DeepDiff Change log + +- v7-0-1 + - Fixes the translation between Difflib opcodes and Delta flat rows. - v7-0-0 - When verbose=2, return `new_path` when the `path` and `new_path` are different (for example when ignore_order=True and the index of items have changed). - Dropping support for Python 3.7 diff --git a/README.md b/README.md index b6590a9..2bdc708 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,10 @@ Tested on Python 3.8+ and PyPy3. Please check the [ChangeLog](CHANGELOG.md) file for the detailed information. +DeepDiff 7-0-1 + +- Fixes the translation between Difflib opcodes and Delta flat rows. + DeepDiff 7-0-0 - DeepDiff 7 comes with an improved delta object. [Delta to flat dictionaries](https://zepworks.com/deepdiff/current/serialization.html#delta-serialize-to-flat-dictionaries) have undergone a major change. We have also introduced [Delta serialize to flat rows](https://zepworks.com/deepdiff/current/serialization.html#delta-serialize-to-flat-rows). diff --git a/docs/changelog.rst b/docs/changelog.rst index 9cd1096..085f2f3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,11 @@ Changelog DeepDiff Changelog + +- v7-0-1 + + - Fixes the translation between Difflib opcodes and Delta flat rows. + - v7-0-0 - When verbose=2, return ``new_path`` when the ``path`` and From b391ae991d1cd7650092f5f66808870aa8a3abd6 Mon Sep 17 00:00:00 2001 From: Seperman Date: Mon, 8 Apr 2024 15:56:05 -0700 Subject: [PATCH 4/4] =?UTF-8?q?Bump=20version:=207.0.0=20=E2=86=92=207.0.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CITATION.cff | 2 +- README.md | 4 ++-- deepdiff/__init__.py | 2 +- docs/conf.py | 4 ++-- docs/index.rst | 2 +- setup.cfg | 2 +- setup.py | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 20de753..6dc8039 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -5,6 +5,6 @@ authors: given-names: "Sep" orcid: "https://orcid.org/0009-0009-5828-4345" title: "DeepDiff" -version: 7.0.0 +version: 7.0.1 date-released: 2024 url: "https://github.com/seperman/deepdiff" diff --git a/README.md b/README.md index 2bdc708..c153747 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# DeepDiff v 7.0.0 +# DeepDiff v 7.0.1 ![Downloads](https://img.shields.io/pypi/dm/deepdiff.svg?style=flat) ![Python Versions](https://img.shields.io/pypi/pyversions/deepdiff.svg?style=flat) @@ -17,7 +17,7 @@ Tested on Python 3.8+ and PyPy3. -- **[Documentation](https://zepworks.com/deepdiff/7.0.0/)** +- **[Documentation](https://zepworks.com/deepdiff/7.0.1/)** ## What is new? diff --git a/deepdiff/__init__.py b/deepdiff/__init__.py index 2f321a7..a3b3ed5 100644 --- a/deepdiff/__init__.py +++ b/deepdiff/__init__.py @@ -1,6 +1,6 @@ """This module offers the DeepDiff, DeepSearch, grep, Delta and DeepHash classes.""" # flake8: noqa -__version__ = '7.0.0' +__version__ = '7.0.1' import logging if __name__ == '__main__': diff --git a/docs/conf.py b/docs/conf.py index d971afe..5fe74ed 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,9 +61,9 @@ # built documents. # # The short X.Y version. -version = '7.0.0' +version = '7.0.1' # The full version, including alpha/beta/rc tags. -release = '7.0.0' +release = '7.0.1' load_dotenv(override=True) DOC_VERSION = os.environ.get('DOC_VERSION', version) diff --git a/docs/index.rst b/docs/index.rst index 7783448..4606c95 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ contain the root `toctree` directive. -DeepDiff 7.0.0 documentation! +DeepDiff 7.0.1 documentation! ============================= ******* diff --git a/setup.cfg b/setup.cfg index 518ad74..51dbd5d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 7.0.0 +current_version = 7.0.1 commit = True tag = True tag_name = {new_version} diff --git a/setup.py b/setup.py index dd90d57..42c8918 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ if os.environ.get('USER', '') == 'vagrant': del os.link -version = '7.0.0' +version = '7.0.1' def get_reqs(filename):