Skip to content

Commit

Permalink
Include type info and change the "unknown" value for flat rows to
Browse files Browse the repository at this point in the history 
something that is friendly for Postgres enums
  • Loading branch information
@seperman
seperman committed Apr 8, 2024
1 parent 4c337cf commit 54ebdb5
Show file tree
Hide file tree
Showing 4 changed files with 93 additions and 62 deletions.
38 changes: 31 additions & 7 deletions deepdiff/delta.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -878,7 +878,7 @@ def to_dict(self):
return dict(self.diff)

@staticmethod
def _get_flat_row(action, info, _parse_path, keys_and_funcs):
def _get_flat_row(action, info, _parse_path, keys_and_funcs, report_type_changes=True):
for path, details in info.items():
row = {'path': _parse_path(path), 'action': action}
for key, new_key, func in keys_and_funcs:
Expand All @@ -887,6 +887,11 @@ def _get_flat_row(action, info, _parse_path, keys_and_funcs):
row[new_key] = func(details[key])
else:
row[new_key] = details[key]
if report_type_changes:
if 'value' in row and 'type' not in row:
row['type'] = type(row['value'])
if 'old_value' in row and 'old_type' not in row:
row['old_type'] = type(row['old_value'])
yield FlatDeltaRow(**row)

@staticmethod
Expand Down Expand Up @@ -1060,6 +1065,9 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) -
'iterable_items_removed_at_indexes': 'unordered_iterable_item_removed',
}
for action, info in self.diff.items():
if action == '_iterable_opcodes':
result.extend(self._flatten_iterable_opcodes())
continue

Check warning on line 1070 in deepdiff/delta.py

View check run for this annotation

Codecov / codecov/patch

deepdiff/delta.py#L1069-L1070 

Added lines #L1069 - L1070 were not covered by tests
if action.startswith('_'):
continue
if action in FLATTENING_NEW_ACTION_MAP:
Expand All @@ -1072,12 +1080,20 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) -
path2.append((index, 'GET'))
else:
path2.append(index)
result.append(FlatDeltaRow(path=path2, value=value, action=new_action))
if report_type_changes:
row = FlatDeltaRow(path=path2, value=value, action=new_action, type=type(value))
else:
row = FlatDeltaRow(path=path2, value=value, action=new_action)

Check warning on line 1086 in deepdiff/delta.py

View check run for this annotation

Codecov / codecov/patch

deepdiff/delta.py#L1086 

Added line #L1086 was not covered by tests
result.append(row)
elif action in {'set_item_added', 'set_item_removed'}:
for path, values in info.items():
path = _parse_path(path)
for value in values:
result.append(FlatDeltaRow(path=path, value=value, action=action))
if report_type_changes:
row = FlatDeltaRow(path=path, value=value, action=action, type=type(value))
else:
row = FlatDeltaRow(path=path, value=value, action=action)
result.append(row)
elif action == 'dictionary_item_added':
for path, value in info.items():
path = _parse_path(path)
Expand All @@ -1092,14 +1108,22 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) -
elif isinstance(value, set) and len(value) == 1:
value = value.pop()
action = 'set_item_added'
result.append(FlatDeltaRow(path=path, value=value, action=action))
if report_type_changes:
row = FlatDeltaRow(path=path, value=value, action=action, type=type(value))
else:
row = FlatDeltaRow(path=path, value=value, action=action)
result.append(row)
elif action in {
'dictionary_item_removed', 'iterable_item_added',
'iterable_item_removed', 'attribute_removed', 'attribute_added'
}:
for path, value in info.items():
path = _parse_path(path)
result.append(FlatDeltaRow(path=path, value=value, action=action))
if report_type_changes:
row = FlatDeltaRow(path=path, value=value, action=action, type=type(value))
else:
row = FlatDeltaRow(path=path, value=value, action=action)
result.append(row)
elif action == 'type_changes':
if not report_type_changes:
action = 'values_changed'
Expand All @@ -1109,16 +1133,16 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) -
info=info,
_parse_path=_parse_path,
keys_and_funcs=keys_and_funcs,
report_type_changes=report_type_changes,
):
result.append(row)
elif action == '_iterable_opcodes':
result.extend(self._flatten_iterable_opcodes())
else:
for row in self._get_flat_row(
action=action,
info=info,
_parse_path=_parse_path,
keys_and_funcs=keys_and_funcs,
report_type_changes=report_type_changes,
):
result.append(row)
return result
Expand Down
2 changes: 1 addition & 1 deletion deepdiff/helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -771,7 +771,7 @@ class FlatDataAction(str, enum.Enum):
unordered_iterable_item_removed = 'unordered_iterable_item_removed'


UnkownValueCode = '*-UNKNOWN-*'
UnkownValueCode = 'unknown___'


class FlatDeltaRow(NamedTuple):
Expand Down
28 changes: 14 additions & 14 deletions docs/serialization.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ Flat Row Specs:
unordered_iterable_item_removed = 'unordered_iterable_item_removed'


UnkownValueCode = '*-UNKNOWN-*'
UnkownValueCode = 'unknown___'


class FlatDeltaRow(NamedTuple):
Expand All @@ -205,7 +205,7 @@ Delta Serialize To Flat Dictionaries

Sometimes, it is desired to serialize a :ref:`delta_label` object to a list of flat dictionaries. For example, to store them in relation databases. In that case, you can use the Delta.to_flat_dicts to achieve the desired outcome.

Since None is a valid value, we use a special hard-coded string to signify "unkown": '*-UNKNOWN-*'
Since None is a valid value, we use a special hard-coded string to signify "unkown": 'unknown___'

.. note::
Many new keys are added to the flat dicts in DeepDiff 7.0.0
Expand All @@ -226,25 +226,25 @@ For example:
>>> pprint(flat_dicts, indent=2)
[ { 'action': 'dictionary_item_added',
'new_path': None,
'old_type': '*-UNKNOWN-*',
'old_value': '*-UNKNOWN-*',
'old_type': 'unknown___',
'old_value': 'unknown___',
'path': ['field2', 'key2'],
't1_from_index': None,
't1_to_index': None,
't2_from_index': None,
't2_to_index': None,
'type': '*-UNKNOWN-*',
'type': 'unknown___',
'value': 'value2'},
{ 'action': 'dictionary_item_removed',
'new_path': None,
'old_type': '*-UNKNOWN-*',
'old_value': '*-UNKNOWN-*',
'old_type': 'unknown___',
'old_value': 'unknown___',
'path': ['key1'],
't1_from_index': None,
't1_to_index': None,
't2_from_index': None,
't2_to_index': None,
'type': '*-UNKNOWN-*',
'type': 'unknown___',
'value': 'value1'}]


Expand All @@ -261,25 +261,25 @@ Example 2:
>>> pprint(flat_dicts, indent=2)
[ { 'action': 'iterable_item_added',
'new_path': None,
'old_type': '*-UNKNOWN-*',
'old_value': '*-UNKNOWN-*',
'old_type': 'unknown___',
'old_value': 'unknown___',
'path': [2],
't1_from_index': None,
't1_to_index': None,
't2_from_index': None,
't2_to_index': None,
'type': '*-UNKNOWN-*',
'type': 'unknown___',
'value': 'C'},
{ 'action': 'iterable_item_added',
'new_path': None,
'old_type': '*-UNKNOWN-*',
'old_value': '*-UNKNOWN-*',
'old_type': 'unknown___',
'old_value': 'unknown___',
'path': [3],
't1_from_index': None,
't1_to_index': None,
't2_from_index': None,
't2_to_index': None,
'type': '*-UNKNOWN-*',
'type': 'unknown___',
'value': 'D'}]


Expand Down
87 changes: 47 additions & 40 deletions tests/test_delta.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,8 @@ def test_list_difference_add_delta(self):

flat_result1 = delta.to_flat_rows()
flat_expected1 = [
FlatDeltaRow(path=[3], value=5, action='iterable_item_added'),
FlatDeltaRow(path=[2], value=3, action='iterable_item_added'),
FlatDeltaRow(path=[3], value=5, action='iterable_item_added', type=int),
FlatDeltaRow(path=[2], value=3, action='iterable_item_added', type=int),
]

assert flat_expected1 == flat_result1
Expand Down Expand Up @@ -291,9 +291,9 @@ def test_list_difference3_delta(self):

flat_result1 = delta.to_flat_rows()
flat_expected1 = [
FlatDeltaRow(path=[4, 'b', 2], action='values_changed', value=2, old_value=5),
FlatDeltaRow(path=[4, 'b', 1], action='values_changed', value=3, old_value=2),
FlatDeltaRow(path=[4, 'b', 3], value=5, action='iterable_item_added'),
FlatDeltaRow(path=[4, 'b', 2], action='values_changed', value=2, old_value=5, type=int, old_type=int),
FlatDeltaRow(path=[4, 'b', 1], action='values_changed', value=3, old_value=2, type=int, old_type=int),
FlatDeltaRow(path=[4, 'b', 3], value=5, action='iterable_item_added', type=int),
]

assert flat_expected1 == flat_result1
Expand Down Expand Up @@ -332,9 +332,9 @@ def test_list_difference_delta_raises_error_if_prev_value_does_not_match(self):

flat_result2 = delta2.to_flat_rows()
flat_expected2 = [
FlatDeltaRow(path=[2], action='values_changed', value=2, old_value=5),
FlatDeltaRow(path=[1], action='values_changed', value=3, old_value=2),
FlatDeltaRow(path=[3], value=5, action='iterable_item_added'),
FlatDeltaRow(path=[2], action='values_changed', value=2, old_value=5, type=int, old_type=int),
FlatDeltaRow(path=[1], action='values_changed', value=3, old_value=2, type=int, old_type=int),
FlatDeltaRow(path=[3], value=5, action='iterable_item_added', type=int),
]

assert flat_expected2 == flat_result2
Expand Down Expand Up @@ -363,8 +363,8 @@ def test_list_difference_delta1(self):

flat_result = delta.to_flat_rows()
flat_expected = [
FlatDeltaRow(path=[4, 'b', 2], value='to_be_removed', action='iterable_item_removed'),
FlatDeltaRow(path=[4, 'b', 3], value='to_be_removed2', action='iterable_item_removed'),
FlatDeltaRow(path=[4, 'b', 2], value='to_be_removed', action='iterable_item_removed', type=str),
FlatDeltaRow(path=[4, 'b', 3], value='to_be_removed2', action='iterable_item_removed', type=str),
]

assert flat_expected == flat_result
Expand Down Expand Up @@ -567,7 +567,8 @@ def compare_func(item1, item2, level=None):
'professionalDesignation': '',
'suffix': 'SR',
'nameIdentifier': '00003'},
action='unordered_iterable_item_added'),
action='unordered_iterable_item_added',
type=dict),
FlatDeltaRow(path=['individualNames', 1],
value={'firstName': 'John',
'lastName': 'Doe',
Expand All @@ -577,7 +578,9 @@ def compare_func(item1, item2, level=None):
'professionalDesignation': '',
'suffix': 'SR',
'nameIdentifier': '00002'},
action='unordered_iterable_item_removed')]
action='unordered_iterable_item_removed',
type=dict),
]

preserved_flat_dict_list = copy.deepcopy(flat_rows_list) # Use this later for assert comparison

Expand Down Expand Up @@ -1405,13 +1408,13 @@ def test_list_ignore_order_various_deltas2(self):

flat_result1 = delta1.to_flat_rows()
flat_expected1 = [
{'path': [0], 'value': 7, 'action': 'unordered_iterable_item_added'},
{'path': [6], 'value': 8, 'action': 'unordered_iterable_item_added'},
{'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added'},
{'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added'},
{'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added'},
{'path': [6], 'value': 6, 'action': 'unordered_iterable_item_removed'},
{'path': [0], 'value': 5, 'action': 'unordered_iterable_item_removed'},
{'path': [0], 'value': 7, 'action': 'unordered_iterable_item_added', 'type': int},
{'path': [6], 'value': 8, 'action': 'unordered_iterable_item_added', 'type': int},
{'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int},
{'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int},
{'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int},
{'path': [6], 'value': 6, 'action': 'unordered_iterable_item_removed', 'type': int},
{'path': [0], 'value': 5, 'action': 'unordered_iterable_item_removed', 'type': int},
]
flat_expected1 = [FlatDeltaRow(**i) for i in flat_expected1]
assert flat_expected1 == flat_result1
Expand All @@ -1422,11 +1425,11 @@ def test_list_ignore_order_various_deltas2(self):

flat_result2 = delta2.to_flat_rows()
flat_expected2 = [
{'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added'},
{'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added'},
{'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added'},
{'path': [6], 'action': 'values_changed', 'value': 7},
{'path': [0], 'action': 'values_changed', 'value': 8},
{'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int},
{'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int},
{'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int},
{'path': [6], 'action': 'values_changed', 'value': 7, 'type': int},
{'path': [0], 'action': 'values_changed', 'value': 8, 'type': int},
]
flat_expected2 = [FlatDeltaRow(**i) for i in flat_expected2]
assert flat_expected2 == flat_result2
Expand Down Expand Up @@ -1565,7 +1568,7 @@ def test_apply_delta_to_incompatible_object6_value_change(self):
assert [] == t4

flat_result2 = delta2.to_flat_rows()
flat_expected2 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5}]
flat_expected2 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5, 'type': int}]
flat_expected2 = [FlatDeltaRow(**i) for i in flat_expected2]

assert flat_expected2 == flat_result2
Expand All @@ -1575,7 +1578,7 @@ def test_apply_delta_to_incompatible_object6_value_change(self):

delta3 = Delta(diff, raise_errors=False, bidirectional=True)
flat_result3 = delta3.to_flat_rows()
flat_expected3 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5, 'old_value': 4}]
flat_expected3 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5, 'old_value': 4, 'type': int, 'old_type': int}]
flat_expected3 = [FlatDeltaRow(**i) for i in flat_expected3]

assert flat_expected3 == flat_result3
Expand Down Expand Up @@ -1685,7 +1688,7 @@ def test_delta_to_dict(self):
assert expected == result

flat_result = delta.to_flat_rows()
flat_expected = [{'action': 'unordered_iterable_item_removed', 'path': [2], 'value': 'B'}]
flat_expected = [{'action': 'unordered_iterable_item_removed', 'path': [2], 'value': 'B', 'type': str}]
flat_expected = [FlatDeltaRow(**i) for i in flat_expected]

assert flat_expected == flat_result
Expand Down Expand Up @@ -1766,10 +1769,10 @@ def test_delta_set_in_objects(self):
delta = Delta(DeepDiff(t1, t2))
flat_result = delta.to_flat_rows()
flat_expected = [
{'path': [0, 1], 'value': 10, 'action': 'set_item_added'},
{'path': [0, 0], 'action': 'values_changed', 'value': 2},
{'path': [0, 1], 'value': 'A', 'action': 'set_item_removed'},
{'path': [0, 1], 'value': 'C', 'action': 'set_item_added'},
{'path': [0, 1], 'value': 10, 'action': 'set_item_added', 'type': int},
{'path': [0, 0], 'action': 'values_changed', 'value': 2, 'type': int},
{'path': [0, 1], 'value': 'A', 'action': 'set_item_removed', 'type': str},
{'path': [0, 1], 'value': 'C', 'action': 'set_item_added', 'type': str},
]
flat_expected = [FlatDeltaRow(**i) for i in flat_expected]

Expand Down Expand Up @@ -1885,11 +1888,11 @@ def test_compare_func_with_duplicates_removed(self):

flat_result = delta.to_flat_rows()
flat_expected = [
{'path': [2], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed'},
{'path': [0], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed'},
{'path': [3], 'value': {'id': 3, 'val': 3}, 'action': 'iterable_item_removed'},
{'path': [0], 'action': 'iterable_item_moved', 'value': {'id': 1, 'val': 3}, 'new_path': [2]},
{'path': [3], 'action': 'iterable_item_moved', 'value': {'id': 3, 'val': 3}, 'new_path': [0]},
{'path': [2], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed', 'type': dict},
{'path': [0], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed', 'type': dict},
{'path': [3], 'value': {'id': 3, 'val': 3}, 'action': 'iterable_item_removed', 'type': dict},
{'path': [0], 'action': 'iterable_item_moved', 'value': {'id': 1, 'val': 3}, 'new_path': [2], 'type': dict},
{'path': [3], 'action': 'iterable_item_moved', 'value': {'id': 3, 'val': 3}, 'new_path': [0], 'type': dict},
]
flat_expected = [FlatDeltaRow(**i) for i in flat_expected]

Expand Down Expand Up @@ -2289,11 +2292,13 @@ def test_subtract_delta_made_from_flat_dicts1(self):
expected_flat_dicts = [{
'path': ['field_name1', 0],
'value': 'xxx',
'action': 'iterable_item_removed'
'action': 'iterable_item_removed',
'type': str,
}, {
'path': ['field_name1', 1],
'value': 'yyy',
'action': 'iterable_item_removed'
'action': 'iterable_item_removed',
'type': str,
}]
expected_flat_dicts = [FlatDeltaRow(**i) for i in expected_flat_dicts]

Expand All @@ -2318,11 +2323,13 @@ def test_subtract_delta_made_from_flat_dicts2(self):
expected_flat_dicts = [{
'path': ['field_name1', 0],
'value': 'xxx',
'action': 'iterable_item_added'
'action': 'iterable_item_added',
'type': str,
}, {
'path': ['field_name1', 1],
'value': 'yyy',
'action': 'iterable_item_added'
'action': 'iterable_item_added',
'type': str,
}]
expected_flat_dicts = [FlatDeltaRow(**i) for i in expected_flat_dicts]

Expand Down

0 comments on commit 54ebdb5

Please sign in to comment.