Cannot delete query with BinaryField field: Cannot pickle memoryview

[stuaxo]

Djangos BinaryField returns a memoryview when you retrieve it, this can't be pickled which causes .delete() to fail when it calls deepcopy -

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Input In [24], in <cell line: 12>()
---> 12 TrackedModelCheck.objects.all().delete()

File ~/.virtualenvs/dit-tamato/lib/python3.8/site-packages/django/db/models/query.py:746, in QuerySet.delete(self)
    743 del_query.query.clear_ordering(force_empty=True)
    745 collector = Collector(using=del_query.db)
--> 746 collector.collect(del_query)
    747 deleted, _rows_count = collector.delete()
    749 # Clear the result cache, in case this QuerySet gets reused.

File ~/.virtualenvs/dit-tamato/lib/python3.8/site-packages/django/db/models/deletion.py:313, in Collector.collect(self, objs, source, nullable, collect_related, source_attr, reverse_dependency, keep_parents, fail_on_restricted)
    311     batches = self.get_del_batches(new_objs, related_fields)
    312     for batch in batches:
--> 313         sub_objs = self.related_objects(related_model, related_fields, batch)
    314         self.fast_deletes.append(sub_objs)
    315 for field in model._meta.private_fields:

File ~/.virtualenvs/dit-tamato/lib/python3.8/site-packages/django/db/models/deletion.py:354, in Collector.related_objects(self, related_model, related_fields, objs)
    347 """
    348 Get a QuerySet of the related model to objs via related fields.
    349 """
    350 predicate = reduce(operator.or_, (
    351     query_utils.Q(**{'%s__in' % related_field.name: objs})
    352     for related_field in related_fields
    353 ))
--> 354 return related_model._base_manager.using(self.using).filter(predicate)

File ~/.virtualenvs/dit-tamato/lib/python3.8/site-packages/django/db/models/query.py:942, in QuerySet.filter(self, *args, **kwargs)
    937 """
    938 Return a new QuerySet instance with the args ANDed to the existing
    939 set.
    940 """
    941 self._not_support_combined_queries('filter')
--> 942 return self._filter_or_exclude(False, *args, **kwargs)

File ~/.virtualenvs/dit-tamato/lib/python3.8/site-packages/polymorphic/query.py:179, in PolymorphicQuerySet._filter_or_exclude(self, negate, *args, **kwargs)
    177 def _filter_or_exclude(self, negate, *args, **kwargs):
    178     # We override this internal Django function as it is used for all filter member functions.
--> 179     q_objects = translate_polymorphic_filter_definitions_in_args(
    180         self.model, args, using=self.db
    181     )
    182     # filter_field='data'
    183     additional_args = translate_polymorphic_filter_definitions_in_kwargs(
    184         self.model, kwargs, using=self.db
    185     )

File ~/.virtualenvs/dit-tamato/lib/python3.8/site-packages/polymorphic/query_translate.py:100, in translate_polymorphic_filter_definitions_in_args(queryset_model, args, using)
     87 def translate_polymorphic_filter_definitions_in_args(
     88     queryset_model, args, using=DEFAULT_DB_ALIAS
     89 ):
     90     """
     91     Translate the non-keyword argument list for PolymorphicQuerySet.filter()
     92 
   (...)
     98     Returns: modified Q objects
     99     """
--> 100     return [
    101         translate_polymorphic_Q_object(queryset_model, copy.deepcopy(q), using=using)
    102         for q in args
    103     ]

File ~/.virtualenvs/dit-tamato/lib/python3.8/site-packages/polymorphic/query_translate.py:101, in <listcomp>(.0)
     87 def translate_polymorphic_filter_definitions_in_args(
     88     queryset_model, args, using=DEFAULT_DB_ALIAS
     89 ):
     90     """
     91     Translate the non-keyword argument list for PolymorphicQuerySet.filter()
     92 
   (...)
     98     Returns: modified Q objects
     99     """
    100     return [
--> 101         translate_polymorphic_Q_object(queryset_model, copy.deepcopy(q), using=using)
    102         for q in args
    103     ]

File /usr/local/lib/python3.8/copy.py:153, in deepcopy(x, memo, _nil)
    151 copier = getattr(x, "__deepcopy__", None)
    152 if copier is not None:
--> 153     y = copier(memo)
    154 else:
    155     reductor = dispatch_table.get(cls)

File ~/.virtualenvs/dit-tamato/lib/python3.8/site-packages/django/utils/tree.py:53, in Node.__deepcopy__(self, memodict)
     51 obj = Node(connector=self.connector, negated=self.negated)
     52 obj.__class__ = self.__class__
---> 53 obj.children = copy.deepcopy(self.children, memodict)
     54 return obj

File /usr/local/lib/python3.8/copy.py:146, in deepcopy(x, memo, _nil)
    144 copier = _deepcopy_dispatch.get(cls)
    145 if copier is not None:
--> 146     y = copier(x, memo)
    147 else:
    148     if issubclass(cls, type):

File /usr/local/lib/python3.8/copy.py:205, in _deepcopy_list(x, memo, deepcopy)
    203 append = y.append
    204 for a in x:
--> 205     append(deepcopy(a, memo))
    206 return y

File /usr/local/lib/python3.8/copy.py:146, in deepcopy(x, memo, _nil)
    144 copier = _deepcopy_dispatch.get(cls)
    145 if copier is not None:
--> 146     y = copier(x, memo)
    147 else:
    148     if issubclass(cls, type):

File /usr/local/lib/python3.8/copy.py:210, in _deepcopy_tuple(x, memo, deepcopy)
    209 def _deepcopy_tuple(x, memo, deepcopy=deepcopy):
--> 210     y = [deepcopy(a, memo) for a in x]
    211     # We're not going to put the tuple in the memo, but it's still important we
    212     # check for it, in case the tuple contains recursive mutable structures.
    213     try:

File /usr/local/lib/python3.8/copy.py:210, in <listcomp>(.0)
    209 def _deepcopy_tuple(x, memo, deepcopy=deepcopy):
--> 210     y = [deepcopy(a, memo) for a in x]
    211     # We're not going to put the tuple in the memo, but it's still important we
    212     # check for it, in case the tuple contains recursive mutable structures.
    213     try:

File /usr/local/lib/python3.8/copy.py:146, in deepcopy(x, memo, _nil)
    144 copier = _deepcopy_dispatch.get(cls)
    145 if copier is not None:
--> 146     y = copier(x, memo)
    147 else:
    148     if issubclass(cls, type):

File /usr/local/lib/python3.8/copy.py:205, in _deepcopy_list(x, memo, deepcopy)
    203 append = y.append
    204 for a in x:
--> 205     append(deepcopy(a, memo))
    206 return y

File /usr/local/lib/python3.8/copy.py:172, in deepcopy(x, memo, _nil)
    170                 y = x
    171             else:
--> 172                 y = _reconstruct(x, memo, *rv)
    174 # If is its own copy, don't memoize.
    175 if y is not x:

File /usr/local/lib/python3.8/copy.py:270, in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    268 if state is not None:
    269     if deep:
--> 270         state = deepcopy(state, memo)
    271     if hasattr(y, '__setstate__'):
    272         y.__setstate__(state)

File /usr/local/lib/python3.8/copy.py:146, in deepcopy(x, memo, _nil)
    144 copier = _deepcopy_dispatch.get(cls)
    145 if copier is not None:
--> 146     y = copier(x, memo)
    147 else:
    148     if issubclass(cls, type):

File /usr/local/lib/python3.8/copy.py:230, in _deepcopy_dict(x, memo, deepcopy)
    228 memo[id(x)] = y
    229 for key, value in x.items():
--> 230     y[deepcopy(key, memo)] = deepcopy(value, memo)
    231 return y

File /usr/local/lib/python3.8/copy.py:161, in deepcopy(x, memo, _nil)
    159 reductor = getattr(x, "__reduce_ex__", None)
    160 if reductor is not None:
--> 161     rv = reductor(4)
    162 else:
    163     reductor = getattr(x, "__reduce__", None)

TypeError: cannot pickle 'memoryview' object

[stuaxo]

I think this may actually be a python error, since I guess deepcopy probably should work with memoryviews ?

[stuaxo]

To let me delete django polymorphic items I have employed the workaround of updating any fields that are bytefields to None before deleting them.

# `TrackedModelCheck` is my polymorphic model, and `content_hash` is my bytefield
models = TrackedModelCheck.objects.filter(model__transaction__workbasket__pk=workbasket_pk)
models.update(content_hash=None)
models.delete()