fix: deadlock in Heartbeat() (#3530)

* acquire and immediately release db_slice.GetSerializationMutex() in Heartbeat()

---------

Signed-off-by: kostas <[email protected]>

src/server/engine_shard.cc

@@ -633,8 +633,15 @@ void EngineShard::Heartbeat() {
 void EngineShard::RetireExpiredAndEvict() {
   // TODO: iterate over all namespaces
   DbSlice& db_slice = namespaces.GetDefaultNamespace().GetDbSlice(shard_id());
-  // Some of the functions below might acquire the lock again so we need to unlock it
-  std::unique_lock lk(db_slice.GetSerializationMutex());
+  // Some of the functions below might acquire the same lock again so we need to unlock it
+  // asap. We won't yield before we relock the mutex again, so the code below is atomic
+  // in respect to preemptions of big values. An example of that is the call to
+  // DeleteExpiredStep() below, which eventually calls ExpireIfNeeded()
+  // and within that the call to RecordExpiry() will trigger the registered
+  // callback OnJournalEntry which locks the exact same mutex.
+  // We need to lock below and immediately release because there should be no other fiber
+  // that is serializing a big value.
+  { std::unique_lock lk(db_slice.GetSerializationMutex()); }
   constexpr double kTtlDeleteLimit = 200;
   constexpr double kRedLimitFactor = 0.1;
 
@@ -676,8 +683,6 @@ void EngineShard::RetireExpiredAndEvict() {
     }
   }
 
-  // Because TriggerOnJournalWriteToSink will lock the same lock leading to a deadlock.
-  lk.unlock();
   // Journal entries for expired entries are not writen to socket in the loop above.
   // Trigger write to socket when loop finishes.
   if (auto journal = EngineShard::tlocal()->journal(); journal) {