diff --git a/src/app/mainview/components/ConversationSmartListContextMenu.qml b/src/app/mainview/components/ConversationSmartListContextMenu.qml
index 10b85a0d28f994b3f716bee47cee13f691fd0e15..068cda26f6abfb20118a6570ebb243d8d48a6c25 100644
--- a/src/app/mainview/components/ConversationSmartListContextMenu.qml
+++ b/src/app/mainview/components/ConversationSmartListContextMenu.qml
@@ -91,7 +91,7 @@ ContextMenuAutoLoader {
                         "confirmLabel": JamiStrings.optionRemove
                     });
                 dlg.accepted.connect(function () {
-                        if (!isCoreDialog)
+                        if (mode !== Conversation.Mode.NON_SWARM)
                             MessagesAdapter.removeConversation(responsibleConvUid);
                         else
                             MessagesAdapter.removeContact(responsibleConvUid);
diff --git a/src/libclient/conversationmodel.cpp b/src/libclient/conversationmodel.cpp
index 34c588c14a4fbf2795a6cebba5b5fd02235e2ec1..d6c625704a3b217f8c4ccfe0b31e3d6fb2b9b5aa 100644
--- a/src/libclient/conversationmodel.cpp
+++ b/src/libclient/conversationmodel.cpp
@@ -832,12 +832,19 @@ ConversationModel::removeConversation(const QString& uid, bool banned)
         return;
     }
     if (conversation.isSwarm() && !banned && !conversation.isCoreDialog()) {
-        if (conversation.isRequest)
+        if (conversation.isRequest) {
             ConfigurationManager::instance().declineConversationRequest(owner.id, uid);
-        else
+        } else {
             ConfigurationManager::instance().removeConversation(owner.id, uid);
+        }
     } else {
-        owner.contactModel->removeContact(peers.front(), banned);
+        try {
+            auto& contact = owner.contactModel->getContact(peers.front());
+            owner.contactModel->removeContact(peers.front(), banned);
+        } catch (const std::out_of_range&) {
+            qWarning() << "Contact not found: " << peers.front();
+            ConfigurationManager::instance().removeConversation(owner.id, uid);
+        }
     }
 }