diff --git a/src/conversationmodel.cpp b/src/conversationmodel.cpp
index ff92720250fb2dfb6e9656a654d89e3b162cb08e..4e93f33cb1feae2cb86e257e508d3f44b165174d 100644
--- a/src/conversationmodel.cpp
+++ b/src/conversationmodel.cpp
@@ -1949,7 +1949,7 @@ void
 ConversationModelPimpl::slotCallAddedToConference(const QString& callId, const QString& confId)
 {
     for (auto& conversation: conversations) {
-        if (conversation.callId == callId || conversation.confId == confId) {
+        if (conversation.callId == callId && conversation.confId != confId) {
             conversation.confId = confId;
             dirtyConversations = {true, true};