diff --git a/src/contactmodel.cpp b/src/contactmodel.cpp
index 50fd2baa21cc942113cb79f69d177aa8ccde5228..1dee9092b80c48f1adbac50f48ebaf8c8e93d857 100644
--- a/src/contactmodel.cpp
+++ b/src/contactmodel.cpp
@@ -891,13 +891,12 @@ ContactModelPimpl::slotContactRemoved(const QString& accountId,
         }
     }
 
+    // Update the smartlist
+    linked.owner.conversationModel->refreshFilter();
     if (banned) {
-        // Update the smartlist
-        linked.owner.conversationModel->refreshFilter();
         emit linked.bannedStatusChanged(contactUri, true);
-    } else {
-        emit linked.contactRemoved(contactUri);
     }
+    emit linked.contactRemoved(contactUri);
 }
 
 void