contactmodel: make refreshFilter and emit contactRemoved general

Change-Id: I9679df3bda97d4401ea137088d006820154d141b

src/contactmodel.cpp

@@ -891,13 +891,12 @@ ContactModelPimpl::slotContactRemoved(const QString& accountId,
         }
     }
 
+    // Update the smartlist
+    linked.owner.conversationModel->refreshFilter();
     if (banned) {
-        // Update the smartlist
-        linked.owner.conversationModel->refreshFilter();
         emit linked.bannedStatusChanged(contactUri, true);
-    } else {
-        emit linked.contactRemoved(contactUri);
     }
+    emit linked.contactRemoved(contactUri);
 }
 
 void