diff --git a/src/security/tls_session.cpp b/src/security/tls_session.cpp
index e2b959d403013282676eac4ebd08db318c6d0a86..3e6d79a30a8198adcb8edbed37b525c19080dbaf 100644
--- a/src/security/tls_session.cpp
+++ b/src/security/tls_session.cpp
@@ -1158,7 +1158,7 @@ TlsSession::TlsSessionImpl::handleStateHandshake(TlsSessionState state)
              and state_.load() != TlsSessionState::SHUTDOWN);
 
     // Stop on fatal error
-    if (gnutls_error_is_fatal(ret)) {
+    if (gnutls_error_is_fatal(ret) || state_.load() == TlsSessionState::SHUTDOWN) {
         JAMI_ERR("[TLS] handshake failed: %s", gnutls_strerror(ret));
         return TlsSessionState::SHUTDOWN;
     }