diff --git a/plugins/addressbook/evolution/eds.c b/plugins/addressbook/evolution/eds.c
index 345989fb46419fbf9dcee6b20398786e3bf1140d..82d088dd7d5f8984a57c4127d13cc8dd53d0e2b6 100644
--- a/plugins/addressbook/evolution/eds.c
+++ b/plugins/addressbook/evolution/eds.c
@@ -205,9 +205,9 @@ pixbuf_from_contact(EContact *contact)
             /* Force remaining parse operations to complete */
             gdk_pixbuf_loader_close(loader, NULL);
             pixbuf = gdk_pixbuf_loader_get_pixbuf(loader);
+            /* Ensure that pixbuf isn't destroyed with loader */
+            g_object_ref(pixbuf);
         }
-        /* Ensure that pixbuf isn't destroyed with loader */
-        g_object_ref(pixbuf);
         g_object_unref(loader);
     } else if (photo->type == E_CONTACT_PHOTO_TYPE_URI) {
         gchar *filename = g_filename_from_uri(photo->data.uri, NULL, NULL);