diff --git a/CMakeLists.txt b/CMakeLists.txt
index b9f3044..8097158 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -224,20 +224,6 @@ endif(LIBVNCSERVER_HAVE_SYS_UIO_H)
 
 if(WITH_WEBSOCKETS AND LIBVNCSERVER_HAVE_CRYPTO)
   set(LIBVNCSERVER_WITH_WEBSOCKETS 1)
-  if(GNUTLS_FOUND)
-    message(STATUS "Building websockets with GnuTLS")
-    set(WEBSOCKET_LIBRARIES ${GNUTLS_LIBRARIES})
-    set(WSSRCS ${LIBVNCSERVER_DIR}/rfbssl_gnutls)
-    include_directories(${GNUTLS_INCLUDE_DIR})
-  elseif(OPENSSL_FOUND)
-    message(STATUS "Building websockets with OpenSSL")
-    set(WEBSOCKET_LIBRARIES ${OPENSSL_LIBRARIES})
-    set(WSSRCS ${LIBVNCSERVER_DIR}/rfbssl_openssl)
-  else()
-    message(STATUS "Building websockets without SSL")
-    set(WEBSOCKET_LIBRARIES)
-    set(WSSRCS ${LIBVNCSERVER_DIR}/rfbssl_none.c)
-  endif()
 endif(WITH_WEBSOCKETS AND LIBVNCSERVER_HAVE_CRYPTO)
 
 if(WITH_GCRYPT AND LIBGCRYPT_LIBRARIES)
@@ -351,17 +337,30 @@ if(GNUTLS_FOUND)
     ${LIBVNCCLIENT_SOURCES}
     ${LIBVNCCLIENT_DIR}/tls_gnutls.c
   )
+  set(LIBVNCSERVER_SOURCES
+    ${LIBVNCSERVER_SOURCES}
+    ${LIBVNCSERVER_DIR}/rfbssl_gnutls.c
+    )
+  include_directories(${GNUTLS_INCLUDE_DIR})
 elseif(OPENSSL_FOUND)
   set(LIBVNCCLIENT_SOURCES
     ${LIBVNCCLIENT_SOURCES}
     ${LIBVNCCLIENT_DIR}/tls_openssl.c
   )
+  set(LIBVNCSERVER_SOURCES
+    ${LIBVNCSERVER_SOURCES}
+    ${LIBVNCSERVER_DIR}/rfbssl_openssl.c
+  )
   include_directories(${OPENSSL_INCLUDE_DIR})
 else()
   set(LIBVNCCLIENT_SOURCES
     ${LIBVNCCLIENT_SOURCES}
     ${LIBVNCCLIENT_DIR}/tls_none.c
   )
+  set(LIBVNCSERVER_SOURCES
+    ${LIBVNCSERVER_SOURCES}
+    ${LIBVNCSERVER_DIR}/rfbssl_none.c
+  )
 endif()
 
 if(LIBVNCSERVER_HAVE_SASL)
@@ -417,7 +416,6 @@ if(LIBVNCSERVER_WITH_WEBSOCKETS)
     ${LIBVNCSERVER_DIR}/ws_decode.c
     ${COMMON_DIR}/base64.c
     ${CRYPTO_SOURCES}
-    ${WSSRCS}
   )
 endif(LIBVNCSERVER_WITH_WEBSOCKETS)
 
@@ -440,7 +438,8 @@ target_link_libraries(vncserver
                       ${JPEG_LIBRARIES}
 		      ${PNG_LIBRARIES}
 		      ${CRYPTO_LIBRARIES}
-		      ${WEBSOCKET_LIBRARIES}
+                      ${GNUTLS_LIBRARIES}
+                      ${OPENSSL_LIBRARIES}
 )
 
 SET_TARGET_PROPERTIES(vncclient vncserver
diff --git a/rfb/rfb.h b/rfb/rfb.h
index 6401e78..3d6d31e 100644
--- a/rfb/rfb.h
+++ b/rfb/rfb.h
@@ -378,10 +378,8 @@ typedef struct _rfbScreenInfo
     rfbDisplayFinishedHookPtr displayFinishedHook;
     /** xvpHook is called to handle an xvp client message */
     rfbXvpHookPtr xvpHook;
-#ifdef LIBVNCSERVER_WITH_WEBSOCKETS
     char *sslkeyfile;
     char *sslcertfile;
-#endif
     int ipv6port; /**< The port to listen on when using IPv6.  */
     char* listen6Interface;
     /* We have an additional IPv6 listen socket since there are systems that
@@ -690,12 +688,9 @@ typedef struct _rfbClientRec {
     int turboQualityLevel;  /* 1-100 scale */
 #endif
 #endif
-
-#ifdef LIBVNCSERVER_WITH_WEBSOCKETS
     rfbSslCtx *sslctx;
     wsCtx     *wsctx;
     char *wspath;                          /* Requests path component */
-#endif
 } rfbClientRec, *rfbClientPtr;
 
 /**