Use htobeNN(3) to convert numbers in websocket.c.

byteswap.h exists only on glibc, so building libvncserver with websockets
support was not possible in other systems.

Replace the inclusion of byteswap.h and the WS_* definitions with calls to
htobeNN, which should perform the same conversions, be more portable and
avoid the need to check for the platform's endianness.

CMakeLists.txt

@@ -122,8 +122,10 @@ if(LIBGCRYPT_LIBRARIES)
 endif(LIBGCRYPT_LIBRARIES)
 
 
+check_include_file("endian.h"      LIBVNCSERVER_HAVE_ENDIAN_H)
 check_include_file("fcntl.h"       LIBVNCSERVER_HAVE_FCNTL_H)
 check_include_file("netinet/in.h"  LIBVNCSERVER_HAVE_NETINET_IN_H)
+check_include_file("sys/endian.h"  LIBVNCSERVER_HAVE_SYS_ENDIAN_H)
 check_include_file("sys/socket.h"  LIBVNCSERVER_HAVE_SYS_SOCKET_H)
 check_include_file("sys/stat.h"    LIBVNCSERVER_HAVE_SYS_STAT_H)
 check_include_file("sys/time.h"    LIBVNCSERVER_HAVE_SYS_TIME_H)

configure.ac

@@ -885,7 +885,7 @@ fi
 
 # Checks for header files.
 AC_HEADER_STDC
-AC_CHECK_HEADERS([arpa/inet.h fcntl.h netdb.h netinet/in.h stdlib.h string.h sys/socket.h sys/time.h sys/timeb.h syslog.h unistd.h ws2tcpip.h])
+AC_CHECK_HEADERS([arpa/inet.h endian.h fcntl.h netdb.h netinet/in.h stdlib.h string.h sys/endian.h sys/socket.h sys/time.h sys/timeb.h syslog.h unistd.h ws2tcpip.h])
 
 # x11vnc only:
 if test "$build_x11vnc" = "yes"; then

libvncserver/websockets.c

@@ -35,25 +35,22 @@
 /* errno */
 #include <errno.h>
 
-#include <byteswap.h>
+#ifdef LIBVNCSERVER_HAVE_ENDIAN_H
+#include <endian.h>
+#elif LIBVNCSERVER_HAVE_SYS_ENDIAN_H
+#include <sys/endian.h>
+#endif
+
 #include <string.h>
 #include "rfb/rfbconfig.h"
 #include "rfbssl.h"
 #include "rfbcrypto.h"
 
-#if defined(__BYTE_ORDER) && defined(__BIG_ENDIAN) && __BYTE_ORDER == __BIG_ENDIAN
-#define WS_NTOH64(n) (n)
-#define WS_NTOH32(n) (n)
-#define WS_NTOH16(n) (n)
-#define WS_HTON64(n) (n)
-#define WS_HTON16(n) (n)
-#else
-#define WS_NTOH64(n) bswap_64(n)
-#define WS_NTOH32(n) bswap_32(n)
-#define WS_NTOH16(n) bswap_16(n)
-#define WS_HTON64(n) bswap_64(n)
-#define WS_HTON16(n) bswap_16(n)
-#endif
+#define WS_NTOH64(n) htobe64(n)
+#define WS_NTOH32(n) htobe32(n)
+#define WS_NTOH16(n) htobe16(n)
+#define WS_HTON64(n) htobe64(n)
+#define WS_HTON16(n) htobe16(n)
 
 #define B64LEN(__x) (((__x + 2) / 3) * 12 / 3)
 #define WSHLENMAX 14  /* 2 + sizeof(uint64_t) + sizeof(uint32_t) */

rfb/rfbconfig.h.cmake

@@ -9,6 +9,9 @@
 /* work around when write() returns ENOENT but does not mean it */
 #cmakedefine LIBVNCSERVER_ENOENT_WORKAROUND 1
 
+/* Define to 1 if you have the <endian.h> header file. */
+#cmakedefine LIBVNCSERVER_HAVE_ENDIAN_H 1
+
 /* Define to 1 if you have the <fcntl.h> header file. */
 #cmakedefine LIBVNCSERVER_HAVE_FCNTL_H  1 
 
@@ -30,6 +33,9 @@
 /* Define to 1 if you have the <netinet/in.h> header file. */
 #cmakedefine LIBVNCSERVER_HAVE_NETINET_IN_H  1 
 
+/* Define to 1 if you have the <sys/endian.h> header file. */
+#cmakedefine LIBVNCSERVER_HAVE_SYS_ENDIAN_H 1
+
 /* Define to 1 if you have the <sys/socket.h> header file. */
 #cmakedefine LIBVNCSERVER_HAVE_SYS_SOCKET_H  1