Merge pull request #84 from plettix/master

fix for issue 81

common/md5.c

@@ -46,7 +46,7 @@
 
 #ifdef WORDS_BIGENDIAN
 # define SWAP(n)                                                        \
-    (((n) << 24) | (((n) & 0xff00) << 8) | (((n) >> 8) & 0xff00) | ((n) >> 24))
+    ((((n) & 0x00ff) << 24) | (((n) & 0xff00) << 8) | (((n) >> 8) & 0xff00) | (((n) >> 24) & 0x00ff))
 #else
 # define SWAP(n) (n)
 #endif

libvncserver/websockets.c

@@ -79,8 +79,9 @@ typedef int (*wsEncodeFunc)(rfbClientPtr cl, const char *src, int len, char **ds
 typedef int (*wsDecodeFunc)(rfbClientPtr cl, char *dst, int len);
 
 typedef struct ws_ctx_s {
-    char codeBuf[B64LEN(UPDATE_BUF_SIZE) + WSHLENMAX]; /* base64 + maximum frame header length */
-    char readbuf[8192];
+    char codeBufDecode[B64LEN(UPDATE_BUF_SIZE) + WSHLENMAX]; /* base64 + maximum frame header length */
+	char codeBufEncode[B64LEN(UPDATE_BUF_SIZE) + WSHLENMAX]; /* base64 + maximum frame header length */
+	char readbuf[8192];
     int readbufstart;
     int readbuflen;
     int dblen;
@@ -490,15 +491,15 @@ webSocketsEncodeHixie(rfbClientPtr cl, const char *src, int len, char **dst)
     int sz = 0;
     ws_ctx_t *wsctx = (ws_ctx_t *)cl->wsctx;
 
-    wsctx->codeBuf[sz++] = '\x00';
-    len = __b64_ntop((unsigned char *)src, len, wsctx->codeBuf+sz, sizeof(wsctx->codeBuf) - (sz + 1));
+    wsctx->codeBufEncode[sz++] = '\x00';
+    len = __b64_ntop((unsigned char *)src, len, wsctx->codeBufEncode+sz, sizeof(wsctx->codeBufEncode) - (sz + 1));
     if (len < 0) {
         return len;
     }
     sz += len;
 
-    wsctx->codeBuf[sz++] = '\xff';
-    *dst = wsctx->codeBuf;
+    wsctx->codeBufEncode[sz++] = '\xff';
+    *dst = wsctx->codeBufEncode;
     return sz;
 }
 
@@ -536,7 +537,7 @@ webSocketsDecodeHixie(rfbClientPtr cl, char *dst, int len)
     char *buf, *end = NULL;
     ws_ctx_t *wsctx = (ws_ctx_t *)cl->wsctx;
 
-    buf = wsctx->codeBuf;
+    buf = wsctx->codeBufDecode;
 
     n = ws_peek(cl, buf, len*2+2);
 
@@ -657,8 +658,8 @@ webSocketsDecodeHybi(rfbClientPtr cl, char *dst, int len)
       goto spor;
     }
 
-    buf = wsctx->codeBuf;
-    header = (ws_header_t *)wsctx->codeBuf;
+    buf = wsctx->codeBufDecode;
+    header = (ws_header_t *)wsctx->codeBufDecode;
 
     ret = ws_peek(cl, buf, B64LEN(len) + WSHLENMAX);
 
@@ -742,11 +743,11 @@ webSocketsDecodeHybi(rfbClientPtr cl, char *dst, int len)
 	errno = ECONNRESET;
 	break;
       case WS_OPCODE_TEXT_FRAME:
-	if (-1 == (flength = __b64_pton(payload, (unsigned char *)wsctx->codeBuf, sizeof(wsctx->codeBuf)))) {
+	if (-1 == (flength = __b64_pton(payload, (unsigned char *)wsctx->codeBufDecode, sizeof(wsctx->codeBufDecode)))) {
 	  rfbErr("%s: Base64 decode error; %m\n", __func__);
 	  break;
 	}
-	payload = wsctx->codeBuf;
+	payload = wsctx->codeBufDecode;
 	/* fall through */
       case WS_OPCODE_BINARY_FRAME:
 	if (flength > len) {
@@ -790,7 +791,7 @@ webSocketsEncodeHybi(rfbClientPtr cl, const char *src, int len, char **dst)
 	  return 0;
     }
 
-    header = (ws_header_t *)wsctx->codeBuf;
+    header = (ws_header_t *)wsctx->codeBufEncode;
 
     if (wsctx->base64) {
 	opcode = WS_OPCODE_TEXT_FRAME;
@@ -816,7 +817,7 @@ webSocketsEncodeHybi(rfbClientPtr cl, const char *src, int len, char **dst)
     }
 
     if (wsctx->base64) {
-        if (-1 == (ret = __b64_ntop((unsigned char *)src, len, wsctx->codeBuf + sz, sizeof(wsctx->codeBuf) - sz))) {
+        if (-1 == (ret = __b64_ntop((unsigned char *)src, len, wsctx->codeBufEncode + sz, sizeof(wsctx->codeBufEncode) - sz))) {
 	  rfbErr("%s: Base 64 encode failed\n", __func__);
 	} else {
 	  if (ret != blen)
@@ -824,11 +825,12 @@ webSocketsEncodeHybi(rfbClientPtr cl, const char *src, int len, char **dst)
 	  ret += sz;
 	}
     } else {
-      memcpy(wsctx->codeBuf + sz, src, len);
+      memcpy(wsctx->codeBufEncode + sz, src, len);
       ret =  sz + len;
     }
 
-    *dst = wsctx->codeBuf;
+    *dst = wsctx->codeBufEncode;
+
     return ret;
 }
 

rfb/rfb.h

@@ -720,10 +720,10 @@ typedef struct _rfbClientRec {
 #define Swap24(l) ((((l) & 0xff) << 16) | (((l) >> 16) & 0xff) | \
                    (((l) & 0x00ff00)))
 
-#define Swap32(l) (((l) >> 24) | \
+#define Swap32(l) ((((l) >> 24) & 0x000000ff)| \
                    (((l) & 0x00ff0000) >> 8)  | \
                    (((l) & 0x0000ff00) << 8)  | \
-                   ((l) << 24))
+                   (((l) & 0x000000ff) << 24))
 
 
 extern char rfbEndianTest;

rfb/rfbclient.h

@@ -47,13 +47,13 @@
     (*(char *)&client->endianTest ? ((((s) & 0xff) << 8) | (((s) >> 8) & 0xff)) : (s))
 
 #define rfbClientSwap32IfLE(l) \
-    (*(char *)&client->endianTest ? ((((l) & 0xff000000) >> 24) | \
+    (*(char *)&client->endianTest ? ((((l) >> 24) & 0x000000ff) | \
 			     (((l) & 0x00ff0000) >> 8)  | \
 			     (((l) & 0x0000ff00) << 8)  | \
 			     (((l) & 0x000000ff) << 24))  : (l))
 
 #define rfbClientSwap64IfLE(l) \
-    (*(char *)&client->endianTest ? ((((l) & 0xff00000000000000ULL) >> 56) | \
+    (*(char *)&client->endianTest ? ((((l) >> 56 ) & 0xff00000000000000ULL) | \
 			     (((l) & 0x00ff000000000000ULL) >> 40)  | \
 			     (((l) & 0x0000ff0000000000ULL) >> 24)  | \
 			     (((l) & 0x000000ff00000000ULL) >> 8)  | \