Fix heap overflows in the various rectangle fill functions

Altough rfbproto.c does check whether the overall FramebufferUpdate rectangle is
too large, some of the individual encoding decoders do not, which allows a
malicious server to overwrite parts of the heap.

libvncclient/rfbproto.c

@@ -147,6 +147,10 @@ void* rfbClientGetClientData(rfbClient* client, void* tag)
 
 /* messages */
 
+static boolean CheckRect(rfbClient* client, int x, int y, int w, int h) {
+  return x + w <= client->width && y + h <= client->height;
+}
+
 static void FillRectangle(rfbClient* client, int x, int y, int w, int h, uint32_t colour) {
   int i,j;
 
@@ -154,6 +158,11 @@ static void FillRectangle(rfbClient* client, int x, int y, int w, int h, uint32_
       return;
   }
 
+  if (!CheckRect(client, x, y, w, h)) {
+    rfbClientLog("Rect out of bounds: %dx%d at (%d, %d)\n", x, y, w, h);
+    return;
+  }
+
 #define FILL_RECT(BPP) \
     for(j=y*client->width;j<(y+h)*client->width;j+=client->width) \
       for(i=x;i<x+w;i++) \
@@ -175,6 +184,11 @@ static void CopyRectangle(rfbClient* client, uint8_t* buffer, int x, int y, int
       return;
   }
 
+  if (!CheckRect(client, x, y, w, h)) {
+    rfbClientLog("Rect out of bounds: %dx%d at (%d, %d)\n", x, y, w, h);
+    return;
+  }
+
 #define COPY_RECT(BPP) \
   { \
     int rs = w * BPP / 8, rs2 = client->width * BPP / 8; \
@@ -201,6 +215,16 @@ static void CopyRectangleFromRectangle(rfbClient* client, int src_x, int src_y,
       return;
   }
 
+  if (!CheckRect(client, src_x, src_y, w, h)) {
+    rfbClientLog("Source rect out of bounds: %dx%d at (%d, %d)\n", src_x, src_y, w, h);
+    return;
+  }
+
+  if (!CheckRect(client, dest_x, dest_y, w, h)) {
+    rfbClientLog("Dest rect out of bounds: %dx%d at (%d, %d)\n", dest_x, dest_y, w, h);
+    return;
+  }
+
 #define COPY_RECT_FROM_RECT(BPP) \
   { \
     uint##BPP##_t* _buffer=((uint##BPP##_t*)client->frameBuffer)+(src_y-dest_y)*client->width+src_x-dest_x; \