Fix integer overflow in MallocFrameBuffer()

Promote integers to uint64_t to avoid integer overflow issue during
frame buffer allocation for very large screen sizes
pull/1/head
newsoft 10 years ago
parent 9453be4201
commit 045a044e8a

@ -82,9 +82,27 @@ static char* ReadPassword(rfbClient* client) {
#endif
}
static rfbBool MallocFrameBuffer(rfbClient* client) {
uint64_t allocSize;
if(client->frameBuffer)
free(client->frameBuffer);
client->frameBuffer=malloc(client->width*client->height*client->format.bitsPerPixel/8);
/* SECURITY: promote 'width' into uint64_t so that the multiplication does not overflow
'width' and 'height' are 16-bit integers per RFB protocol design
SIZE_MAX is the maximum value that can fit into size_t
*/
allocSize = (uint64_t)client->width * client->height * client->format.bitsPerPixel/8;
if (allocSize >= SIZE_MAX) {
rfbClientErr("CRITICAL: cannot allocate frameBuffer, requested size is too large\n");
return FALSE;
}
client->frameBuffer=malloc( (size_t)allocSize );
if (client->frameBuffer == NULL)
rfbClientErr("CRITICAL: frameBuffer allocation failed, requested size too large or not enough memory?\n");
return client->frameBuffer?TRUE:FALSE;
}

Loading…
Cancel
Save