From 492116d535173eeb7c8008f6687cd82ee60165f4 Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Wed, 14 Dec 2016 23:55:13 -0800 Subject: [PATCH] fixes for rle bitmap compress limits and raw bitmaps --- libxrdp/xrdp_bitmap32_compress.c | 122 ++++++++++++++++++------------- libxrdp/xrdp_bitmap_compress.c | 8 +- libxrdp/xrdp_orders.c | 115 +++++++++++++++++++---------- 3 files changed, 153 insertions(+), 92 deletions(-) diff --git a/libxrdp/xrdp_bitmap32_compress.c b/libxrdp/xrdp_bitmap32_compress.c index 083c4409..daec9f28 100644 --- a/libxrdp/xrdp_bitmap32_compress.c +++ b/libxrdp/xrdp_bitmap32_compress.c @@ -107,7 +107,7 @@ fsplit3(char *in_data, int start_line, int width, int e, } start_line--; cy++; - if (out_index > 64 * 64) + if (out_index + width + e > 64 * 64) { break; } @@ -195,7 +195,7 @@ fsplit4(char *in_data, int start_line, int width, int e, } start_line--; cy++; - if (out_index > 64 * 64) + if (out_index + width + e > 64 * 64) { break; } @@ -422,6 +422,7 @@ xrdp_bitmap32_compress(char *in_data, int width, int height, char *sr_data; char *sg_data; char *sb_data; + char *hold_p; int a_bytes; int r_bytes; int g_bytes; @@ -449,6 +450,7 @@ xrdp_bitmap32_compress(char *in_data, int width, int height, r_data = a_data + max_bytes; g_data = r_data + max_bytes; b_data = g_data + max_bytes; + hold_p = s->p; if (header & FLAGS_NOALPHA) { @@ -459,35 +461,44 @@ xrdp_bitmap32_compress(char *in_data, int width, int height, fdelta(sr_data, r_data, cx, cy); fdelta(sg_data, g_data, cx, cy); fdelta(sb_data, b_data, cx, cy); - out_uint8(s, header); - r_bytes = fpack(r_data, cx, cy, s); - g_bytes = fpack(g_data, cx, cy, s); - b_bytes = fpack(b_data, cx, cy, s); - total_bytes = r_bytes + g_bytes + b_bytes; - if (1 + total_bytes > byte_limit) + while (cy > 0) { - /* failed */ - LLOGLN(0, ("xrdp_bitmap32_compress: too big, rgb " - "bytes %d %d %d total_bytes %d cx %d cy %d " - "byte_limit %d", r_bytes, g_bytes, b_bytes, - total_bytes, cx, cy, byte_limit)); - return 0; - } - max_bytes = cx * cy * 3; - if (total_bytes > max_bytes) - { - /* raw is better */ - LLOGLN(10, ("xrdp_bitmap32_compress: too big, rgb " - "bytes %d %d %d total_bytes %d cx %d cy %d " - "max_bytes %d", r_bytes, g_bytes, b_bytes, - total_bytes, cx, cy, max_bytes)); - init_stream(s, 0); - foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data); + s->p = hold_p; + out_uint8(s, header); + r_bytes = fpack(r_data, cx, cy, s); + g_bytes = fpack(g_data, cx, cy, s); + b_bytes = fpack(b_data, cx, cy, s); + max_bytes = cx * cy * 3; + total_bytes = r_bytes + g_bytes + b_bytes; + if (total_bytes > max_bytes) + { + if (2 + max_bytes <= byte_limit) + { + s->p = hold_p; + foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data); + break; + } + } + if (1 + total_bytes <= byte_limit) + { + break; + } + cy--; } } else { - foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data); + while (cy > 0) + { + max_bytes = cx * cy * 3; + if (2 + max_bytes <= byte_limit) + { + s->p = hold_p; + foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data); + break; + } + cy--; + } } } else @@ -500,36 +511,45 @@ xrdp_bitmap32_compress(char *in_data, int width, int height, fdelta(sr_data, r_data, cx, cy); fdelta(sg_data, g_data, cx, cy); fdelta(sb_data, b_data, cx, cy); - out_uint8(s, header); - a_bytes = fpack(a_data, cx, cy, s); - r_bytes = fpack(r_data, cx, cy, s); - g_bytes = fpack(g_data, cx, cy, s); - b_bytes = fpack(b_data, cx, cy, s); - max_bytes = cx * cy * 4; - total_bytes = a_bytes + r_bytes + g_bytes + b_bytes; - if (1 + total_bytes > byte_limit) + while (cy > 0) { - /* failed */ - LLOGLN(0, ("xrdp_bitmap32_compress: too big, argb " - "bytes %d %d %d %d total_bytes %d cx %d cy %d " - "byte_limit %d", a_bytes, r_bytes, g_bytes, b_bytes, - total_bytes, cx, cy, byte_limit)); - return 0; - } - if (total_bytes > max_bytes) - { - /* raw is better */ - LLOGLN(10, ("xrdp_bitmap32_compress: too big, argb " - "bytes %d %d %d %d total_bytes %d cx %d cy %d " - "max_bytes %d", a_bytes, r_bytes, g_bytes, b_bytes, - total_bytes, cx, cy, max_bytes)); - init_stream(s, 0); - foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data); + s->p = hold_p; + out_uint8(s, header); + a_bytes = fpack(a_data, cx, cy, s); + r_bytes = fpack(r_data, cx, cy, s); + g_bytes = fpack(g_data, cx, cy, s); + b_bytes = fpack(b_data, cx, cy, s); + max_bytes = cx * cy * 4; + total_bytes = a_bytes + r_bytes + g_bytes + b_bytes; + if (total_bytes > max_bytes) + { + if (2 + max_bytes <= byte_limit) + { + s->p = hold_p; + foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data); + break; + } + } + if (1 + total_bytes <= byte_limit) + { + break; + } + cy--; } } else { - foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data); + while (cy > 0) + { + max_bytes = cx * cy * 4; + if (2 + max_bytes <= byte_limit) + { + s->p = hold_p; + foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data); + break; + } + cy--; + } } } return cy; diff --git a/libxrdp/xrdp_bitmap_compress.c b/libxrdp/xrdp_bitmap_compress.c index 03c56f10..56898776 100644 --- a/libxrdp/xrdp_bitmap_compress.c +++ b/libxrdp/xrdp_bitmap_compress.c @@ -22,6 +22,8 @@ #include "libxrdp.h" +#define BC_MAX_BYTES (16 * 1024) + /*****************************************************************************/ #define IN_PIXEL8(in_ptr, in_x, in_y, in_w, in_last_pixel, in_pixel); \ do { \ @@ -695,7 +697,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height, out_count = end; line = in_data + width * start_line; - while (start_line >= 0 && out_count < 32768) + while (start_line >= 0 && out_count <= BC_MAX_BYTES) { i = (s->p - s->data) + count; @@ -987,7 +989,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height, out_count = end * 2; line = in_data + width * start_line * 2; - while (start_line >= 0 && out_count < 32768) + while (start_line >= 0 && out_count <= BC_MAX_BYTES) { i = (s->p - s->data) + count * 2; @@ -1279,7 +1281,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height, out_count = end * 3; line = in_data + width * start_line * 4; - while (start_line >= 0 && out_count < 32768) + while (start_line >= 0 && out_count <= BC_MAX_BYTES) { i = (s->p - s->data) + count * 3; diff --git a/libxrdp/xrdp_orders.c b/libxrdp/xrdp_orders.c index 18a5ad38..b0c28ed8 100644 --- a/libxrdp/xrdp_orders.c +++ b/libxrdp/xrdp_orders.c @@ -2235,6 +2235,11 @@ xrdp_orders_send_raw_bitmap(struct xrdp_orders *self, Bpp = (bpp + 7) / 8; bufsize = (width + e) * height * Bpp; + while (bufsize + 16 > MAX_ORDERS_SIZE) + { + height--; + bufsize = (width + e) * height * Bpp; + } if (xrdp_orders_check(self, bufsize + 16) != 0) { return 1; @@ -2254,33 +2259,58 @@ xrdp_orders_send_raw_bitmap(struct xrdp_orders *self, out_uint16_le(self->out_s, bufsize); out_uint16_le(self->out_s, cache_idx); - for (i = height - 1; i >= 0; i--) + if (Bpp == 4) { - for (j = 0; j < width; j++) + for (i = height - 1; i >= 0; i--) { - if (Bpp == 3) + for (j = 0; j < width; j++) { pixel = GETPIXEL32(data, j, i, width); out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel >> 8); out_uint8(self->out_s, pixel >> 16); + out_uint8(self->out_s, pixel >> 24); } - else if (Bpp == 2) + out_uint8s(self->out_s, Bpp * e); + } + } + else if (Bpp == 3) + { + for (i = height - 1; i >= 0; i--) + { + for (j = 0; j < width; j++) { - pixel = GETPIXEL16(data, j, i, width); + pixel = GETPIXEL32(data, j, i, width); out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel >> 8); + out_uint8(self->out_s, pixel >> 16); } - else if (Bpp == 1) + out_uint8s(self->out_s, Bpp * e); + } + } + else if (Bpp == 2) + { + for (i = height - 1; i >= 0; i--) + { + for (j = 0; j < width; j++) { - pixel = GETPIXEL8(data, j, i, width); + pixel = GETPIXEL16(data, j, i, width); out_uint8(self->out_s, pixel); + out_uint8(self->out_s, pixel >> 8); } + out_uint8s(self->out_s, Bpp * e); } - - for (j = 0; j < e; j++) + } + else if (Bpp == 1) + { + for (i = height - 1; i >= 0; i--) { - out_uint8s(self->out_s, Bpp); + for (j = 0; j < width; j++) + { + pixel = GETPIXEL8(data, j, i, width); + out_uint8(self->out_s, pixel); + } + out_uint8s(self->out_s, Bpp * e); } } @@ -2334,20 +2364,19 @@ xrdp_orders_send_bitmap(struct xrdp_orders *self, if (bpp > 24) { lines_sending = xrdp_bitmap32_compress(data, width, height, s, - bpp, 16384, + bpp, MAX_ORDERS_SIZE, i - 1, temp_s, e, 0x10); } else { - lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384, + lines_sending = xrdp_bitmap_compress(data, width, height, s, + bpp, MAX_ORDERS_SIZE, i - 1, temp_s, e); } if (lines_sending != height) { - g_writeln("error in xrdp_orders_send_bitmap, lines_sending(%d) != \ -height(%d)", lines_sending, height); - return 1; + height = lines_sending; } bufsize = (int)(s->p - p); @@ -2458,6 +2487,7 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self, int pixel = 0; int e = 0; + g_writeln("xrdp_orders_send_raw_bitmap2:"); if (width > 64) { g_writeln("error, width > 64"); @@ -2479,6 +2509,11 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self, Bpp = (bpp + 7) / 8; bufsize = (width + e) * height * Bpp; + while (bufsize + 14 > MAX_ORDERS_SIZE) + { + height--; + bufsize = (width + e) * height * Bpp; + } if (xrdp_orders_check(self, bufsize + 14) != 0) { return 1; @@ -2499,7 +2534,7 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self, i = cache_idx & 0xff; out_uint8(self->out_s, i); - if (1 && Bpp == 3) + if (Bpp == 4) { for (i = height - 1; i >= 0; i--) { @@ -2509,44 +2544,49 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self, out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel >> 8); out_uint8(self->out_s, pixel >> 16); + out_uint8(self->out_s, pixel >> 24); } - for (j = 0; j < e; j++) - { - out_uint8s(self->out_s, Bpp); - } + out_uint8s(self->out_s, Bpp * e); } } - else - { - for (i = height - 1; i >= 0; i--) + else if (Bpp == 3) { - for (j = 0; j < width; j++) + for (i = height - 1; i >= 0; i--) { - if (Bpp == 3) + for (j = 0; j < width; j++) { pixel = GETPIXEL32(data, j, i, width); out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel >> 8); out_uint8(self->out_s, pixel >> 16); } - else if (Bpp == 2) + out_uint8s(self->out_s, Bpp * e); + } + } + else if (Bpp == 2) + { + for (i = height - 1; i >= 0; i--) + { + for (j = 0; j < width; j++) { pixel = GETPIXEL16(data, j, i, width); out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel >> 8); } - else if (Bpp == 1) + out_uint8s(self->out_s, Bpp * e); + } + } + else if (Bpp == 1) + { + for (i = height - 1; i >= 0; i--) + { + for (j = 0; j < width; j++) { pixel = GETPIXEL8(data, j, i, width); out_uint8(self->out_s, pixel); } + out_uint8s(self->out_s, Bpp * e); } - - for (j = 0; j < e; j++) - { - out_uint8s(self->out_s, Bpp); - } - } } return 0; @@ -2599,20 +2639,19 @@ xrdp_orders_send_bitmap2(struct xrdp_orders *self, if (bpp > 24) { lines_sending = xrdp_bitmap32_compress(data, width, height, s, - bpp, 16384, + bpp, MAX_ORDERS_SIZE, i - 1, temp_s, e, 0x10); } else { - lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384, + lines_sending = xrdp_bitmap_compress(data, width, height, s, + bpp, MAX_ORDERS_SIZE, i - 1, temp_s, e); } if (lines_sending != height) { - g_writeln("error in xrdp_orders_send_bitmap2, lines_sending(%d) != \ -height(%d)", lines_sending, height); - return 1; + height = lines_sending; } bufsize = (int)(s->p - p);