From 79796b77b774fc07edae3663e3419f26492f7c85 Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Sat, 18 Feb 2017 21:35:04 -0800 Subject: [PATCH] changes and cleanup to process_enc_rfx --- xrdp/xrdp_encoder.c | 117 +++++++++++++++++++++++++------------------- xrdp/xrdp_encoder.h | 1 + 2 files changed, 67 insertions(+), 51 deletions(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index ae6a1b35..e6cfec27 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -39,6 +39,8 @@ } \ while (0) +#define XRDP_SURCMD_PREFIX_BYTES 256 + /*****************************************************************************/ static int process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc); @@ -129,6 +131,7 @@ xrdp_encoder_create(struct xrdp_mm *mm) self->xrdp_encoder_event_processed = g_create_wait_obj(buf); g_snprintf(buf, 1024, "xrdp_%8.8x_encoder_term", pid); self->xrdp_encoder_term = g_create_wait_obj(buf); + self->max_compressed_bytes = client_info->max_fastpath_frag_bytes & ~15; /* create thread to process messages */ tc_thread_create(proc_enc_msg, self); @@ -312,6 +315,7 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) tbus event_processed; struct rfx_tile *tiles; struct rfx_rect *rfxrects; + int alloc_bytes; LLOGLN(10, ("process_enc_rfx:")); LLOGLN(10, ("process_enc_rfx: num_crects %d num_drects %d", @@ -320,64 +324,75 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) mutex = self->mutex; event_processed = self->xrdp_encoder_event_processed; - if ((enc->num_crects > 4096) || (enc->num_drects > 4096)) - { - return 0; - } + error = 1; + out_data = NULL; + out_data_bytes = 0; - out_data_bytes = 16 * 1024 * 1024; - index = 256 + sizeof(struct rfx_tile) * 4096 + - sizeof(struct rfx_rect) * 4096; - out_data = (char *) g_malloc(out_data_bytes + index, 0); - if (out_data == 0) + if ((enc->num_crects > 0) && (enc->num_crects <= UINT16_MAX) && + (enc->num_drects > 0) && (enc->num_drects <= UINT16_MAX)) { - return 0; - } - tiles = (struct rfx_tile *) (out_data + out_data_bytes + 256); - rfxrects = (struct rfx_rect *) (tiles + 4096); + alloc_bytes = XRDP_SURCMD_PREFIX_BYTES; + alloc_bytes += self->max_compressed_bytes; + alloc_bytes += sizeof(struct rfx_tile) * enc->num_crects + + sizeof(struct rfx_rect) * enc->num_drects; + out_data = g_new(char, alloc_bytes); + if (out_data != NULL) + { + tiles = (struct rfx_tile *) + (out_data + XRDP_SURCMD_PREFIX_BYTES + + self->max_compressed_bytes); + rfxrects = (struct rfx_rect *) (tiles + enc->num_crects); - count = enc->num_crects; - for (index = 0; index < count; index++) - { - x = enc->crects[index * 4 + 0]; - y = enc->crects[index * 4 + 1]; - cx = enc->crects[index * 4 + 2]; - cy = enc->crects[index * 4 + 3]; - LLOGLN(10, ("process_enc_rfx:")); - tiles[index].x = x; - tiles[index].y = y; - tiles[index].cx = cx; - tiles[index].cy = cy; - LLOGLN(10, ("x %d y %d cx %d cy %d", x, y, cx, cy)); - tiles[index].quant_y = 0; - tiles[index].quant_cb = 0; - tiles[index].quant_cr = 0; - } + count = enc->num_crects; + for (index = 0; index < count; index++) + { + x = enc->crects[index * 4 + 0]; + y = enc->crects[index * 4 + 1]; + cx = enc->crects[index * 4 + 2]; + cy = enc->crects[index * 4 + 3]; + tiles[index].x = x; + tiles[index].y = y; + tiles[index].cx = cx; + tiles[index].cy = cy; + tiles[index].quant_y = 0; + tiles[index].quant_cb = 0; + tiles[index].quant_cr = 0; + } - count = enc->num_drects; - for (index = 0; index < count; index++) - { - x = enc->drects[index * 4 + 0]; - y = enc->drects[index * 4 + 1]; - cx = enc->drects[index * 4 + 2]; - cy = enc->drects[index * 4 + 3]; - LLOGLN(10, ("process_enc_rfx:")); - rfxrects[index].x = x; - rfxrects[index].y = y; - rfxrects[index].cx = cx; - rfxrects[index].cy = cy; + count = enc->num_drects; + for (index = 0; index < count; index++) + { + x = enc->drects[index * 4 + 0]; + y = enc->drects[index * 4 + 1]; + cx = enc->drects[index * 4 + 2]; + cy = enc->drects[index * 4 + 3]; + rfxrects[index].x = x; + rfxrects[index].y = y; + rfxrects[index].cx = cx; + rfxrects[index].cy = cy; + } + + out_data_bytes = self->max_compressed_bytes; + error = rfxcodec_encode(self->codec_handle, + out_data + XRDP_SURCMD_PREFIX_BYTES, + &out_data_bytes, enc->data, + enc->width, enc->height, enc->width * 4, + rfxrects, enc->num_drects, + tiles, enc->num_crects, 0, 0); + } } - error = rfxcodec_encode(self->codec_handle, out_data + 256, &out_data_bytes, - enc->data, enc->width, enc->height, enc->width * 4, - rfxrects, enc->num_drects, - tiles, enc->num_crects, 0, 0); LLOGLN(10, ("process_enc_rfx: rfxcodec_encode rv %d", error)); - - enc_done = (XRDP_ENC_DATA_DONE *) - g_malloc(sizeof(XRDP_ENC_DATA_DONE), 1); - enc_done->comp_bytes = out_data_bytes; - enc_done->pad_bytes = 256; + /* only if enc_done->comp_bytes is not zero is something sent + to the client but you must always send something back even + on error so Xorg can get ack */ + enc_done = g_new0(XRDP_ENC_DATA_DONE, 1); + if (enc_done == NULL) + { + return 1; + } + enc_done->comp_bytes = error == 0 ? out_data_bytes : 0; + enc_done->pad_bytes = XRDP_SURCMD_PREFIX_BYTES; enc_done->comp_pad_data = out_data; enc_done->enc = enc; enc_done->last = 1; diff --git a/xrdp/xrdp_encoder.h b/xrdp/xrdp_encoder.h index f138d749..4c57471c 100644 --- a/xrdp/xrdp_encoder.h +++ b/xrdp/xrdp_encoder.h @@ -14,6 +14,7 @@ struct xrdp_encoder int in_codec_mode; int codec_id; int codec_quality; + int max_compressed_bytes; tbus xrdp_encoder_event_to_proc; tbus xrdp_encoder_event_processed; tbus xrdp_encoder_term;