o caps: advertise remotefx codec capsets only when supported (RFX mode,

proxy mode)
o xrdp_encoder: do not initialze encoder context for RFX when rfxcodec
is not supported

libxrdp/xrdp_caps.c

@@ -831,12 +831,14 @@ xrdp_caps_send_demand_active(struct xrdp_rdp *self)
     out_uint8(s, 0x01); /* fAllowDynamicFidelity */
     out_uint8(s, 0x01); /* fAllowSubsampling */
     out_uint8(s, 0x03); /* colorLossLevel */
+#if defined(XRDP_RFXCODEC) || defined(XRDP_NEUTRINORDP)
     /* remotefx */
     codec_caps_count++;
     out_uint8a(s, XR_CODEC_GUID_REMOTEFX, 16);
     out_uint8(s, 0); /* codec id, client sets */
     out_uint16_le(s, 256);
     out_uint8s(s, 256);
+#endif
     /* jpeg */
     codec_caps_count++;
     out_uint8a(s, XR_CODEC_GUID_JPEG, 16);

xrdp/xrdp_encoder.c

@@ -42,8 +42,10 @@
 /*****************************************************************************/
 static int
 process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
+#ifdef XRDP_RFXCODEC
 static int
 process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
+#endif
 static int
 process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
 
@@ -87,6 +89,7 @@ xrdp_encoder_create(struct xrdp_mm *mm)
             (32 << 24) | (3 << 16) | (8 << 12) | (8 << 8) | (8 << 4) | 8;
         self->process_enc = process_enc_jpg;
     }
+#ifdef XRDP_RFXCODEC
     else if (client_info->rfx_codec_id != 0)
     {
         LLOGLN(0, ("xrdp_encoder_create: starting rfx codec session"));
@@ -94,13 +97,11 @@ xrdp_encoder_create(struct xrdp_mm *mm)
         self->in_codec_mode = 1;
         client_info->capture_code = 2;
         self->process_enc = process_enc_rfx;
-#ifdef XRDP_RFXCODEC
+        self->codec_handle = rfxcodec_encode_create(mm->wm->screen->width,
-        self->codec_handle =
+                                                    mm->wm->screen->height,
-            rfxcodec_encode_create(mm->wm->screen->width,
+                                                    RFX_FORMAT_YUV, 0);
-                                   mm->wm->screen->height,
-                                   RFX_FORMAT_YUV, 0);
-#endif
     }
+#endif
     else if (client_info->h264_codec_id != 0)
     {
         LLOGLN(0, ("xrdp_encoder_create: starting h264 codec session"));
@@ -296,7 +297,6 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
 }
 
 #ifdef XRDP_RFXCODEC
-
 /*****************************************************************************/
 /* called from encoder thread */
 static int
@@ -399,17 +399,6 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
 
     return 0;
 }
-
-#else
-
-/*****************************************************************************/
-/* called from encoder thread */
-static int
-process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
-{
-    return 0;
-}
-
 #endif
 
 /*****************************************************************************/