Merge pull request #596 from speidy/rfx_frameacks

RFX: fix frame acks handling

libxrdp/xrdp_caps.c

@@ -539,6 +539,12 @@ xrdp_caps_process_frame_ack(struct xrdp_rdp *self, struct stream *s, int len)
     g_writeln("xrdp_caps_process_frame_ack:");
     self->client_info.use_frame_acks = 1;
     in_uint32_le(s, self->client_info.max_unacknowledged_frame_count);
+    if (self->client_info.max_unacknowledged_frame_count < 0)
+    {
+        g_writeln("  invalid max_unacknowledged_frame_count value (%d), setting to 0",
+                  self->client_info.max_unacknowledged_frame_count);
+        self->client_info.max_unacknowledged_frame_count = 0;
+    }
     g_writeln("  max_unacknowledged_frame_count %d", self->client_info.max_unacknowledged_frame_count);
     return 0;
 }

xrdp/xrdp_mm.c

@@ -2153,7 +2153,6 @@ xrdp_mm_check_wait_objs(struct xrdp_mm *self)
     int cx;
     int cy;
     int use_frame_acks;
-    int ex;
 
     if (self == 0)
     {
@@ -2258,21 +2257,6 @@ xrdp_mm_check_wait_objs(struct xrdp_mm *self)
                                                  enc_done->enc->flags,
                                                  enc_done->enc->frame_id);
                     }
-                    else
-                    {
-#if 1
-                        ex = self->wm->client_info->max_unacknowledged_frame_count;
-                        if (self->encoder->frame_id_client + ex > self->encoder->frame_id_server)
-                        {
-                            if (self->encoder->frame_id_server > self->encoder->frame_id_server_sent)
-                            {
-                                LLOGLN(10, ("xrdp_mm_check_wait_objs: 1 -- %d", self->encoder->frame_id_server));
-                                self->encoder->frame_id_server_sent = self->encoder->frame_id_server;
-                                self->mod->mod_frame_ack(self->mod, 0, self->encoder->frame_id_server);
-                            }
-                        }
-#endif
-                    }
                     g_free(enc_done->enc->drects);
                     g_free(enc_done->enc->crects);
                     g_free(enc_done->enc);
@@ -2296,14 +2280,16 @@ xrdp_mm_frame_ack(struct xrdp_mm *self, int frame_id)
 {
     int ex;
 
-    LLOGLN(10, ("xrdp_mm_frame_ack:"));
+    LLOGLN(10, ("xrdp_mm_frame_ack: incoming %d, client %d, server %d", frame_id,
+        self->encoder->frame_id_client, self->encoder->frame_id_server));
     self->encoder->frame_id_client = frame_id;
     if (self->wm->client_info->use_frame_acks == 0)
     {
         return 1;
     }
     ex = self->wm->client_info->max_unacknowledged_frame_count;
-    if (self->encoder->frame_id_client + ex > self->encoder->frame_id_server)
+    /* make sure we won't have too many in-flight frames */
+    if (self->encoder->frame_id_client + ex >= self->encoder->frame_id_server)
     {
         if (self->encoder->frame_id_server > self->encoder->frame_id_server_sent)
         {