Support Mac OS X vnc client with no password

Support connections from the Mac OS X built-in VNC client to
LibVNCServers running with no password and advertising a server
version of 3.7 or greater.

libvncserver/auth.c

@@ -127,12 +127,37 @@ rfbVncAuthSendChallenge(rfbClientPtr cl)
  * Send the NO AUTHENTICATION. SCARR
  */
 
+/*
+ * The rfbVncAuthNone function is currently the only function that contains
+ * special logic for the built-in Mac OS X VNC client which is activated by
+ * a protocolMinorVersion == 889 coming from the Mac OS X VNC client.
+ * The rfbProcessClientInitMessage function does understand how to handle the
+ * RFB_INITIALISATION_SHARED state which was introduced to support the built-in
+ * Mac OS X VNC client, but rfbProcessClientInitMessage does not examine the
+ * protocolMinorVersion version field and so its support for the
+ * RFB_INITIALISATION_SHARED state is not restricted to just the OS X client.
+ */
+
 static void
 rfbVncAuthNone(rfbClientPtr cl)
 {
+    /* The built-in Mac OS X VNC client behaves in a non-conforming fashion
+     * when the server version is 3.7 or later AND the list of security types
+     * sent to the OS X client contains the 'None' authentication type AND
+     * the OS X client sends back the 'None' type as its choice.  In this case,
+     * and this case ONLY, the built-in Mac OS X VNC client will NOT send the
+     * ClientInit message and instead will behave as though an implicit
+     * ClientInit message containing a shared-flag of true has been sent.
+     * The special state RFB_INITIALISATION_SHARED represents this case.
+     * The Mac OS X VNC client can be detected by checking protocolMinorVersion
+     * for a value of 889.  No other VNC client is known to use this value
+     * for protocolMinorVersion. */
     uint32_t authResult;
 
-    if (cl->protocolMajorVersion==3 && cl->protocolMinorVersion > 7) {
+    /* The built-in Mac OS X VNC client expects to NOT receive a SecurityResult
+     * message for authentication type 'None'.  Since its protocolMinorVersion
+     * is greater than 7 (it is 889) this case must be tested for specially. */
+    if (cl->protocolMajorVersion==3 && cl->protocolMinorVersion > 7 && cl->protocolMinorVersion != 889) {
         rfbLog("rfbProcessClientSecurityType: returning securityResult for client rfb version >= 3.8\n");
         authResult = Swap32IfLE(rfbVncAuthOK);
         if (rfbWriteExact(cl, (char *)&authResult, 4) < 0) {
@@ -141,7 +166,12 @@ rfbVncAuthNone(rfbClientPtr cl)
             return;
         }
     }
-    cl->state = RFB_INITIALISATION;
+    cl->state = cl->protocolMinorVersion == 889 ? RFB_INITIALISATION_SHARED : RFB_INITIALISATION;
+    if (cl->state == RFB_INITIALISATION_SHARED)
+        /* In this case we must call rfbProcessClientMessage now because
+         * otherwise we would hang waiting for data to be received from the
+         * client (the ClientInit message which will never come). */
+        rfbProcessClientMessage(cl);
     return;
 }
 

libvncserver/rfbserver.c

@@ -593,6 +593,7 @@ rfbProcessClientMessage(rfbClientPtr cl)
         rfbAuthProcessClientMessage(cl);
         return;
     case RFB_INITIALISATION:
+    case RFB_INITIALISATION_SHARED:
         rfbProcessClientInitMessage(cl);
         return;
     default:
@@ -720,6 +721,14 @@ rfbProcessClientInitMessage(rfbClientPtr cl)
     rfbClientPtr otherCl;
     rfbExtensionData* extension;
 
+    if (cl->state == RFB_INITIALISATION_SHARED) {
+        /* In this case behave as though an implicit ClientInit message has
+         * already been received with a shared-flag of true. */
+        ci.shared = 1;
+        /* Avoid the possibility of exposing the RFB_INITIALISATION_SHARED
+         * state to calling software. */
+        cl->state = RFB_INITIALISATION;
+    } else {
         if ((n = rfbReadExact(cl, (char *)&ci,sz_rfbClientInitMsg)) <= 0) {
             if (n == 0)
                 rfbLog("rfbProcessClientInitMessage: client gone\n");
@@ -728,6 +737,7 @@ rfbProcessClientInitMessage(rfbClientPtr cl)
             rfbCloseClient(cl);
             return;
         }
+    }
 
     memset(u.buf,0,sizeof(u.buf));
 

rfb/rfb.h

@@ -456,13 +456,44 @@ typedef struct _rfbClientRec {
 #ifdef LIBVNCSERVER_HAVE_LIBPTHREAD
     pthread_t client_thread;
 #endif
+
+    /* Note that the RFB_INITIALISATION_SHARED state is provided to support
+       clients that under some circumstances do not send a ClientInit message.
+       In particular the Mac OS X built-in VNC client (with protocolMinorVersion
+       == 889) is one of those.  However, it only requires this support under
+       special circumstances that can only be determined during the initial
+       authentication.  If the right conditions are met this state will be
+       set (see the auth.c file) when rfbProcessClientInitMessage is called.
+
+       If the state is RFB_INITIALISATION_SHARED we should not expect to recieve
+       any ClientInit message, but instead should proceed to the next stage
+       of initialisation as though an implicit ClientInit message was received
+       with a shared-flag of true.  (There is currently no corresponding
+       RFB_INITIALISATION_NOTSHARED state to represent an implicit ClientInit
+       message with a shared-flag of false because no known existing client
+       requires such support at this time.)
+
+       Note that software using LibVNCServer to provide a VNC server will only
+       ever have a chance to see the state field set to
+       RFB_INITIALISATION_SHARED if the software is multi-threaded and manages
+       to examine the state field during the extremely brief window after the
+       'None' authentication type selection has been received from the built-in
+       OS X VNC client and before the rfbProcessClientInitMessage function is
+       called -- control cannot return to the caller during this brief window
+       while the state field is set to RFB_INITIALISATION_SHARED. */
+
                                 /** Possible client states: */
     enum {
         RFB_PROTOCOL_VERSION,   /**< establishing protocol version */
 	RFB_SECURITY_TYPE,      /**< negotiating security (RFB v.3.7) */
         RFB_AUTHENTICATION,     /**< authenticating */
         RFB_INITIALISATION,     /**< sending initialisation messages */
-        RFB_NORMAL              /**< normal protocol messages */
+        RFB_NORMAL,             /**< normal protocol messages */
+
+        /* Ephemeral internal-use states that will never be seen by software
+         * using LibVNCServer to provide services: */
+
+        RFB_INITIALISATION_SHARED /**< sending initialisation messages with implicit shared-flag already true */
     } state;
 
     rfbBool reverseConnection;