The PseudoEncoding extension code was getting silly:

If the client asked for an encoding, and no enabled extension handled it,
LibVNCServer would walk through all extensions, and if they promised to handle
the encoding, execute the extension's newClient() if it was not NULL.

However, if newClient is not NULL, it will be called when a client connects,
and if it returns TRUE, the extension will be enabled. Since all the state of
the extension should be in the client data, there is no good reason why
newClient should return FALSE the first time (thus not enabling the extension),
but TRUE when called just before calling enablePseudoEncoding().

So in effect, the extension got enabled all the time, even if that was not
necessary.

The resolution is to pass a void** to enablePseudoEncoding. This has the
further advantage that enablePseudoEncoding can remalloc() or free() the
data without problems. Though keep in mind that if enablePseudoEncoding()
is called on a not-yet-enabled extension, the passed data points to NULL.

examples/backchannel.c

@@ -33,7 +33,7 @@ typedef struct backChannelMsg {
 	uint32_t size;
 } backChannelMsg;
 
-rfbBool enableBackChannel(rfbClientPtr cl, void* data, int encoding)
+rfbBool enableBackChannel(rfbClientPtr cl, void** data, int encoding)
 {
 	if(encoding == rfbBackChannel) {
 		backChannelMsg msg;

libvncserver/rfbserver.c

@@ -921,7 +921,7 @@ rfbProcessClientNormalMessage(rfbClientPtr cl)
 				rfbExtensionData* next = e->next;
 				if(e->extension->enablePseudoEncoding &&
 					e->extension->enablePseudoEncoding(cl,
-						e->data, (int)enc))
+						&e->data, (int)enc))
 					/* ext handles this encoding */
 					break;
 				e = next;
@@ -938,9 +938,7 @@ rfbProcessClientNormalMessage(rfbClientPtr cl)
 					while(encs && *encs!=0) {
 						if(*encs==(int)enc) {
 							void* data = NULL;
-							if(e->newClient)
-								e->newClient(cl, &data);
-							if(!e->enablePseudoEncoding(cl, data, (int)enc)) {
+							if(!e->enablePseudoEncoding(cl, &data, (int)enc)) {
 								rfbLog("Installed extension pretends to handle pseudo encoding 0x%x, but does not!\n",(int)enc);
 							} else {
 								rfbEnableExtension(cl, e, data);

rfb/rfb.h

@@ -170,7 +170,7 @@ typedef struct _rfbProtocolExtension {
 	/* returns TRUE if that pseudo encoding is handled by the extension.
 	   encodingNumber==0 means "reset encodings". */
 	rfbBool (*enablePseudoEncoding)(struct _rfbClientRec* client,
-			void* data, int encodingNumber);
+			void** data, int encodingNumber);
 	/* returns TRUE if message was handled */
 	rfbBool (*handleMessage)(struct _rfbClientRec* client,
 				void* data,