for Xvnc sesman session, use hash of password for vnc auth

sesman/env.c

@@ -34,25 +34,44 @@ extern struct config_sesman *g_cfg;  /* in sesman.c */
 
 /******************************************************************************/
 int DEFAULT_CC
-env_check_password_file(char *filename, char *password)
+env_check_password_file(char *filename, char *passwd)
 {
     char encryptedPasswd[16];
     char key[24];
+    char passwd_hash[20];
+    char passwd_hash_text[40];
     int fd;
-    void* des;
+    int passwd_bytes;
+    void *des;
+    void *sha1;
 
+    /* create password hash from passowrd */
+    passwd_bytes = g_strlen(passwd);
+    sha1 = ssl_sha1_info_create();
+    ssl_sha1_transform(sha1, "xrdp_vnc", 8);
+    ssl_sha1_transform(sha1, passwd, passwd_bytes);
+    ssl_sha1_transform(sha1, passwd, passwd_bytes);
+    ssl_sha1_complete(sha1, passwd_hash);
+    ssl_sha1_info_delete(sha1);
+    g_snprintf(passwd_hash_text, 39, "%2.2x%2.2x%2.2x%2.2x",
+               (tui8)passwd_hash[0], (tui8)passwd_hash[1],
+               (tui8)passwd_hash[2], (tui8)passwd_hash[3]);
+    passwd_hash_text[39] = 0;
+    passwd = passwd_hash_text;
+
+    /* create file from password */
     g_memset(encryptedPasswd, 0, sizeof(encryptedPasswd));
-    g_strncpy(encryptedPasswd, password, 8);
+    g_strncpy(encryptedPasswd, passwd, 8);
     g_memset(key, 0, sizeof(key));
     g_mirror_memcpy(key, g_fixedkey, 8);
     des = ssl_des3_encrypt_info_create(key, 0); 
     ssl_des3_encrypt(des, 8, encryptedPasswd, encryptedPasswd);
     ssl_des3_info_delete(des);
-    fd = g_file_open(filename);
+    fd = g_file_open_ex(filename, 0, 1, 1, 1);
     if (fd == -1)
     {
         log_message(LOG_LEVEL_WARNING,
-                    "can't read vnc password file - %s",
+                    "can't write vnc password hash file - %s",
                     filename);
         return 1;
     }

vnc/vnc.c

@@ -53,8 +53,26 @@ void DEFAULT_CC
 rfbEncryptBytes(char *bytes, char *passwd)
 {
     char key[24];
+    char passwd_hash[20];
+    char passwd_hash_text[40];
     void *des;
+    void *sha1;
     int len;
+    int passwd_bytes;
+
+    /* create password hash from passowrd */
+    passwd_bytes = g_strlen(passwd);
+    sha1 = ssl_sha1_info_create();
+    ssl_sha1_transform(sha1, "xrdp_vnc", 8);
+    ssl_sha1_transform(sha1, passwd, passwd_bytes);
+    ssl_sha1_transform(sha1, passwd, passwd_bytes);
+    ssl_sha1_complete(sha1, passwd_hash);
+    ssl_sha1_info_delete(sha1);
+    g_snprintf(passwd_hash_text, 39, "%2.2x%2.2x%2.2x%2.2x",
+               (tui8)passwd_hash[0], (tui8)passwd_hash[1],
+               (tui8)passwd_hash[2], (tui8)passwd_hash[3]);
+    passwd_hash_text[39] = 0;
+    passwd = passwd_hash_text;
 
     /* key is simply password padded with nulls */
     g_memset(key, 0, sizeof(key));