diff --git a/sesman/env.c b/sesman/env.c
index 1a4fa679..d8eb3106 100644
--- a/sesman/env.c
+++ b/sesman/env.c
@@ -150,7 +150,7 @@ env_set_user(const char *username, char **passwd_file, int display,
                 if (0 == g_cfg->auth_file_path)
                 {
                     /* if no auth_file_path is set, then we go for
-                     $HOME/.vnc/sesman_username_passwd */
+                     $HOME/.vnc/sesman_username_passwd:DISPLAY */
                     if (!g_directory_exist(".vnc"))
                     {
                         if (g_mkdir(".vnc") < 0)
@@ -161,12 +161,24 @@ env_set_user(const char *username, char **passwd_file, int display,
                         }
                     }
 
-                    len = g_snprintf(NULL, 0, "%s/.vnc/sesman_%s_passwd", pw_dir, username);
+                    len = g_snprintf(NULL, 0, "%s/.vnc/sesman_%s_passwd:%d",
+                                     pw_dir, username, display);
 
                     *passwd_file = (char *) g_malloc(len + 1, 1);
                     if (*passwd_file != NULL)
                     {
-                        g_sprintf(*passwd_file, "%s/.vnc/sesman_%s_passwd", pw_dir, username);
+                        /* Try legacy name first, remove if found */
+                        g_sprintf(*passwd_file, "%s/.vnc/sesman_%s_passwd",
+                                  pw_dir, username);
+                        if (g_file_exist(*passwd_file))
+                        {
+                            log_message(LOG_LEVEL_WARNING, "Removing insecure "
+                                        "password file %s", *passwd_file);
+                            g_file_delete(*passwd_file);
+                        }
+
+                        g_sprintf(*passwd_file, "%s/.vnc/sesman_%s_passwd:%d",
+                                  pw_dir, username, display);
                     }
                 }
                 else