removed built in keygen funtion, wasn't working anyway

common/ssl_calls.c

@@ -220,200 +220,14 @@ ssl_mod_exp(char* out, int out_len, char* in, int in_len,
 
 #if defined(OLD_RSA_GEN1)
 /*****************************************************************************/
-/* stolen from openssl-0.9.8d, rsa_gen.c, rsa_builtin_keygen
-   RSAerr function calls just commented out
-   returns boolean */
-static int
-rsa_builtin_keygen1(RSA* rsa, int bits, BIGNUM* e_value, BN_GENCB* cb)
+/* returns error, always errors for now */
+int APP_CC
+ssl_gen_key_xrdp1(int key_size_in_bits, char* exp, int exp_len,
+                  char* mod, int mod_len, char* pri, int pri_len)
 {
-  BIGNUM* r0 = NULL;
-  BIGNUM* r1 = NULL;
-  BIGNUM* r2 = NULL;
-  BIGNUM* r3 = NULL;
-  BIGNUM* tmp;
-  int bitsp;
-  int bitsq;
-  int ok = -1;
-  int n = 0;
-  BN_CTX* ctx = NULL;
-  unsigned int degenerate = 0;
-
-  ctx = BN_CTX_new();
-  if (ctx == NULL)
-  {
-    goto err;
-  }
-  BN_CTX_start(ctx);
-  r0 = BN_CTX_get(ctx);
-  r1 = BN_CTX_get(ctx);
-  r2 = BN_CTX_get(ctx);
-  r3 = BN_CTX_get(ctx);
-  if (r3 == NULL)
-  {
-    goto err;
-  }
-  bitsp = (bits + 1) / 2;
-  bitsq = bits - bitsp;
-  /* We need the RSA components non-NULL */
-  if (!rsa->n && ((rsa->n = BN_new()) == NULL))
-  {
-    goto err;
-  }
-  if (!rsa->d && ((rsa->d = BN_new()) == NULL))
-  {
-    goto err;
-  }
-  if (!rsa->e && ((rsa->e = BN_new()) == NULL))
-  {
-    goto err;
-  }
-  if (!rsa->p && ((rsa->p = BN_new()) == NULL))
-  {
-    goto err;
-  }
-  if (!rsa->q && ((rsa->q = BN_new()) == NULL))
-  {
-    goto err;
-  }
-  if (!rsa->dmp1 && ((rsa->dmp1 = BN_new()) == NULL))
-  {
-    goto err;
-  }
-  if (!rsa->dmq1 && ((rsa->dmq1 = BN_new()) == NULL))
-  {
-    goto err;
-  }
-  if (!rsa->iqmp && ((rsa->iqmp = BN_new()) == NULL))
-  {
-    goto err;
-  }
-  BN_copy(rsa->e, e_value);
-  /* generate p and q */
-  for (;;)
-  {
-    if (!BN_generate_prime_ex(rsa->p, bitsp, 0, NULL, NULL, cb))
-    {
-      goto err;
-    }
-    if (!BN_sub(r2, rsa->p, BN_value_one()))
-    {
-      goto err;
-    }
-    if (!BN_gcd(r1, r2, rsa->e, ctx))
-    {
-      goto err;
-    }
-    if (BN_is_one(r1))
-    {
-      break;
-    }
-    if (!BN_GENCB_call(cb, 2, n++))
-    {
-      goto err;
-    }
-  }
-  if (!BN_GENCB_call(cb, 3, 0))
-  {
-    goto err;
-  }
-  for (;;)
-  {
-    /* When generating ridiculously small keys, we can get stuck
-     * continually regenerating the same prime values. Check for
-     * this and bail if it happens 3 times. */
-    do
-    {
-      if (!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
-      {
-        goto err;
-      }
-    } while ((BN_cmp(rsa->p, rsa->q) == 0) && (++degenerate < 3));
-    if (degenerate == 3)
-    {
-      ok = 0; /* we set our own err */
-      /*RSAerr(RSA_F_RSA_BUILTIN_KEYGEN,RSA_R_KEY_SIZE_TOO_SMALL);*/
-      goto err;
-    }
-    if (!BN_sub(r2, rsa->q, BN_value_one()))
-    {
-      goto err;
-    }
-    if (!BN_gcd(r1, r2, rsa->e, ctx))
-    {
-      goto err;
-    }
-    if (BN_is_one(r1))
-    {
-      break;
-    }
-    if (!BN_GENCB_call(cb, 2, n++))
-    {
-      goto err;
-    }
-  }
-  if (!BN_GENCB_call(cb, 3, 1))
-  {
-    goto err;
-  }
-  if (BN_cmp(rsa->p, rsa->q) < 0)
-  {
-    tmp = rsa->p;
-    rsa->p = rsa->q;
-    rsa->q = tmp;
-  }
-  /* calculate n */
-  if (!BN_mul(rsa->n, rsa->p, rsa->q, ctx))
-  {
-    goto err;
-  }
-  /* calculate d */
-  if (!BN_sub(r1, rsa->p, BN_value_one()))
-  {
-    goto err; /* p-1 */
-  }
-  if (!BN_sub(r2, rsa->q, BN_value_one()))
-  {
-    goto err; /* q-1 */
-  }
-  if (!BN_mul(r0, r1, r2, ctx))
-  {
-    goto err; /* (p-1)(q-1) */
-  }
-  if (!BN_mod_inverse(rsa->d, rsa->e, r0, ctx))
-  {
-    goto err; /* d */
-  }
-  /* calculate d mod (p-1) */
-  if (!BN_mod(rsa->dmp1, rsa->d, r1, ctx))
-  {
-    goto err;
-  }
-  /* calculate d mod (q-1) */
-  if (!BN_mod(rsa->dmq1, rsa->d, r2, ctx))
-  {
-    goto err;
-  }
-  /* calculate inverse of q mod p */
-  if (!BN_mod_inverse(rsa->iqmp, rsa->q, rsa->p, ctx))
-  {
-    goto err;
-  }
-  ok = 1;
-err:
-  if (ok == -1)
-  {
-    /*RSAerr(RSA_F_RSA_BUILTIN_KEYGEN,ERR_LIB_BN);*/
-    ok = 0;
-  }
-  if (ctx != NULL)
-  {
-    BN_CTX_end(ctx);
-    BN_CTX_free(ctx);
-  }
-  return ok;
+  return 1;
 }
-#endif
-
+#else
 /*****************************************************************************/
 /* returns error
    generates a new rsa key
@@ -442,11 +256,7 @@ ssl_gen_key_xrdp1(int key_size_in_bits, char* exp, int exp_len,
   my_e = BN_new();
   BN_bin2bn((unsigned char*)lexp, exp_len, my_e);
   my_key = RSA_new();
-#if defined(OLD_RSA_GEN1)
-  error = rsa_builtin_keygen1(my_key, key_size_in_bits, my_e, 0) == 0;
-#else
   error = RSA_generate_key_ex(my_key, key_size_in_bits, my_e, 0) == 0;
-#endif
   if (error == 0)
   {
     len = BN_num_bytes(my_key->n);
@@ -479,3 +289,4 @@ ssl_gen_key_xrdp1(int key_size_in_bits, char* exp, int exp_len,
   g_free(lpri);
   return error;
 }
+#endif