Fix DH_check() excessive time with oversized modulus (aws#1109)

(cherry picked from commit 9545d9d)

crypto/dh_extra/dh_test.cc

@@ -112,6 +112,33 @@ TEST(DHTest, Basic) {
   EXPECT_GE(key1.size(), 4u);
 }
 
+TEST(DHTest, OversizedModulus) {
+  bssl::UniquePtr<DH> a(DH_new());
+  ASSERT_TRUE(a);
+
+  const size_t LARGE_MOD_P = 4097;  // OPENSSL_DH_CHECK_MAX_MODULUS_BITS / 8 + 1
+
+  // Create a BigNumber which will be interpreted as a big-endian value
+  auto number = std::unique_ptr<uint8_t[], std::default_delete<uint8_t[]>>(
+      new uint8_t[LARGE_MOD_P]);
+  for (size_t i = 0; i < LARGE_MOD_P; i++) {
+    number[i] = 255;
+  }
+
+  bssl::UniquePtr<BIGNUM> p(BN_bin2bn(number.get(), LARGE_MOD_P, nullptr));
+  bssl::UniquePtr<BIGNUM> q(BN_new());
+  bssl::UniquePtr<BIGNUM> g(BN_new());
+
+  // Q and G don't matter for this test, they just can't be null
+  ASSERT_TRUE(DH_set0_pqg(a.get(), p.release(), q.release(), g.release()));
+
+  int check_result;
+  ASSERT_FALSE(DH_check(a.get(), &check_result));
+  uint32_t error = ERR_get_error();
+  ASSERT_EQ(ERR_LIB_DH, ERR_GET_LIB(error));
+  ASSERT_EQ(DH_R_MODULUS_TOO_LARGE, ERR_GET_REASON(error));
+}
+
 // The following parameters are taken from RFC 5114, section 2.2. This is not a
 // safe prime. Do not use these parameters.
 static const uint8_t kRFC5114_2048_224P[] = {

crypto/fipsmodule/dh/check.c

@@ -60,6 +60,7 @@
 
 #include "internal.h"
 
+#define OPENSSL_DH_CHECK_MAX_MODULUS_BITS  32768
 
 int DH_check_pub_key(const DH *dh, const BIGNUM *pub_key, int *out_flags) {
   *out_flags = 0;
@@ -125,6 +126,13 @@ int DH_check(const DH *dh, int *out_flags) {
   BIGNUM *t1 = NULL, *t2 = NULL;
 
   *out_flags = 0;
+
+  /* Don't do any checks at all with an excessively large modulus */
+  if (BN_num_bits(dh->p) > OPENSSL_DH_CHECK_MAX_MODULUS_BITS) {
+    OPENSSL_PUT_ERROR(DH, DH_R_MODULUS_TOO_LARGE);
+    return 0;
+  }
+
   ctx = BN_CTX_new();
   if (ctx == NULL) {
     goto err;