crypto: clear OpenSSL error queue after calling X509_check_private_key()

Backport-PR-URL: https://github.com/nodejs-private/node-private/pull/375
CVE-ID: CVE-2023-23919
Fixes: nodejs/node#45485
PR-URL: nodejs/node#45495
Refs: https://hackerone.com/bugs?subject=nodejs&report_id=1808596
Reviewed-By: Tobias Nießen <tniessen@tnie.de>
Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com>

src/crypto/crypto_x509.cc

@@ -453,6 +453,8 @@ void X509Certificate::CheckPrivateKey(const FunctionCallbackInfo<Value>& args) {
   ASSIGN_OR_RETURN_UNWRAP(&key, args[0]);
   CHECK_EQ(key->Data()->GetKeyType(), kKeyTypePrivate);
 
+  ClearErrorOnReturn clear_error_on_return;
+
   args.GetReturnValue().Set(
       X509_check_private_key(
           cert->get(),

test/parallel/test-crypto-x509.js

@@ -8,6 +8,7 @@ if (!common.hasCrypto)
 const {
   X509Certificate,
   createPrivateKey,
+  generateKeyPairSync,
 } = require('crypto');
 
 const {
@@ -187,10 +188,18 @@ const der = Buffer.from(
     code: 'ERR_INVALID_ARG_VALUE'
   });
 
-  // Confirm failure of X509Certificate:verify() doesn't affect other functions that use OpenSSL.
-  assert(!x509.verify(x509.publicKey));
-  // This call should not throw.
-  createPrivateKey(key);
+  {
+    // https://github.com/nodejs/node/issues/45377
+    // https://github.com/nodejs/node/issues/45485
+    // Confirm failures of X509Certificate:verify() and X509Certificate:CheckPrivateKey()
+    // do not affect other functions that use OpenSSL.
+    // Subsequent calls to e.g. createPrivateKey should not throw.
+    const keyPair = generateKeyPairSync('ed25519');
+    assert(!x509.verify(keyPair.publicKey));
+    createPrivateKey(key);
+    assert(!x509.checkPrivateKey(keyPair.privateKey));
+    createPrivateKey(key);
+  }
 
   // X509Certificate can be cloned via MessageChannel/MessagePort
   const mc = new MessageChannel();