Disable implicit rejection for RSA PKCS#1

Starting in OpenSSL 3.2, RSA PKCS#1 v1.5 decryption no longer fails for invalid RSA padding. Instead, it produces random output data.

Some Linux distributions back ported this to OpenSSL 3.1.x which resulted in some outerloop test failures.

This disables the "implicit rejection" of PKCS#1 v1.5 RSA decryption so that RSA.Encrypt and RSA.Decrypt continue to follow their documented behavior and cross-platform behavior.  It also adds a non-outerloop test so we have better optics on this problem.

src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/RSA/EncryptDecrypt.cs

@@ -353,19 +353,10 @@ private void RsaCryptRoundtrip(RSAEncryptionPadding paddingMode, bool expectSucc
             Assert.Equal(TestData.HelloBytes, output);
         }
 
-        [ConditionalFact]
+        [ConditionalFact(nameof(PlatformSupportsEmptyRSAEncryption))]
         [SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework)]
         public void RoundtripEmptyArray()
         {
-            if (OperatingSystem.IsIOS() && !OperatingSystem.IsIOSVersionAtLeast(13, 6))
-            {
-                throw new SkipTestException("iOS prior to 13.6 does not reliably support RSA encryption of empty data.");
-            }
-            if (OperatingSystem.IsTvOS() && !OperatingSystem.IsTvOSVersionAtLeast(14, 0))
-            {
-                throw new SkipTestException("tvOS prior to 14.0 does not reliably support RSA encryption of empty data.");
-            }
-
             using (RSA rsa = RSAFactory.Create(TestData.RSA2048Params))
             {
                 void RoundtripEmpty(RSAEncryptionPadding paddingMode)
@@ -725,6 +716,26 @@ public void NotSupportedValueMethods()
             }
         }
 
+        [ConditionalTheory]
+        [InlineData(new byte[] { 1, 2, 3, 4 })]
+        [InlineData(new byte[0])]
+        public void Decrypt_Pkcs1_ErrorsForInvalidPadding(byte[] data)
+        {
+            if (data.Length == 0 && !PlatformSupportsEmptyRSAEncryption)
+            {
+                throw new SkipTestException("Platform does not support RSA encryption of empty data.");
+            }
+
+            using (RSA rsa = RSAFactory.Create(TestData.RSA2048Params))
+            {
+                byte[] encrypted = Encrypt(rsa, data, RSAEncryptionPadding.Pkcs1);
+                encrypted[1] ^= 0xFF;
+
+                // PKCS#1, the data, and the key are all deterministic so this should always throw an exception.
+                Assert.ThrowsAny<CryptographicException>(() => Decrypt(rsa, encrypted, RSAEncryptionPadding.Pkcs1));
+            }
+        }
+
         public static IEnumerable<object[]> OaepPaddingModes
         {
             get
@@ -746,5 +757,23 @@ public static IEnumerable<object[]> OaepPaddingModes
                 }
             }
         }
+
+        public static bool PlatformSupportsEmptyRSAEncryption
+        {
+            get
+            {
+                if (OperatingSystem.IsIOS() && !OperatingSystem.IsIOSVersionAtLeast(13, 6))
+                {
+                    return false;
+                }
+
+                if (OperatingSystem.IsTvOS() && !OperatingSystem.IsTvOSVersionAtLeast(14, 0))
+                {
+                    return false;
+                }
+
+                return true;
+            }
+        }
     }
 }

src/native/libs/System.Security.Cryptography.Native/opensslshim.h

@@ -296,8 +296,10 @@ int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, size_t len);
     REQUIRED_FUNCTION(ERR_peek_error) \
     REQUIRED_FUNCTION(ERR_peek_error_line) \
     REQUIRED_FUNCTION(ERR_peek_last_error) \
+    REQUIRED_FUNCTION(ERR_pop_to_mark) \
     FALLBACK_FUNCTION(ERR_put_error) \
     REQUIRED_FUNCTION(ERR_reason_error_string) \
+    REQUIRED_FUNCTION(ERR_set_mark) \
     LIGHTUP_FUNCTION(ERR_set_debug) \
     LIGHTUP_FUNCTION(ERR_set_error) \
     REQUIRED_FUNCTION(EVP_aes_128_cbc) \
@@ -353,6 +355,7 @@ int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, size_t len);
     REQUIRED_FUNCTION(EVP_PKCS82PKEY) \
     REQUIRED_FUNCTION(EVP_PKEY2PKCS8) \
     REQUIRED_FUNCTION(EVP_PKEY_CTX_ctrl) \
+    REQUIRED_FUNCTION(EVP_PKEY_CTX_ctrl_str) \
     REQUIRED_FUNCTION(EVP_PKEY_CTX_free) \
     REQUIRED_FUNCTION(EVP_PKEY_CTX_get0_pkey) \
     REQUIRED_FUNCTION(EVP_PKEY_CTX_new) \
@@ -794,8 +797,10 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define ERR_peek_error_line ERR_peek_error_line_ptr
 #define ERR_peek_last_error ERR_peek_last_error_ptr
 #define ERR_put_error ERR_put_error_ptr
+#define ERR_pop_to_mark ERR_pop_to_mark_ptr
 #define ERR_reason_error_string ERR_reason_error_string_ptr
 #define ERR_set_debug ERR_set_debug_ptr
+#define ERR_set_mark ERR_set_mark_ptr
 #define ERR_set_error ERR_set_error_ptr
 #define EVP_aes_128_cbc EVP_aes_128_cbc_ptr
 #define EVP_aes_128_cfb8 EVP_aes_128_cfb8_ptr
@@ -850,6 +855,7 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define EVP_PKCS82PKEY EVP_PKCS82PKEY_ptr
 #define EVP_PKEY2PKCS8 EVP_PKEY2PKCS8_ptr
 #define EVP_PKEY_CTX_ctrl EVP_PKEY_CTX_ctrl_ptr
+#define EVP_PKEY_CTX_ctrl_str EVP_PKEY_CTX_ctrl_str_ptr
 #define EVP_PKEY_CTX_free EVP_PKEY_CTX_free_ptr
 #define EVP_PKEY_CTX_get0_pkey EVP_PKEY_CTX_get0_pkey_ptr
 #define EVP_PKEY_CTX_new EVP_PKEY_CTX_new_ptr

src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey_rsa.c

@@ -67,6 +67,19 @@ static bool ConfigureEncryption(EVP_PKEY_CTX* ctx, RsaPaddingMode padding, const
         {
             return false;
         }
+
+        // OpenSSL 3.2 introduced a change where PKCS#1 RSA decryption does not fail for invalid padding.
+        // If the padding is invalid, the decryption operation returns random data.
+        // See https://github.com/openssl/openssl/pull/13817 for background.
+        // Some Linux distributions backported this change to previous versions of OpenSSL.
+        // Here we do a best-effort to set a flag to revert the behavior to failing if the padding is invalid.
+        ERR_set_mark();
+
+        EVP_PKEY_CTX_ctrl_str(ctx, "rsa_pkcs1_implicit_rejection", "0");
+
+        // Undo any changes to the error queue that may have occured while configuring implicit rejection if the
+        // current version does not support implicit rejection.
+        ERR_pop_to_mark();
     }
     else
     {