Check if OpenSSL is in FIPS mode and use Java for some algorithms

If the OpenSSL library used is in FIPS mode, avoid using it
for algorithms that are not FIPS compliant and revert to the
original Java implementation.

Signed-off-by: Kostas Tsiounis <kostas.tsiounis@ibm.com>

closed/src/java.base/share/classes/jdk/crypto/jniprovider/NativeCrypto.java

@@ -79,6 +79,8 @@ private static final class InstanceHolder {
     // or one of the OPENSSL_VERSION_x_x_x constants
     private final long ossl_ver;
 
+    private final boolean isOpenSSLFIPS;
+
     private static long loadCryptoLibraries() {
         long osslVersion;
 
@@ -105,6 +107,11 @@ private static long loadCryptoLibraries() {
     @SuppressWarnings("removal")
     private NativeCrypto() {
         ossl_ver = AccessController.doPrivileged((PrivilegedAction<Long>) () -> loadCryptoLibraries()).longValue();
+        if (ossl_ver != -1) {
+            isOpenSSLFIPS = isOpenSSLFIPS();
+        } else {
+            isOpenSSLFIPS = false;
+        }
     }
 
     /**
@@ -178,6 +185,54 @@ public static final boolean isTraceEnabled() {
         return traceEnabled;
     }
 
+    public static final boolean isOpenSSLFIPSVersion() {
+        return InstanceHolder.instance.isOpenSSLFIPS;
+    }
+
+    /**
+     * Check whether a native implementation is available in the loaded OpenSSL library.
+     * Note that, an algorithm could be unavailable due to options used to build the
+     * OpenSSL version utilized, or using a FIPS version that doesn't allow it.
+     *
+     * @param algorithm the algorithm checked
+     * @return whether a native implementation of the given crypto algorithm is available
+     */
+    public static final boolean isAlgorithmAvailable(String algorithm) {
+        boolean isAlgorithmAvailable = false;
+        if (isAllowedAndLoaded()) {
+            if (isOpenSSLFIPSVersion()) {
+                switch (algorithm) {
+                case "ChaCha20":
+                case "MD5":
+                    // not available
+                    break;
+                default:
+                    isAlgorithmAvailable = true;
+                    break;
+                }
+            } else {
+                switch (algorithm) {
+                case "MD5":
+                    isAlgorithmAvailable = isMD5Available();
+                    break;
+                default:
+                    isAlgorithmAvailable = true;
+                    break;
+                }
+            }
+        }
+
+        // Issue a message indicating whether the crypto implementation is available.
+        if (traceEnabled) {
+            if (isAlgorithmAvailable) {
+                System.err.println(algorithm + " native crypto implementation is available.");
+            } else {
+                System.err.println(algorithm + " native crypto implementation is not available.");
+            }
+        }
+        return isAlgorithmAvailable;
+    }
+
     @CallerSensitive
     public static NativeCrypto getNativeCrypto() {
         ClassLoader callerClassLoader = Reflection.getCallerClass().getClassLoader();
@@ -204,6 +259,8 @@ public void run() {
 
     public static final native boolean isMD5Available();
 
+    private static final native boolean isOpenSSLFIPS();
+
     public final native long DigestCreateContext(long nativeBuffer,
                                                  int algoIndex);
 

closed/src/java.base/share/native/libjncrypto/NativeCrypto.c

@@ -82,6 +82,9 @@ int OSSL102_RSA_set0_crt_params(RSA *, BIGNUM *, BIGNUM *, BIGNUM *);
 #define EVP_CTRL_AEAD_SET_TAG EVP_CTRL_GCM_SET_TAG
 #endif
 
+/* Whether loaded library is in FIPS mode. */
+static jboolean OSSL_IS_FIPS;
+
 /* Header for EC algorithm */
 jboolean OSSL_ECGF2M;
 int setECPublicCoordinates(EC_KEY *, BIGNUM *, BIGNUM *, int);
@@ -365,6 +368,18 @@ static jlong extractVersionToJlong(const char *astring)
 }
 
 static void *crypto_library = NULL;
+
+/*
+ * Class:     jdk_crypto_jniprovider_NativeCrypto
+ * Method:    isOpenSSLFIPS
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_jdk_crypto_jniprovider_NativeCrypto_isOpenSSLFIPS
+  (JNIEnv *env, jclass clazz)
+{
+    return OSSL_IS_FIPS;
+}
+
 /*
  * Class:     jdk_crypto_jniprovider_NativeCrypto
  * Method:    loadCrypto
@@ -440,6 +455,25 @@ JNIEXPORT jlong JNICALL Java_jdk_crypto_jniprovider_NativeCrypto_loadCrypto
         }
     }
 
+    /* Check whether the loaded OpenSSL library is in FIPS mode. */
+    if (ossl_ver >= OPENSSL_VERSION_3_0_0) {
+        typedef int OSSL_fipsmode_t(OSSL_LIB_CTX *);
+        OSSL_fipsmode_t *ossl_fipsmode = (OSSL_fipsmode_t *)find_crypto_symbol(crypto_library, "EVP_default_properties_is_fips_enabled");
+        if ((NULL != ossl_fipsmode) && (1 == (*ossl_fipsmode)(NULL))) {
+            OSSL_IS_FIPS = JNI_TRUE;
+        } else {
+            OSSL_IS_FIPS = JNI_FALSE;
+        }
+    } else {
+        typedef int OSSL_fipsmode_t(void);
+        OSSL_fipsmode_t *ossl_fipsmode = (OSSL_fipsmode_t *)find_crypto_symbol(crypto_library, "FIPS_mode");
+        if ((NULL != ossl_fipsmode) && (1 == (*ossl_fipsmode)())) {
+            OSSL_IS_FIPS = JNI_TRUE;
+        } else {
+            OSSL_IS_FIPS = JNI_FALSE;
+        }
+    }
+
     /* Load the function symbols for OpenSSL errors. */
     OSSL_error_string_n = (OSSL_error_string_n_t*)find_crypto_symbol(crypto_library, "ERR_error_string_n");
     OSSL_error_string = (OSSL_error_string_t*)find_crypto_symbol(crypto_library, "ERR_error_string");

src/java.base/share/classes/com/sun/crypto/provider/SunJCE.java

@@ -24,7 +24,7 @@
  */
 /*
  * ===========================================================================
- * (c) Copyright IBM Corp. 2018, 2023 All Rights Reserved
+ * (c) Copyright IBM Corp. 2018, 2024 All Rights Reserved
  * ===========================================================================
  */
 
@@ -336,7 +336,10 @@ void putEntries() {
         attrs.clear();
         attrs.put("SupportedKeyFormats", "RAW");
 
-        if (useNativeChaCha20Cipher && (NativeCrypto.getVersionIfAvailable() >= NativeCrypto.OPENSSL_VERSION_1_1_0)) {
+        if (useNativeChaCha20Cipher
+            && NativeCrypto.isAlgorithmAvailable("ChaCha20")
+            && (NativeCrypto.getVersionIfAvailable() >= NativeCrypto.OPENSSL_VERSION_1_1_0)
+        ) {
             ps("Cipher", "ChaCha20",
                     "com.sun.crypto.provider.NativeChaCha20Cipher$ChaCha20Only",
                     null, attrs);

src/java.base/share/classes/sun/security/provider/SunEntries.java

@@ -317,8 +317,7 @@ public final class SunEntries {
          */
         /* Don't use native MD5 on AIX due to an observed performance regression. */
         if (useNativeMD5
-            && NativeCrypto.isAllowedAndLoaded()
-            && NativeCrypto.isMD5Available()
+            && NativeCrypto.isAlgorithmAvailable("MD5")
             && !isAIX
         ) {
             providerMD5 = "sun.security.provider.NativeMD5";