Fix build with OpenSSL 3.0

- FIPS_mode_set() does not exist in OpenSSL 3.0 [1]
- X509_check_* functions declarated in openssl/x509v3.h instead of openssl/x509.h [2]
- X509_chack_* functions have const char arg inserad of const unsigned char [2]
- skip MD4 tests if it is unsupported by OpenSSL
- the patch does not change behavior under OpenSSL version != 3
- the patch just fixes build under OpenSSL 3.0 and doesn't update deprecated code
or behavior

1. https://wiki.openssl.org/index.php/OpenSSL_3.0#Upgrading_from_the_OpenSSL_2.0_FIPS_Object_Module
2. https://www.openssl.org/docs/man3.0/man3/X509_check_host.html

fips.go

@@ -16,16 +16,29 @@ package openssl
 
 /*
 #include <openssl/ssl.h>
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+	int FIPS_mode_set(int ONOFF) {
+		return 0;
+	}
+#endif
+
 */
 import "C"
+import "errors"
 import "runtime"
 
 // FIPSModeSet enables a FIPS 140-2 validated mode of operation.
 // https://wiki.openssl.org/index.php/FIPS_mode_set()
+// This call has been deleted from OpenSSL 3.0.
 func FIPSModeSet(mode bool) error {
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
 
+	if C.OPENSSL_VERSION_NUMBER >= 0x30000000 {
+		return errors.New("FIPS_mode_set() has been deleted from OpenSSL 3.0")
+	}
+
 	var r C.int
 	if mode {
 		r = C.FIPS_mode_set(1)

hostname.go

@@ -17,18 +17,26 @@ package openssl
 /*
 #include <openssl/ssl.h>
 #include <openssl/conf.h>
-#include <openssl/x509.h>
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+	#include <openssl/x509v3.h>
+	typedef const char x509char;
+#else
+	#include <openssl/x509.h>
 
-#ifndef X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT
-#define X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT	0x1
-#define X509_CHECK_FLAG_NO_WILDCARDS	0x2
+	#ifndef X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT
+		#define X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT	0x1
+		#define X509_CHECK_FLAG_NO_WILDCARDS	0x2
 
-extern int X509_check_host(X509 *x, const unsigned char *chk, size_t chklen,
-    unsigned int flags, char **peername);
-extern int X509_check_email(X509 *x, const unsigned char *chk, size_t chklen,
-    unsigned int flags);
-extern int X509_check_ip(X509 *x, const unsigned char *chk, size_t chklen,
-		unsigned int flags);
+		extern int X509_check_host(X509 *x, const unsigned char *chk, size_t chklen,
+			unsigned int flags, char **peername);
+		extern int X509_check_email(X509 *x, const unsigned char *chk, size_t chklen,
+			unsigned int flags);
+		extern int X509_check_ip(X509 *x, const unsigned char *chk, size_t chklen,
+			unsigned int flags);
+		typedef const unsigned char x509char;
+	#else
+		typedef const char x509char;
+	#endif
 #endif
 */
 import "C"
@@ -59,7 +67,7 @@ func (c *Certificate) CheckHost(host string, flags CheckFlags) error {
 	chost := unsafe.Pointer(C.CString(host))
 	defer C.free(chost)
 
-	rv := C.X509_check_host(c.x, (*C.uchar)(chost), C.size_t(len(host)),
+	rv := C.X509_check_host(c.x, (*C.x509char)(chost), C.size_t(len(host)),
 		C.uint(flags), nil)
 	if rv > 0 {
 		return nil
@@ -78,7 +86,7 @@ func (c *Certificate) CheckHost(host string, flags CheckFlags) error {
 func (c *Certificate) CheckEmail(email string, flags CheckFlags) error {
 	cemail := unsafe.Pointer(C.CString(email))
 	defer C.free(cemail)
-	rv := C.X509_check_email(c.x, (*C.uchar)(cemail), C.size_t(len(email)),
+	rv := C.X509_check_email(c.x, (*C.x509char)(cemail), C.size_t(len(email)),
 		C.uint(flags))
 	if rv > 0 {
 		return nil

md4_test.go

@@ -56,7 +56,19 @@ var md4Examples = []struct{ out, in string }{
 	{"6e593341e62194911d5cc31e39835f27", "c5e4bc73821faa34adf9468441ffd97520a96cd5debda4d51edcaaf2b23fbd"},
 }
 
+func skipIfMD4Unsupported(t testing.TB) {
+	t.Helper()
+
+	hash, err := NewMD4Hash()
+	if err != nil {
+		t.Skip("MD4 is not supported by OpenSSL")
+	}
+	hash.Close()
+}
+
 func TestMD4Examples(t *testing.T) {
+	skipIfMD4Unsupported(t)
+
 	for _, ex := range md4Examples {
 		buf, err := hex.DecodeString(ex.in)
 		if err != nil {
@@ -75,6 +87,8 @@ func TestMD4Examples(t *testing.T) {
 }
 
 func TestMD4Writer(t *testing.T) {
+	skipIfMD4Unsupported(t)
+
 	ohash, err := NewMD4Hash()
 	if err != nil {
 		t.Fatal(err)
@@ -120,9 +134,13 @@ func benchmarkMD4(b *testing.B, length int64, fn md4func) {
 }
 
 func BenchmarkMD4Large_openssl(b *testing.B) {
+	skipIfMD4Unsupported(b)
+
 	benchmarkMD4(b, 1024*1024, func(buf []byte) { MD4(buf) })
 }
 
 func BenchmarkMD4Small_openssl(b *testing.B) {
+	skipIfMD4Unsupported(b)
+
 	benchmarkMD4(b, 1, func(buf []byte) { MD4(buf) })
 }