Add wolfSSL support

- Add WITH_WOLFSSL option to cmake
- Gate out functions with potential missing wolfSSL API
- Update github action to test wolfSSL

.github/workflows/ci.yml

@@ -5,6 +5,7 @@ on: [push, pull_request]
 jobs:
   build:
     strategy:
+      fail-fast: false
       matrix:
         os:
           - ubuntu-latest
@@ -14,6 +15,7 @@ jobs:
           - "-DWITH_OPENSSL=ON  -DWITH_GNUTLS=OFF -DWITH_GCRYPT=OFF" # build with OpenSSL
           - "-DWITH_OPENSSL=OFF -DWITH_GNUTLS=ON  -DWITH_GCRYPT=ON"  # build with GnuTLS and Libgrypt
           - "-DWITH_OPENSSL=OFF -DWITH_GNUTLS=OFF -DWITH_GCRYPT=OFF" # build without external encryption libraries
+          - "-DWITH_OPENSSL=OFF -DWITH_GNUTLS=OFF -DWITH_GCRYPT=OFF -DWITH_WOLFSSL=ON" # build with wolfSSL
           - "-DCMAKE_TOOLCHAIN_FILE=../cmake/Toolchain-cross-mingw32-linux.cmake" # crosscompile with MinGW toolchain
         include:
           - os: macos-latest
@@ -31,29 +33,29 @@ jobs:
             cmake_options: "-DWITH_OPENSSL=OFF -DWITH_GNUTLS=ON  -DWITH_GCRYPT=ON"  # don't have GnuTLS and libgcrypt on Windows (yet)
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Ubuntu Build Dependencies
         if: ${{ matrix.os == 'ubuntu-latest' }}
         run: |
           sudo apt update
-          sudo apt install libsdl2-dev liblzo2-dev libssl-dev gnutls-dev libgcrypt-dev mingw-w64-x86-64-dev binutils-mingw-w64-x86-64 gcc-mingw-w64-x86-64 wine
+          sudo apt install libsdl2-dev liblzo2-dev libssl-dev gnutls-dev libgcrypt-dev libwolfssl-dev mingw-w64-x86-64-dev binutils-mingw-w64-x86-64 gcc-mingw-w64-x86-64 wine
       - name: Install MacOS Build Dependencies
         if: ${{ matrix.os == 'macos-latest' }}
         run: |
           unset HOMEBREW_NO_INSTALL_FROM_API
           brew untap homebrew/core homebrew/cask
           brew update
-          brew install sdl2 lzo
+          brew install sdl2 lzo wolfssl
       - name: Install Windows Build Dependencies
         if: ${{ matrix.os == 'windows-latest' }}
         run: |
-          vcpkg install zlib libjpeg-turbo libpng --triplet=x64-windows # could install more but should use run-vcpkg with caching for this
+          vcpkg install zlib libjpeg-turbo libpng pkgconf wolfssl --triplet=x64-windows # could install more but should use run-vcpkg with caching for this
       - name: Build
         run: |
           mkdir build
           cd build
           cmake ${{ matrix.cmake_options }} ${{ matrix.macos_cmake_options }} ${{ matrix.windows_cmake_options }} ..
-          cmake --build .
+          cmake --build . --verbose
       - name: Prepare Test
         if: ${{ matrix.os == 'ubuntu-latest' }} # only ubuntu does crosscompile with MinGW toolchain
         run: |

CMakeLists.txt

@@ -64,6 +64,7 @@ option(WITH_THREADS "Search for a threading library to build with multithreading
 option(PREFER_WIN32THREADS "When searching for a threading library, prefer win32 threads if they are found" OFF)
 option(WITH_GNUTLS "Search for the GnuTLS secure communications library to support TLS" ON)
 option(WITH_OPENSSL "Search for the OpenSSL cryptography library to support TLS and use as crypto backend" ON)
+option(WITH_WOLFSSL "Search for the wolfSSL cryptography library to support TLS and use as crypto backend" OFF)
 option(WITH_SYSTEMD "Search for libsystemd to build with systemd socket activation support" ON)
 option(WITH_GCRYPT "Search for Libgcrypt to use as crypto backend" ON)
 option(WITH_FFMPEG "Search for FFMPEG to build an example VNC to MPEG encoder" ON)
@@ -182,6 +183,14 @@ if(WITH_OPENSSL)
   find_package(OpenSSL)
 endif(WITH_OPENSSL)
 
+if(WITH_WOLFSSL)
+  find_package(PkgConfig)
+  pkg_check_modules(WOLFSSL "wolfssl")
+  if(WOLFSSL_FOUND)
+    link_directories(${WOLFSSL_LIBRARY_DIRS})
+  endif(WOLFSSL_FOUND)
+endif(WITH_WOLFSSL)
+
 
 if(WITH_SYSTEMD AND NOT ANDROID AND NOT WIN32)
   find_package(PkgConfig)
@@ -275,6 +284,9 @@ endif(PNG_FOUND)
 if(NOT OPENSSL_FOUND)
     unset(OPENSSL_LIBRARIES) # would otherwise contain -NOTFOUND, confusing target_link_libraries()
 endif()
+if(NOT WOLFSSL_FOUND)
+  unset(WOLFSSL_LIBRARIES) # would otherwise contain -NOTFOUND, confusing target_link_libraries()
+endif()
 if(SYSTEMD_FOUND)
   add_definitions(-DLIBVNCSERVER_WITH_SYSTEMD)
   include_directories(${SYSTEMD_INCLUDE_DIRS})
@@ -290,6 +302,10 @@ elseif(OPENSSL_FOUND)
   message(STATUS "Building crypto with OpenSSL")
   set(CRYPTO_LIBRARIES ${OPENSSL_LIBRARIES})
   set(CRYPTO_SOURCES ${COMMON_DIR}/crypto_openssl.c)
+elseif(WOLFSSL_FOUND)
+  message(STATUS "Building crypto with wolfSSL")
+  set(CRYPTO_LIBRARIES ${WOLFSSL_LIBRARIES})
+  set(CRYPTO_SOURCES ${COMMON_DIR}/crypto_openssl.c)
 else()
   message(STATUS "Building crypto with builtin functions, only including SHA1 and D3DES")
   set(CRYPTO_SOURCES ${COMMON_DIR}/crypto_included.c ${COMMON_DIR}/sha1.c ${COMMON_DIR}/d3des.c)
@@ -312,6 +328,13 @@ if(OPENSSL_FOUND)
   set(LIBVNCSERVER_HAVE_LIBSSL 1)
 endif(OPENSSL_FOUND)
 
+if(WOLFSSL_FOUND)
+  include_directories("${WOLFSSL_INCLUDE_DIRS}" "${WOLFSSL_INCLUDEDIR}/wolfssl")
+  set(LIBVNCSERVER_HAVE_LIBWOLFSSL 1)
+  set(LIBVNCSERVER_HAVE_LIBSSL 1)
+  add_definitions(-DEXTERNAL_OPTS_OPENVPN -DOPENSSL_VERSION_NUMBER=0x10001040L)
+endif(WOLFSSL_FOUND)
+
 if(WITH_IPv6)
   if(WIN32 AND LIBVNCSERVER_HAVE_WS2TCPIP_H AND LIBVNCSERVER_HAVE_VPRINTF)
     set(LIBVNCSERVER_IPv6 1)
@@ -448,6 +471,17 @@ elseif(OPENSSL_FOUND)
     ${LIBVNCSERVER_DIR}/rfbssl_openssl.c
   )
   include_directories(${OPENSSL_INCLUDE_DIR})
+elseif(WOLFSSL_FOUND)
+  message(STATUS "Building TLS with wolfSSL")
+  set(LIBVNCCLIENT_SOURCES
+    ${LIBVNCCLIENT_SOURCES}
+    ${LIBVNCCLIENT_DIR}/tls_openssl.c
+  )
+  set(LIBVNCSERVER_SOURCES
+    ${LIBVNCSERVER_SOURCES}
+    ${LIBVNCSERVER_DIR}/rfbssl_openssl.c
+  )
+  include_directories("${WOLFSSL_INCLUDE_DIRS}" "${WOLFSSL_INCLUDEDIR}/wolfssl")
 else()
   message(STATUS "Building without TLS")
   set(LIBVNCCLIENT_SOURCES
@@ -548,6 +582,7 @@ target_link_libraries(vncclient
 		      ${CRYPTO_LIBRARIES}
                       ${GNUTLS_LIBRARIES}
                       ${OPENSSL_LIBRARIES}
+                      ${WOLFSSL_LIBRARIES}
 )
 target_link_libraries(vncserver
                       ${ADDITIONAL_LIBS}
@@ -558,6 +593,7 @@ target_link_libraries(vncserver
 		      ${CRYPTO_LIBRARIES}
                       ${GNUTLS_LIBRARIES}
                       ${OPENSSL_LIBRARIES}
+                      ${WOLFSSL_LIBRARIES}
 )
 
 SET_TARGET_PROPERTIES(vncclient vncserver

include/rfb/rfbconfig.h.cmakein

@@ -160,6 +160,9 @@
 /* Define to 1 if OpenSSL is present */
 #cmakedefine LIBVNCSERVER_HAVE_LIBSSL 1
 
+/* Define to 1 if wolfSSL is present */
+#cmakedefine LIBVNCSERVER_HAVE_LIBWOLFSSL 1
+
 /* Define to 1 if Cyrus SASL is present */
 #cmakedefine LIBVNCSERVER_HAVE_SASL 1
 

src/common/crypto_openssl.c

@@ -42,6 +42,10 @@ static unsigned char reverseByte(unsigned char b) {
 
 int hash_md5(void *out, const void *in, const size_t in_len)
 {
+#if defined(LIBVNCSERVER_HAVE_LIBWOLFSSL) && defined(NO_MD5)
+    /* wolfSSL in MacOS brew doesn't have md5 enabled */
+    return 0;
+#else
     MD5_CTX md5;
     if(!MD5_Init(&md5))
 	return 0;
@@ -50,6 +54,7 @@ int hash_md5(void *out, const void *in, const size_t in_len)
     if(!MD5_Final(out, &md5))
 	return 0;
     return 1;
+#endif
 }
 
 int hash_sha1(void *out, const void *in, const size_t in_len)
@@ -71,6 +76,10 @@ void random_bytes(void *out, size_t len)
 
 int encrypt_rfbdes(void *out, int *out_len, const unsigned char key[8], const void *in, const size_t in_len)
 {
+#if defined(LIBVNCSERVER_HAVE_LIBWOLFSSL) && (defined(NO_DES3) || !defined(WOLFSSL_DES_ECB))
+    /* wolfSSL in MacOS brew doesn't have des enabled */
+    return 0;
+#else
     int result = 0;
     EVP_CIPHER_CTX *des = NULL;
     unsigned char mungedkey[8];
@@ -110,10 +119,15 @@ int encrypt_rfbdes(void *out, int *out_len, const unsigned char key[8], const vo
       OSSL_PROVIDER_unload(providerDefault);
 #endif
     return result;
+#endif
 }
 
 int decrypt_rfbdes(void *out, int *out_len, const unsigned char key[8], const void *in, const size_t in_len)
 {
+#if defined(LIBVNCSERVER_HAVE_LIBWOLFSSL) && (defined(NO_DES3) || !defined(WOLFSSL_DES_ECB))
+    /* wolfSSL in MacOS brew doesn't have des enabled */
+    return 0;
+#else
     int result = 0;
     EVP_CIPHER_CTX *des = NULL;
     unsigned char mungedkey[8];
@@ -155,10 +169,15 @@ int decrypt_rfbdes(void *out, int *out_len, const unsigned char key[8], const vo
 	OSSL_PROVIDER_unload(providerDefault);
 #endif
     return result;
+#endif
 }
 
 int encrypt_aes128ecb(void *out, int *out_len, const unsigned char key[16], const void *in, const size_t in_len)
 {
+#if defined(LIBVNCSERVER_HAVE_LIBWOLFSSL) && (!defined(HAVE_AES_ECB) || !defined(WOLFSSL_AES_128))
+    /* wolfSSL in MacOS brew doesn't have aes-ecb enabled */
+    return 0;
+#else
     int result = 0;
     EVP_CIPHER_CTX *aes;
 
@@ -175,6 +194,7 @@ int encrypt_aes128ecb(void *out, int *out_len, const unsigned char key[16], cons
  out:
     EVP_CIPHER_CTX_free(aes);
     return result;
+#endif
 }
 
 static void pad_leading_zeros(uint8_t *out, const size_t current_len, const size_t expected_len) {

src/libvncclient/tls_openssl.c

@@ -212,6 +212,10 @@ static int wait_for_data(SSL *ssl, int ret, int timeout)
 static rfbBool
 load_crls_from_file(char *file, SSL_CTX *ssl_ctx)
 {
+#if defined(LIBVNCSERVER_HAVE_LIBWOLFSSL) && (!defined(OPENSSL_ALL) || defined(NO_BIO))
+  /* wolfSSL in Windows vcpkg is missing API */
+  return 0;
+#else
   X509_STORE *st;
   int i;
   int count = 0;
@@ -245,6 +249,7 @@ load_crls_from_file(char *file, SSL_CTX *ssl_ctx)
     return TRUE;
   else
     return FALSE;
+#endif
 }
 
 static SSL *
@@ -350,7 +355,10 @@ open_ssl_connection (rfbClient *client, int sockfd, rfbBool anonTLS, rfbCredenti
   }
 
   SSL_set_fd (ssl, sockfd);
+#if !defined(LIBVNCSERVER_HAVE_LIBWOLFSSL) || defined(OPENSSL_ALL)
+  /* wolfSSL in Windows vcpkg is missing API */
   SSL_CTX_set_app_data (ssl_ctx, client);
+#endif
 
   do
   {