Server: Allow anon TLS key length to be specified

This provides a backward compatibility path for
b47cfd1.

ChangeLog.md

@@ -48,7 +48,10 @@ running on a RHEL 8 host.  It also fixed an error
 ("javax.net.ssl.SSLHandshakeException: DHPublicKey does not comply to algorithm
 constraints") that occurred when attempting to connect, using one of the TLS*
 security types, to a TurboVNC session with the Linux TurboVNC Viewer running on
-a RHEL 8 client.
+a RHEL 8 client.  A new security configuration file directive
+(`tls-key-length`) can be used to restore the behavior of previous releases of
+TurboVNC (generating a 1024-bit DSA key) or to increase the key length for
+additional security.
 
 
 2.2.2

unix/Xvnc/programs/Xserver/Xvnc.man.in

@@ -662,6 +662,10 @@ order in which Xvnc will advertise the corresponding RFB security types and
 authentication schemes to the VNC viewer.  This ordering may affect which
 security type the VNC viewer chooses as its default.
 
+.IP \fItls-key-length\fR
+This specifies the length of the key, in bits, that the TurboVNC Server will
+generate for any of the TLS* (anonymous TLS) security types.
+
 .SH SECURITY CONCERNS
 .P
 Even when used with encryption, there are other security problems inherent in

unix/Xvnc/programs/Xserver/hw/vnc/auth.c

@@ -641,6 +641,22 @@ static void ReadConfigFile(void)
 
       continue;
     }
+
+    n = 15;
+    if (!strncmp(buf2, "tls-key-length=", n)) {
+      int t;
+
+      if (buf2[n] == '\0')
+        FatalError("ERROR in %s: tls-key-length is empty!",
+                   rfbAuthConfigFile);
+
+      if (sscanf(&buf2[n], "%d", &t) < 1 || t <= 0)
+        FatalError("ERROR in %s: tls-key-length value must be > 0!",
+                   rfbAuthConfigFile);
+
+      rfbTLSKeyLength = (CARD32)t;
+      continue;
+    }
 #endif
 
     n = 17;

unix/Xvnc/programs/Xserver/hw/vnc/rfb.h

@@ -884,6 +884,8 @@ extern void rfbSendServerCutText(char *str, int len);
 
 /* rfbssl_*.c */
 
+extern CARD32 rfbTLSKeyLength;
+
 rfbSslCtx *rfbssl_init(rfbClientPtr cl, Bool anon);
 int rfbssl_accept(rfbClientPtr cl);
 int rfbssl_pending(rfbClientPtr cl);

unix/Xvnc/programs/Xserver/hw/vnc/rfbssl_gnutls.c

@@ -26,9 +26,10 @@
 #include <gnutls/gnutls.h>
 #include <errno.h>
 
+CARD32 rfbTLSKeyLength = 2048;
+
 
 #define BUFSIZE 1024
-#define DH_BITS 2048
 
 
 static char errStr[BUFSIZE] = "No error";
@@ -88,7 +89,7 @@ rfbSslCtx *rfbssl_init(rfbClientPtr cl, Bool anon)
     rfbssl_error("gnutls_dh_params_init()", ret);
     goto bailout;
   }
-  if ((ret = gnutls_dh_params_generate2(ctx->dh_params, DH_BITS)) !=
+  if ((ret = gnutls_dh_params_generate2(ctx->dh_params, rfbTLSKeyLength)) !=
       GNUTLS_E_SUCCESS) {
     rfbssl_error("gnutls_dh_params_generate2()", ret);
     goto bailout;

unix/Xvnc/programs/Xserver/hw/vnc/rfbssl_openssl.c

@@ -34,6 +34,8 @@
 
 static void rfbErr(const char *format, ...);
 
+CARD32 rfbTLSKeyLength = 2048;
+
 
 #define BUFSIZE 1024
 
@@ -61,6 +63,7 @@ typedef struct ossl_init_settings_st OPENSSL_INIT_SETTINGS;
 
 typedef void (*DH_free_type) (DH *);
 typedef int (*DH_generate_key_type) (DH *);
+typedef int (*DH_size_type) (const DH *dh);
 typedef DH *(*DSA_dup_DH_type) (const DSA *);
 typedef void (*DSA_free_type) (DSA *);
 typedef int (*DSA_generate_parameters_ex_type) (DSA *, int,
@@ -74,6 +77,7 @@ typedef char *(*ERR_error_string_type) (unsigned long, char *);
 struct rfbcrypto_functions {
   DH_free_type DH_free;
   DH_generate_key_type DH_generate_key;
+  DH_size_type DH_size;
   DSA_dup_DH_type DSA_dup_DH;
   DSA_free_type DSA_free;
   DSA_generate_parameters_ex_type DSA_generate_parameters_ex;
@@ -86,8 +90,8 @@ static struct rfbcrypto_functions crypto = {
 #ifdef DLOPENSSL
   NULL
 #else
-  DH_free, DH_generate_key, DSA_dup_DH, DSA_free, DSA_generate_parameters_ex,
-  DSA_new, ERR_get_error, ERR_error_string
+  DH_free, DH_generate_key, DH_size, DSA_dup_DH, DSA_free,
+  DSA_generate_parameters_ex, DSA_new, ERR_get_error, ERR_error_string
 #endif
 };
 
@@ -290,6 +294,7 @@ static int loadFunctions(void)
     }
 #endif
     LOADSYM(crypto, DH_free);
+    LOADSYM(crypto, DH_size);
     LOADSYM(crypto, DSA_dup_DH);
     LOADSYM(crypto, DSA_free);
     LOADSYM(crypto, DSA_generate_parameters_ex);
@@ -378,8 +383,8 @@ rfbSslCtx *rfbssl_init(rfbClientPtr cl, Bool anon)
       rfbssl_error("DSA_new()");
       goto bailout;
     }
-    if (!crypto.DSA_generate_parameters_ex(dsa, 2048, NULL, 0, NULL, NULL,
-                                           NULL)) {
+    if (!crypto.DSA_generate_parameters_ex(dsa, rfbTLSKeyLength, NULL, 0, NULL,
+                                           NULL, NULL)) {
       rfbssl_error("DSA_generate_paramters_ex()");
       goto bailout;
     }
@@ -392,6 +397,7 @@ rfbSslCtx *rfbssl_init(rfbClientPtr cl, Bool anon)
       rfbssl_error("DH_generate_key()");
       goto bailout;
     }
+    rfbLog("Anonymous TLS key length: %d bits\n", crypto.DH_size(dh) * 8);
     if (!ssl.SSL_CTX_ctrl(ctx->ssl_ctx, SSL_CTRL_SET_TMP_DH, 0, (char *)dh)) {
       rfbssl_error("SSL_CTX_set_tmp_dh()");
       goto bailout;

unix/turbovncserver-security.conf

@@ -96,3 +96,9 @@
 # (Default: TLSVnc, TLSOtp, TLSPlain, TLSNone, X509Vnc, X509Otp, X509Plain, X509None, VNC, OTP, UnixLogin, Plain, None)
 
 #permitted-security-types = TLSVnc, TLSOtp, TLSPlain, X509Vnc, X509Otp, X509Plain, VNC, OTP, UnixLogin, Plain
+
+# This specifies the length of the key, in bits, that the TurboVNC Server will
+# generate for any of the TLS* (anonymous TLS) security types.
+# (Default: 2048)
+
+#tls-key-length = 2048