From f2a575f07623026e9212fd3f963695f2b79c270a Mon Sep 17 00:00:00 2001
From: DRC <information@virtualgl.org>
Date: Tue, 2 Jan 2024 12:00:14 -0500
Subject: [PATCH] vncserver: Check a display #'s abstract socket

... before assuming that the display number is available.

Newer versions of GDM use Display :0 for the greeter and Display :1 for
the session if WaylandEnable=false.  However, X11 lock files are no
longer used, and the session's X server still functions if its Unix
domain socket (/tmp/.X11-unix/X1) is deleted.  Thus, if a user logged in
locally, accidentally deleted /tmp/.X11-unix/X1, then tried to start a
TurboVNC session, vncserver would attempt to use Display :1, and the
window manager in the TurboVNC session would subsequently fail to start.

Based on:
https://github.com/algorythmic/turbovnc/commit/1028467a2cc29f4e65589bee15ff649278a51347

Note that the aforementioned commit attempts to actually open the
Unix domain socket rather than checking for the existence of the socket
file.  However, I could find no cases in which the ability to open a
Unix domain socket was not determined by the existence of the socket
file.
---
 unix/vncserver.in | 35 +++++++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/unix/vncserver.in b/unix/vncserver.in
index acd198e25..a05ffdefe 100644
--- a/unix/vncserver.in
+++ b/unix/vncserver.in
@@ -663,9 +663,7 @@ sub CheckGeometryAndDepth
 
 
 #
-# GetDisplayNumber gets the lowest available display number.  A display number
-# n is taken if something is listening on the VNC server port (5900+n) or the
-# X server port (6000+n).
+# GetDisplayNumber gets the lowest available display number.
 #
 
 sub GetDisplayNumber
@@ -682,8 +680,15 @@ sub GetDisplayNumber
 
 #
 # CheckDisplayNumber checks if the given display number is available.  A
-# display number n is taken if something is listening on the VNC server port
-# (5900+n) or the X server port (6000+n).
+# display number $n is taken if any of the following are true:
+#
+# - something is listening on the X server port (6000+$n)
+# - something is listening on the VNC server port (5900+$n)
+# - -novnc/$noVNC is specified and something is listening on the web server
+#   port (5800+$n)
+# - (Linux only) the abstract socket \0/tmp/.X11-unix/X$n is in use
+# - the lock file /tmp/.X$n-lock exists
+# - the Unix domain socket /tmp/.X11-unix/X$n exists
 #
 
 sub CheckDisplayNumber
@@ -716,6 +721,16 @@ sub CheckDisplayNumber
     close(S);
   }
 
+  if ($os eq "Linux") {
+    socket(S, $AF_UNIX, $SOCK_STREAM, 0) || die "$prog: socket failed: $!\n";
+    unless (bind(S, pack_sockaddr_un("\0/tmp/.X11-unix/X$n"))) {
+      warn "\nWARNING: $host:$n is taken because abstract socket \\0/tmp/.X11-unix/X$n is in use.\n";
+      close(S);
+      return 0;
+    }
+    close(S);
+  }
+
   if (-e "/tmp/.X$n-lock") {
     warn "\nWARNING: $host:$n is taken because of /tmp/.X$n-lock\n";
     warn "Remove this file if there is no X server $host:$n\n";
@@ -1139,9 +1154,9 @@ sub SanityCheck
   # Find socket constants. 'use Socket' is a perl5-ism, so we wrap it in an
   # eval, and if it fails we try 'require "sys/socket.ph"'.  If this fails,
   # we just guess at the values.  If you find perl moaning here, just
-  # hard-code the values of AF_INET and SOCK_STREAM.  You can find these out
-  # for your platform by looking in /usr/include/sys/socket.h and related
-  # files.
+  # hard-code the values of AF_UNIX, AF_INET, and SOCK_STREAM.  You can find
+  # these out for your platform by looking in /usr/include/sys/socket.h and
+  # related files.
   #
 
   chop($os = `uname`);
@@ -1152,17 +1167,21 @@ sub SanityCheck
     eval 'require "sys/socket.ph"';
     if ($@) {
       if (($os eq "SunOS") && ($osrev !~ /^4/)) {
+        $AF_UNIX = 1;
         $AF_INET = 2;
         $SOCK_STREAM = 2;
       } else {
+        $AF_UNIX = 1;
         $AF_INET = 2;
         $SOCK_STREAM = 1;
       }
     } else {
+      $AF_UNIX = &AF_UNIX;
       $AF_INET = &AF_INET;
       $SOCK_STREAM = &SOCK_STREAM;
     }
   } else {
+    $AF_UNIX = &AF_UNIX;
     $AF_INET = &AF_INET;
     $SOCK_STREAM = &SOCK_STREAM;
   }