rpctest: make test TCP ports unique per process

This commit adds a new NextAvailablePortForProcess function that takes a
process ID and then assures unique (non-occupied) port numbers are
returned per process.
This uses a temporary file that contains the latest used port and a
secondary temporary lock file to assure only a single goroutine can
request a new port at a time.

The GenerateProcessUniqueListenerAddresses is intened to be used as a
package-level override for the ListenAddressGenerator variable. We don't
use it by default to make sure we don't break any existing assumptions.

integration/rpctest/rpc_harness.go

@@ -561,6 +561,111 @@ func NextAvailablePort() int {
 	panic("no ports available for listening")
 }
 
+// NextAvailablePortForProcess returns the first port that is available for
+// listening by a new node, using a lock file to make sure concurrent access for
+// parallel tasks within the same process don't re-use the same port. It panics
+// if no port is found and the maximum available TCP port is reached.
+func NextAvailablePortForProcess(pid int) int {
+	lockFile := filepath.Join(
+		os.TempDir(), fmt.Sprintf("rpctest-port-pid-%d.lock", pid),
+	)
+	timeout := time.After(time.Second)
+
+	var (
+		lockFileHandle *os.File
+		err error
+	)
+	for {
+		// Attempt to acquire the lock file. If it already exists, wait
+		// for a bit and retry.
+		lockFileHandle, err = os.OpenFile(
+			lockFile, os.O_CREATE|os.O_EXCL, 0600,
+		)
+		if err == nil {
+			// Lock acquired.
+			break
+		}
+
+		// Wait for a bit and retry.
+		select {
+		case <-timeout:
+			panic("timeout waiting for lock file")
+		case <-time.After(10 * time.Millisecond):
+		}
+	}
+
+	// Release the lock file when we're done.
+	defer func() {
+		// Always close file first, Windows won't allow us to remove it
+		// otherwise.
+		_ = lockFileHandle.Close()
+		err := os.Remove(lockFile)
+		if err != nil {
+			panic(fmt.Errorf("couldn't remove lock file: %w", err))
+		}
+	}()
+
+	portFile := filepath.Join(
+		os.TempDir(), fmt.Sprintf("rpctest-port-pid-%d", pid),
+	)
+	port, err := os.ReadFile(portFile)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			panic(fmt.Errorf("error reading port file: %w", err))
+		}
+		port = []byte(strconv.Itoa(int(defaultNodePort)))
+	}
+
+	lastPort, err := strconv.Atoi(string(port))
+	if err != nil {
+		panic(fmt.Errorf("error parsing port: %w", err))
+	}
+
+	// We take the next one.
+	lastPort++
+	for lastPort < 65535 {
+		// If there are no errors while attempting to listen on this
+		// port, close the socket and return it as available. While it
+		// could be the case that some other process picks up this port
+		// between the time the socket is closed and it's reopened in
+		// the harness node, in practice in CI servers this seems much
+		// less likely than simply some other process already being
+		// bound at the start of the tests.
+		addr := fmt.Sprintf(ListenerFormat, lastPort)
+		l, err := net.Listen("tcp4", addr)
+		if err == nil {
+			err := l.Close()
+			if err == nil {
+				err := os.WriteFile(
+					portFile,
+					[]byte(strconv.Itoa(lastPort)), 0600,
+				)
+				if err != nil {
+					panic(fmt.Errorf("error updating "+
+						"port file: %w", err))
+				}
+
+				return lastPort
+			}
+		}
+		lastPort++
+	}
+
+	// No ports available? Must be a mistake.
+	panic("no ports available for listening")
+}
+
+// GenerateProcessUniqueListenerAddresses is a function that returns two
+// listener addresses with unique ports per the given process id and should be
+// used to overwrite rpctest's default generator which is prone to use colliding
+// ports.
+func GenerateProcessUniqueListenerAddresses(pid int) (string, string) {
+	port1 := NextAvailablePortForProcess(pid)
+	port2 := NextAvailablePortForProcess(pid)
+	return fmt.Sprintf(ListenerFormat, port1),
+		fmt.Sprintf(ListenerFormat, port2)
+}
+
 // baseDir is the directory path of the temp directory for all rpctest files.
 func baseDir() (string, error) {
 	dirPath := filepath.Join(os.TempDir(), "btcd", "rpctest")