Test that uses paramiko-ng hangs with many threads stuck in paramiko code

[harixxxx]

Multithreaded test that uses paramiko-ng hangs forever. The stacktraces below are for the threads that are in paramiko code. This has happened at least a couple of times.

2023-02-14 03:26:51 (UTC) | 23913 |    DEBUG :         | ThreadPoolExecutor-2_0 | Stacktrace of <paramiko.Transport at 0xcc102a20 (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>:
  File "/usr/lib/python3.7/threading.py", line 885, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.7/threading.py", line 917, in _bootstrap_inner
    self.run()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py", line 1920, in run
    ptype, m = self.packetizer.read_message()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 437, in read_message
    header = self.read_all(self.__block_size_in, check_rekey=True)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 281, in read_all
    x = self.__socket.recv(n)
2023-02-14 03:26:51 (UTC) | 23913 |    DEBUG :         | ThreadPoolExecutor-2_0 | Stacktrace of <paramiko.Transport at 0xcb74a518 (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>:
  File "/usr/lib/python3.7/threading.py", line 885, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.7/threading.py", line 917, in _bootstrap_inner
    self.run()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py", line 1920, in run
    ptype, m = self.packetizer.read_message()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 437, in read_message
    header = self.read_all(self.__block_size_in, check_rekey=True)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 281, in read_all
    x = self.__socket.recv(n)
2023-02-14 03:26:51 (UTC) | 23913 |    DEBUG :         | ThreadPoolExecutor-2_0 | Stacktrace of <paramiko.Transport at 0xcc4ec630 (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>:
  File "/usr/lib/python3.7/threading.py", line 885, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.7/threading.py", line 917, in _bootstrap_inner
    self.run()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py", line 1920, in run
    ptype, m = self.packetizer.read_message()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 437, in read_message
    header = self.read_all(self.__block_size_in, check_rekey=True)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 281, in read_all
    x = self.__socket.recv(n)
2023-02-14 03:26:51 (UTC) | 23913 |    DEBUG :         | ThreadPoolExecutor-2_0 | Stacktrace of <paramiko.Transport at 0xcc4f5588 (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>:
  File "/usr/lib/python3.7/threading.py", line 885, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.7/threading.py", line 917, in _bootstrap_inner
    self.run()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py", line 1920, in run
    ptype, m = self.packetizer.read_message()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 437, in read_message
    header = self.read_all(self.__block_size_in, check_rekey=True)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 281, in read_all
    x = self.__socket.recv(n)
2023-02-14 03:26:51 (UTC) | 23913 |    DEBUG :         | ThreadPoolExecutor-2_0 | Stacktrace of <ResultStoringThread(Thread-23, started daemon 139933004044032)>:
  File "/usr/lib/python3.7/threading.py", line 885, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.7/threading.py", line 917, in _bootstrap_inner
    self.run()
 ...
  File "/home/os76/slc5-hm10-lp/workspace/harbormaster_functional-test/purity_harbormaster/tools/pure/lib/test_helpers.py", line 2027, in _remote_helper
    si, so, se = paramiko_client.retry_exec(command_string, timeout=timeout)
  File "/home/os76/slc5-hm10-lp/workspace/harbormaster_functional-test/purity_harbormaster/tools/pure/lib/machines/machine.py", line 390, in retry_exec
    return self.exec_command(command_string, timeout=timeout, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/client.py", line 433, in exec_command
    chan = self._transport.open_session(timeout=timeout)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py", line 772, in open_session
    timeout=timeout)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py", line 888, in open_channel
    self._send_user_message(m)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py", line 1738, in _send_user_message
    self._send_message(data)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py", line 1717, in _send_message
    self.packetizer.send_message(data)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 406, in send_message
    self.__mac_engine_out)[:self.__mac_size_out]
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 42, in compute_hmac
    return HMAC(key, message, digest_class).digest()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/file.py", line 66, in __del__
    self.close()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/channel.py", line 1404, in close
    self.channel.shutdown_write()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/channel.py", line 980, in shutdown_write
    self.shutdown(1)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/channel.py", line 956, in shutdown
    self.transport._send_user_message(m, self._close_not_sent)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py", line 1730, in _send_user_message
    self.clear_to_send_lock.acquire()
2023-02-14 03:26:51 (UTC) | 23913 |    DEBUG :         | ThreadPoolExecutor-2_0 | Stacktrace of <paramiko.Transport at 0xcc564470 (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>:
  File "/usr/lib/python3.7/threading.py", line 885, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.7/threading.py", line 917, in _bootstrap_inner
    self.run()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py", line 1950, in run
    self._channel_handler_table[ptype](chan, m)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/channel.py", line 1179, in _handle_close
    self._close_internal_send(*msgs)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/channel.py", line 1260, in _close_internal_send
    self.transport._send_user_message(close, self._set_close_sent)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py", line 1730, in _send_user_message
    self.clear_to_send_lock.acquire()
2023-02-14 03:26:51 (UTC) | 23913 |    DEBUG :         | ThreadPoolExecutor-2_0 | Stacktrace of <paramiko.Transport at 0xcc559208 (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>:
  File "/usr/lib/python3.7/threading.py", line 885, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.7/threading.py", line 917, in _bootstrap_inner
    self.run()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py", line 1920, in run
    ptype, m = self.packetizer.read_message()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 437, in read_message
    header = self.read_all(self.__block_size_in, check_rekey=True)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 281, in read_all
    x = self.__socket.recv(n)
2023-02-14 03:26:51 (UTC) | 23913 |    DEBUG :         | ThreadPoolExecutor-2_0 | Stacktrace of <paramiko.Transport at 0xcc3272b0 (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>:
  File "/usr/lib/python3.7/threading.py", line 885, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.7/threading.py", line 917, in _bootstrap_inner
    self.run()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py", line 1920, in run
    ptype, m = self.packetizer.read_message()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 437, in read_message
    header = self.read_all(self.__block_size_in, check_rekey=True)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 281, in read_all
    x = self.__socket.recv(n)
2023-02-14 03:26:51 (UTC) | 23913 |    DEBUG :         | ThreadPoolExecutor-2_0 | Stacktrace of <paramiko.Transport at 0xcc2e67f0 (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>:
  File "/usr/lib/python3.7/threading.py", line 885, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.7/threading.py", line 917, in _bootstrap_inner
    self.run()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py", line 1920, in run
    ptype, m = self.packetizer.read_message()
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 437, in read_message
    header = self.read_all(self.__block_size_in, check_rekey=True)
  File "/usr/local/lib/python3.7/dist-packages/paramiko/packet.py", line 281, in read_all
    x = self.__socket.recv(n)