Not able to run multi proc OSError: [Errno 9] Bad file descriptor

[sripathivenky]

Keep running into OSError: [Errno 9] Bad file descriptor.
Torando Version 6.0.4
Mac OSX: 11.3 (20E232)
python --version --version
Python 2.7.16

server = HTTPServer(app) server.listen(8888) IOLoop.current().start()

or
server = HTTPServer(app) server.bind(8888) server.start(0) # Forks multiple sub-processes IOLoop.current().start()

Traceback (most recent call last):
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 261, in _add_reader
Traceback (most recent call last):
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 261, in _add_reader
key = self._selector.get_key(fd)
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/selectors.py", line 193, in get_key
key = self._selector.get_key(fd)
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/selectors.py", line 193, in get_key
raise KeyError("{!r} is not registered".format(fileobj)) from None
KeyError: '12 is not registered'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/Users/ssreepathi/Work/mmServices/code/mmannotations/app.py", line 8, in
raise KeyError("{!r} is not registered".format(fileobj)) from None
KeyError: '12 is not registered'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/Users/ssreepathi/Work/mmServices/code/mmannotations/app.py", line 8, in
main_app.run()
File "/Users/ssreepathi/Work/mmServices/code/mmcommon/base_application.py", line 95, in run
main_app.run()
File "/Users/ssreepathi/Work/mmServices/code/mmcommon/base_application.py", line 95, in run
self.server.start(self.server_config['procs'], max_restarts=2)
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/tcpserver.py", line 244, in start
self.server.start(self.server_config['procs'], max_restarts=2)
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/tcpserver.py", line 244, in start
self.add_sockets(sockets)
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/tcpserver.py", line 165, in add_sockets
self.add_sockets(sockets)
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/tcpserver.py", line 165, in add_sockets
self._handlers[sock.fileno()] = add_accept_handler(
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/netutil.py", line 279, in add_accept_handler
self._handlers[sock.fileno()] = add_accept_handler(
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/netutil.py", line 279, in add_accept_handler
io_loop.add_handler(sock, accept_handler, IOLoop.READ)
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/platform/asyncio.py", line 100, in add_handler
io_loop.add_handler(sock, accept_handler, IOLoop.READ)
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/platform/asyncio.py", line 100, in add_handler
self.asyncio_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 336, in add_reader
self.asyncio_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 336, in add_reader
self._add_reader(fd, callback, *args)
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 263, in _add_reader
self._add_reader(fd, callback, *args)
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 263, in _add_reader
self._selector.register(fd, selectors.EVENT_READ,
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/selectors.py", line 523, in register
self._selector.register(fd, selectors.EVENT_READ,
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/selectors.py", line 523, in register
self._selector.control([kev], 0, 0)
OSError: [Errno 9] Bad file descriptor
self._selector.control([kev], 0, 0)
OSError: [Errno 9] Bad file descriptor
child 0 (pid 73864) exited with status 1, restarting
child 1 (pid 73865) exited with status 1, restarting
Traceback (most recent call last):
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 261, in _add_reader
Traceback (most recent call last):
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 261, in _add_reader
key = self._selector.get_key(fd)
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/selectors.py", line 193, in get_key
raise KeyError("{!r} is not registered".format(fileobj)) from None
KeyError: '12 is not registered'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/Users/ssreepathi/Work/mmServices/code/mmannotations/app.py", line 8, in
key = self._selector.get_key(fd)
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/selectors.py", line 193, in get_key
main_app.run()
File "/Users/ssreepathi/Work/mmServices/code/mmcommon/base_application.py", line 95, in run
raise KeyError("{!r} is not registered".format(fileobj)) from None
KeyError: '12 is not registered'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/Users/ssreepathi/Work/mmServices/code/mmannotations/app.py", line 8, in
self.server.start(self.server_config['procs'], max_restarts=2)
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/tcpserver.py", line 244, in start
main_app.run()
File "/Users/ssreepathi/Work/mmServices/code/mmcommon/base_application.py", line 95, in run
self.add_sockets(sockets)
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/tcpserver.py", line 165, in add_sockets
self.server.start(self.server_config['procs'], max_restarts=2)
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/tcpserver.py", line 244, in start
self._handlers[sock.fileno()] = add_accept_handler(
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/netutil.py", line 279, in add_accept_handler
self.add_sockets(sockets)
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/tcpserver.py", line 165, in add_sockets
io_loop.add_handler(sock, accept_handler, IOLoop.READ)
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/platform/asyncio.py", line 100, in add_handler
self._handlers[sock.fileno()] = add_accept_handler(
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/netutil.py", line 279, in add_accept_handler
self.asyncio_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 336, in add_reader
io_loop.add_handler(sock, accept_handler, IOLoop.READ)
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/platform/asyncio.py", line 100, in add_handler
self._add_reader(fd, callback, *args)
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 263, in _add_reader
self.asyncio_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 336, in add_reader
self._selector.register(fd, selectors.EVENT_READ,
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/selectors.py", line 523, in register
self._add_reader(fd, callback, *args)
File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 263, in _add_reader
self._selector.control([kev], 0, 0)
self._selector.register(fd, selectors.EVENT_READ,
OSError: File "/usr/local/Cellar/python@3.9/3.9.5/Frameworks/Python.framework/Versions/3.9/lib/python3.9/selectors.py", line 523, in register
[Errno 9] Bad file descriptor
self._selector.control([kev], 0, 0)
OSError: [Errno 9] Bad file descriptor
child 0 (pid 73866) exited with status 1, restarting
Traceback (most recent call last):
File "/Users/ssreepathi/Work/mmServices/code/mmannotations/app.py", line 8, in
main_app.run()
File "/Users/ssreepathi/Work/mmServices/code/mmcommon/base_application.py", line 95, in run
self.server.start(self.server_config['procs'], max_restarts=2)
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/tcpserver.py", line 241, in start
process.fork_processes(num_processes, max_restarts)
File "/Users/ssreepathi/Work/mmServices/venv/lib/python3.9/site-packages/tornado/process.py", line 175, in fork_processes
raise RuntimeError("Too many child restarts, giving up")
RuntimeError: Too many child restarts, giving up

Process finished with exit code 1

[bdarnell]

You must call start(0) as early as possible in your application's startup - if you do anything that opens sockets or files (without immediately closing them) or interact with the event loop in any way, you may no longer be able to fork new processes. The three lines of code you've quoted are fine, but somewhere before them you must be doing something that interacts with the event loop.

Note that multi-process mode via start(0) is discouraged (see #2801). I recommend using another way to start your processes, either an external process manager like supervisord or using tornado.process.fork_processes (at the very beginning of your application).

[sripathivenky]

@bdarnell Thanks for the suggestion, I am sure nothing else is being done before start(0). Will double-check the flow and update.

[bdarnell]

Closing due to lack of updates; feel free to update with new information (or create a new issue).