-
Notifications
You must be signed in to change notification settings - Fork 568
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Tests which fail in multiprocessing contexts #1018
Changes from 2 commits
84108d8
be4ee65
b13601f
f4752c0
a08d359
6a0f8c8
55bb0db
cc0ac44
5c0848f
b88f3e9
a7a92b1
ae011e7
a443d03
d1a3a6d
4d3e992
48beaa4
baea777
f81e1a2
1434a2a
a818745
bcd5157
f74f2e1
c3ef4b7
9bbcecb
70d47b7
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
{ | ||
"cells": [ | ||
{ | ||
"cell_type": "code", | ||
"execution_count": null, | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [ | ||
"import time" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": null, | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [ | ||
"time.sleep(0.01)" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": null, | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [] | ||
} | ||
], | ||
"metadata": { | ||
"kernelspec": { | ||
"display_name": "Python 3", | ||
"language": "python", | ||
"name": "python3" | ||
}, | ||
"language_info": { | ||
"codemirror_mode": { | ||
"name": "ipython", | ||
"version": 3 | ||
}, | ||
"file_extension": ".py", | ||
"mimetype": "text/x-python", | ||
"name": "python", | ||
"nbconvert_exporter": "python", | ||
"pygments_lexer": "ipython3", | ||
"version": "3.7.0" | ||
} | ||
}, | ||
"nbformat": 4, | ||
"nbformat_minor": 2 | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -309,6 +309,74 @@ def test_parallel_notebooks(capfd, tmpdir): | |
captured = capfd.readouterr() | ||
assert captured.err == "" | ||
|
||
@pytest.mark.xfail | ||
def test_many_parallel_notebooks(capfd): | ||
"""Ensure that when many IPython kernels are run in parallel, nothing awful happens. | ||
|
||
Specifically, many IPython kernels when run simultaneously would enocunter errors | ||
due to using the same SQLite history database. | ||
""" | ||
|
||
# I've put timeout=5, which is a bit aggressive, but if I destroy the ZMQ context | ||
# below, timeout=5 works well enough. | ||
opts = dict(kernel_name="python", timeout=5) | ||
input_name = "HelloWorld.ipynb" | ||
input_file = os.path.join(current_dir, "files", input_name) | ||
res = PreprocessorTestsBase().build_resources() | ||
res["metadata"]["path"] = os.path.join(current_dir, "files") | ||
|
||
# run once, to trigger creating the original context | ||
run_notebook(input_file, opts, res) | ||
|
||
# Destroy the context - if you don't do this, the context | ||
# will survive across the fork, and then fail to start properly. | ||
import zmq | ||
zmq.Context.instance().destroy() | ||
|
||
with mp.Pool(4) as pool: | ||
pool.starmap(run_notebook, [(input_file, opts, res) for _ in range(8)]) | ||
|
||
captured = capfd.readouterr() | ||
assert captured.err == "" | ||
|
||
@pytest.mark.xfail | ||
def test_parallel_fork_notebooks(capfd): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I can't get this test to pass on my ubuntu 18.04 machine. It will always hang if the thread is running when multi-processing launches. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I tried with and without the zeromq cleanup, and with / without the jupyter_client pending release changes. Haven't dug in deeper as to why this still hangs. |
||
"""Ensure that when many IPython kernels are run in parallel, nothing awful happens. | ||
|
||
Specifically, many IPython kernels when run simultaneously would enocunter errors | ||
due to using the same SQLite history database. | ||
""" | ||
|
||
opts = dict(kernel_name="python", timeout=5) | ||
input_name = "Sleep One.ipynb" | ||
input_file = os.path.join(current_dir, "files", input_name) | ||
|
||
fast_name = "HelloWorld.ipynb" | ||
fast_file = os.path.join(current_dir, "files", fast_name) | ||
|
||
|
||
res = PreprocessorTestsBase().build_resources() | ||
res["metadata"]["path"] = os.path.join(current_dir, "files") | ||
|
||
# run once, to trigger creating the original context | ||
thread = threading.Thread(target=run_notebook, args=(input_file, opts, res)) | ||
thread.start() | ||
|
||
try: | ||
# Destroy the context - if you don't do this, the context | ||
# will survive across the fork, and then fail to start properly. | ||
# but if you do do this, then the context will get destroyed | ||
# while the kernel is running in the current thread. | ||
import zmq | ||
zmq.Context.instance().destroy() | ||
|
||
with mp.Pool(4) as pool: | ||
pool.starmap(run_notebook, [(fast_file, opts, res) for _ in range(8)]) | ||
finally: | ||
thread.join(timeout=1) | ||
|
||
captured = capfd.readouterr() | ||
assert captured.err == "" | ||
|
||
class TestExecute(PreprocessorTestsBase): | ||
"""Contains test functions for execute.py""" | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
FYI I tested this with the new
jupyter_client
. Could we add a TODO: delete whenjupyter_client>=5.2.4
releases?