diff --git a/CHANGELOG.md b/CHANGELOG.md index 67da00e418..e430dd4a08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,8 @@ - PR #3132: Add `min_samples_split` + Rename `min_rows_per_node` -> `min_samples_leaf` - PR #3136: Fix stochastic gradient descent example - PR #3156: Force local conda artifact install +- PR #3162: Removing accidentally checked in debug file + # cuML 0.16.0 (Date TBD) diff --git a/python/debug_out b/python/debug_out deleted file mode 100644 index 0e618f0d10..0000000000 --- a/python/debug_out +++ /dev/null @@ -1,9261 +0,0 @@ -============================= test session starts ============================== -platform linux -- Python 3.8.6, pytest-6.1.1, py-1.9.0, pluggy-0.13.1 -benchmark: 3.2.3 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000) -rootdir: /share/workspace/cuml/python, configfile: pytest.ini -plugins: asyncio-0.12.0, benchmark-3.2.3, timeout-1.4.2, hypothesis-5.37.3, cov-2.10.1 -collected 1 item - -cuml/test/test_nearest_neighbors.py ========= CUDA-MEMCHECK -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (399,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (159,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (79,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (319,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (239,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (359,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (199,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (119,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (39,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (279,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (158,0,0) -========= Address 0x00004e04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (278,0,0) -========= Address 0x00004604 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (77,0,0) -========= Address 0x00004d04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (317,0,0) -========= Address 0x00004d04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (157,0,0) -========= Address 0x00004d04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (357,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (197,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (117,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (37,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (277,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (356,0,0) -========= Address 0x00004404 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (196,0,0) -========= Address 0x00004404 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (116,0,0) -========= Address 0x00004404 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (36,0,0) -========= Address 0x00004404 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (276,0,0) -========= Address 0x00004404 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (235,0,0) -========= Address 0x00004b04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (155,0,0) -========= Address 0x00004b04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (115,0,0) -========= Address 0x00004304 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (35,0,0) -========= Address 0x00004304 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (275,0,0) -========= Address 0x00004304 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (354,0,0) -========= Address 0x00004204 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (114,0,0) -========= Address 0x00004204 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (34,0,0) -========= Address 0x00004204 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (274,0,0) -========= Address 0x00004204 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (312,0,0) -========= Address 0x00004804 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (232,0,0) -========= Address 0x00004804 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (391,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (71,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (311,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (231,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (151,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (351,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (111,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (271,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (191,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (150,0,0) -========= Address 0x00004604 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (190,0,0) -========= Address 0x00004e04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (309,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (229,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (308,0,0) -========= Address 0x00004404 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (228,0,0) -========= Address 0x00004404 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (106,0,0) -========= Address 0x00004a04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (266,0,0) -========= Address 0x00004a04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (186,0,0) -========= Address 0x00004a04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (265,0,0) -========= Address 0x00004904 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (185,0,0) -========= Address 0x00004904 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (383,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (223,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (143,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (63,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (303,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (183,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (382,0,0) -========= Address 0x00004e04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (342,0,0) -========= Address 0x00004604 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (102,0,0) -========= Address 0x00004604 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (262,0,0) -========= Address 0x00004604 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (182,0,0) -========= Address 0x00004604 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (101,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (261,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (181,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (219,0,0) -========= Address 0x00004b04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (59,0,0) -========= Address 0x00004b04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (378,0,0) -========= Address 0x00004a04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (138,0,0) -========= Address 0x00004a04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (58,0,0) -========= Address 0x00004a04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (258,0,0) -========= Address 0x00004204 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (178,0,0) -========= Address 0x00004204 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (215,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (55,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (295,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (335,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (95,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (255,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (175,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (373,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (213,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (133,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (53,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (253,0,0) -========= Address 0x00004d04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (332,0,0) -========= Address 0x00004c04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (172,0,0) -========= Address 0x00004c04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (92,0,0) -========= Address 0x00004c04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (252,0,0) -========= Address 0x00004c04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (331,0,0) -========= Address 0x00004b04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (251,0,0) -========= Address 0x00004b04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (171,0,0) -========= Address 0x00004b04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (91,0,0) -========= Address 0x00004b04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (370,0,0) -========= Address 0x00004204 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (210,0,0) -========= Address 0x00004204 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (130,0,0) -========= Address 0x00004204 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (50,0,0) -========= Address 0x00004204 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (290,0,0) -========= Address 0x00004204 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (330,0,0) -========= Address 0x00004a04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (250,0,0) -========= Address 0x00004a04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (170,0,0) -========= Address 0x00004a04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (90,0,0) -========= Address 0x00004a04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (329,0,0) -========= Address 0x00004904 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (249,0,0) -========= Address 0x00004904 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (169,0,0) -========= Address 0x00004904 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (89,0,0) -========= Address 0x00004904 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (367,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (207,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (127,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (47,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (287,0,0) -========= Address 0x00004f04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (167,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (87,0,0) -========= Address 0x00004704 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (366,0,0) -========= Address 0x00004e04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (46,0,0) -========= Address 0x00004e04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (286,0,0) -========= Address 0x00004e04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (166,0,0) -========= Address 0x00004604 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (285,0,0) -========= Address 0x00004d04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (325,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (245,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (165,0,0) -========= Address 0x00004504 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (364,0,0) -========= Address 0x00004c04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (204,0,0) -========= Address 0x00004c04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (124,0,0) -========= Address 0x00004c04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (44,0,0) -========= Address 0x00004c04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (284,0,0) -========= Address 0x00004c04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (84,0,0) -========= Address 0x00004404 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (244,0,0) -========= Address 0x00004404 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (164,0,0) -========= Address 0x00004404 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (123,0,0) -========= Address 0x00004b04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (43,0,0) -========= Address 0x00004b04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (163,0,0) -========= Address 0x00004304 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (362,0,0) -========= Address 0x00004a04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (42,0,0) -========= Address 0x00004a04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (282,0,0) -========= Address 0x00004a04 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (162,0,0) -========= Address 0x00004204 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (41,0,0) -========= Address 0x00004904 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Invalid __shared__ read of size 4 -========= at 0x00001a80 in /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) -========= by thread (0,0,0) in block (281,0,0) -========= Address 0x00004904 is out of bounds -========= Device Frame:/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:941:void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) (void MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout(int*, int, float*, int, int, float, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, MLCommon::Sparse::Distance::semiring_kernel_load_balanced_matvec_layout, float, int) : 0x1a80) -========= Saved host backtrace up to driver entry point at kernel launch time -========= Host Frame:/lib64/libcuda.so.1 (cuLaunchKernel + 0x34e) [0x2d725e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 [0xf62b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaLaunchKernel + 0x1c1) [0x4f5b1] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so [0x40054b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x3f5) [0x410dd5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/entid=0, out_row=0, iter=0 -tid=1, out_row=0, iter=0 -tid=0, out_row=0, iter=0 -tid=1, out_row=0, iter=0 -tid=0, out_row=1, iter=0 -tid=1, out_row=1, iter=0 -vs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaStreamSynchronize. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaStreamSynchronize + 0x144) [0x4f3b4] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x1a1) [0x410b81] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaGetLastError. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaGetLastError + 0x11f) [0x3773f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf + 0x4d9) [0x410eb9] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE + 0x257) [0x4107e7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x123b) [0x40a7ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaFree. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaFree + 0x130) [0x3e150] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN4raft2mr11buffer_baseIfNS0_6device9allocatorEED1Ev + 0xab) [0x1fa40b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x2bd5) [0x40c145] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x452) [0x135d92] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaFree. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaFree + 0x130) [0x3e150] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN4raft2mr11buffer_baseIfNS0_6device9allocatorEED1Ev + 0xab) [0x1fa40b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x2be1) [0x40c151] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x452) [0x135d92] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaFree. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaFree + 0x130) [0x3e150] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN4raft2mr11buffer_baseIiNS0_6device9allocatorEED1Ev + 0x17b) [0x20df1b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x2bed) [0x40c15d] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x452) [0x135d92] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaFree. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaFree + 0x130) [0x3e150] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN4raft2mr11buffer_baseIiNS0_6device9allocatorEED1Ev + 0x17b) [0x20df1b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x2bf9) [0x40c169] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x452) [0x135d92] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaFree. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaFree + 0x130) [0x3e150] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN4raft2mr11buffer_baseIfNS0_6device9allocatorEED1Ev + 0xab) [0x1fa40b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x2c05) [0x40c175] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x452) [0x135d92] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaFree. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaFree + 0x130) [0x3e150] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN4raft2mr11buffer_baseIiNS0_6device9allocatorEED1Ev + 0x17b) [0x20df1b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x2c11) [0x40c181] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x452) [0x135d92] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaFree. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaFree + 0x130) [0x3e150] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN4raft2mr11buffer_baseIfNS0_6device9allocatorEED1Ev + 0xab) [0x1fa40b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x2c1d) [0x40c18d] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x452) [0x135d92] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaFree. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaFree + 0x130) [0x3e150] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN4raft2mr11buffer_baseIiNS0_6device9allocatorEED1Ev + 0x17b) [0x20df1b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x2c29) [0x40c199] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x452) [0x135d92] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaFree. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaFree + 0x130) [0x3e150] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN4raft2mr11buffer_baseIiNS0_6device9allocatorEED1Ev + 0x17b) [0x20df1b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv + 0x2c35) [0x40c1a5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so (_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb + 0x1b4) [0x4002f4] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37968] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x452) [0x135d92] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaFree. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaFree + 0x130) [0x3e150] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/rmm/_lib/device_buffer.cpython-38-x86_64-linux-gnu.so [0x2b32d] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/memory.cpython-38-x86_64-linux-gnu.so [0x10c96] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/memory.cpython-38-x86_64-linux-gnu.so [0x10dfe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/core/core.cpython-38-x86_64-linux-gnu.so [0x18496] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10e2a5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1b23c7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10e2a5] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1b23c7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/memory.cpython-38-x86_64-linux-gnu.so [0x10c96] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/memory.cpython-38-x86_64-linux-gnu.so [0x10dfe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/core/core.cpython-38-x86_64-linux-gnu.so [0x18496] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10e275] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1b23c7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10e275] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1b23c7] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37c35] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaFree. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaFree + 0x130) [0x3e150] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/rmm/_lib/device_buffer.cpython-38-x86_64-linux-gnu.so [0x2b32d] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10e275] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1b23c7] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37c68] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x452) [0x135d92] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaFree. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaFree + 0x130) [0x3e150] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/rmm/_lib/device_buffer.cpython-38-x86_64-linux-gnu.so [0x2b32d] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10e275] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1b23c7] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x37c09] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x33d94] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2903c] -========= Host Frame:/share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so [0x2b9f7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168f30] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x2e9) [0x135c29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_MakeTpCall + 0x3bf) [0x12fa1f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x537a) [0x1e1a8a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/pythData created -Calling kneighbors -[D] [18:07:00.983655] /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/knn.cuh:160 n_query_rows=1000, n_idx_rows=1000 -[D] [18:07:00.983693] /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/knn.cuh:171 Beginning index batch 0 -[D] [18:07:00.983698] /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/knn.cuh:73 Setting batch. batch_start=0, batch_stop=999, batch_rows=1000 -[D] [18:07:00.983703] /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/knn.cuh:178 Slicing query CSR for batch. rows=1000 out of 1000 -[D] [18:07:00.989900] /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/knn.cuh:83 Computed batch offsets. stop_offset=3000, start_offset=0 -[D] [18:07:00.990702] /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/knn.cuh:207 Beginning query batch 0 -[D] [18:07:00.990713] /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/knn.cuh:73 Setting batch. batch_start=0, batch_stop=999, batch_rows=1000 -[D] [18:07:00.990803] /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/knn.cuh:217 Slicing index CSR for batch. rows=1000 out of 1000 -[D] [18:07:00.996629] /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/knn.cuh:83 Computed batch offsets. stop_offset=3000, start_offset=0 -[D] [18:07:00.997598] /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/knn.cuh:414 Computing pairwise distances for batch -[D] [18:07:00.997614] /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:1073 Running sparse pairwise distances with metric=3 -[D] [18:07:00.997618] /home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh:987 Running l1 dists -F - -=================================== FAILURES =================================== -_________ test_nearest_neighbors_sparse[20000-20000-4-0.3-10-1000-l1] __________ - -nrows = 1000, ncols = 10, density = 0.3, metric = 'l1', n_neighbors = 4 -batch_size_index = 20000, batch_size_query = 20000 - - @pytest.mark.parametrize("metric", ["l1"]) # "#valid_metrics(cuml_algo="sparse")) - @pytest.mark.parametrize('nrows', [1000]) - @pytest.mark.parametrize('ncols', [10]) - @pytest.mark.parametrize('density', [0.3]) - @pytest.mark.parametrize('n_neighbors', [4]) - @pytest.mark.parametrize('batch_size_index', [20000]) - @pytest.mark.parametrize('batch_size_query', [20000]) - def test_nearest_neighbors_sparse(nrows, ncols, - density, - metric, - n_neighbors, - batch_size_index, - batch_size_query): - - a = cp.sparse.random(nrows, ncols, format='csr', density=density, - random_state=32) - - print("Data created") - - logger.set_level(logger.level_debug) - nn = cuKNN(metric="l1", n_neighbors=n_neighbors, algorithm="brute", - verbose=logger.level_debug, - algo_params={"batch_size_index": batch_size_index, - "batch_size_query": batch_size_query}) - nn.fit(a) - - import time - - print("Calling kneighbors") - start = time.time() -> cuD, cuI = nn.kneighbors(a) - -cuml/test/test_nearest_neighbors.py:353: -_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ -cuml/neighbors/nearest_neighbors.pyx:364: in cuml.neighbors.nearest_neighbors.NearestNeighbors.kneighbors - return self._kneighbors(X, n_neighbors, return_distance, convert_dtype) -cuml/neighbors/nearest_neighbors.pyx:429: in cuml.neighbors.nearest_neighbors.NearestNeighbors._kneighbors - D_ndarr, I_ndarr = self._kneighbors_sparse(X, n_neighbors, -_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ - -> brute_force_knn(handle_[0], -E RuntimeError: CUDA error encountered at: file=/home/cjnolet/workspace/cuml/cpp/src_prims/sparse/distance.cuh line=993: call='cudaStreamSynchronize(config_.stream)', Reason=cudaErrorLaunchFailure:unspecified launch failure -E Obtained 64 stack frames -E #0 in /share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so(_ZN4raft9exception18collect_call_stackEv+0x33) [0x7f3323905413] -E #1 in /share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so(_ZN4raft10cuda_errorC2ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE+0x59) [0x7f3323905b59] -E #2 in /share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so(_ZN8MLCommon6Sparse8Distance14l1_distances_tIifE7computeEPf+0x599) [0x7f3323b66f79] -E #3 in /share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so(_ZN8MLCommon6Sparse8Distance16pairwiseDistanceIifEEvPT0_NS1_18distances_config_tIT_S3_EEN2ML8Distance12DistanceTypeE+0x257) [0x7f3323b667e7] -E #4 in /share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so(_ZN8MLCommon6Sparse9Selection12sparse_knn_tIifE3runEv+0x123b) [0x7f3323b607ab] -E #5 in /share/software/miniconda3/envs/cuml_017_101920/lib/libcuml++.so(_ZN2ML6Sparse15brute_force_knnERN4raft8handle_tEPKiS5_PKfmiiS5_S5_S7_miiPiPfimmNS_10MetricTypeEfb+0x1b4) [0x7f3323b562f4] -E #6 in /share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so(+0x37968) [0x7f3307700968] -E #7 in /share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so(+0x2903c) [0x7f33076f203c] -E #8 in /share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so(+0x33d94) [0x7f33076fcd94] -E #9 in /share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so(+0x2903c) [0x7f33076f203c] -E #10 in /share/workspace/cuml/python/cuml/neighbors/nearest_neighbors.cpython-38-x86_64-linux-gnu.so(+0x2b9f7) [0x7f33076f49f7] -E #11 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyObject_MakeTpCall+0x3bf) [0x55f9a1317a1f] -E #12 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(+0x168f30) [0x55f9a1350f30] -E #13 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x4e73) [0x55f9a13c9583] -E #14 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalCodeWithName+0x260) [0x55f9a13b0490] -E #15 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyFunction_Vectorcall+0x594) [0x55f9a13b1a14] -E #16 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(PyObject_Call+0x2e9) [0x55f9a131dc29] -E #17 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x1e6a) [0x55f9a13c657a] -E #18 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalCodeWithName+0x929) [0x55f9a13b0b59] -E #19 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyFunction_Vectorcall+0x594) [0x55f9a13b1a14] -E #20 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(PyObject_Call+0x7d) [0x55f9a131d9bd] -E #21 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x1e6a) [0x55f9a13c657a] -E #22 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalCodeWithName+0x929) [0x55f9a13b0b59] -E #23 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyFunction_Vectorcall+0x594) [0x55f9a13b1a14] -E #24 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x15a6) [0x55f9a13c5cb6] -E #25 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalCodeWithName+0x260) [0x55f9a13b0490] -E #26 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyFunction_Vectorcall+0x594) [0x55f9a13b1a14] -E #27 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x4e73) [0x55f9a13c9583] -E #28 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyFunction_Vectorcall+0x1b7) [0x55f9a13b1637] -E #29 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(+0x168dbe) [0x55f9a1350dbe] -E #30 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x4e73) [0x55f9a13c9583] -E #31 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalCodeWithName+0x260) [0x55f9a13b0490] -E #32 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyFunction_Vectorcall+0x594) [0x55f9a13b1a14] -E #33 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyObject_FastCallDict+0xe7) [0x55f9a13460d7] -E #34 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(+0x19c5ab) [0x55f9a13845ab] -E #35 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyObject_MakeTpCall+0x3bf) [0x55f9a1317a1f] -E #36 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x537a) [0x55f9a13c9a8a] -E #37 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyFunction_Vectorcall+0x1b7) [0x55f9a13b1637] -E #38 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x4bf) [0x55f9a13c4bcf] -E #39 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyFunction_Vectorcall+0x1b7) [0x55f9a13b1637] -E #40 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(PyObject_Call+0x7d) [0x55f9a131d9bd] -E #41 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x1e6a) [0x55f9a13c657a] -E #42 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalCodeWithName+0x929) [0x55f9a13b0b59] -E #43 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyFunction_Vectorcall+0x594) [0x55f9a13b1a14] -E #44 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x15a6) [0x55f9a13c5cb6] -E #45 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalCodeWithName+0x260) [0x55f9a13b0490] -E #46 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyFunction_Vectorcall+0x594) [0x55f9a13b1a14] -E #47 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x4e73) [0x55f9a13c9583] -E #48 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyFunction_Vectorcall+0x1b7) [0x55f9a13b1637] -E #49 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(+0x168dbe) [0x55f9a1350dbe] -E #50 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x4e73) [0x55f9a13c9583] -E #51 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalCodeWithName+0x260) [0x55f9a13b0490] -E #52 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyFunction_Vectorcall+0x594) [0x55f9a13b1a14] -E #53 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyObject_FastCallDict+0xe7) [0x55f9a13460d7] -E #54 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(+0x19c5ab) [0x55f9a13845ab] -E #55 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(PyObject_Call+0x452) [0x55f9a131dd92] -E #56 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x1e6a) [0x55f9a13c657a] -E #57 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalCodeWithName+0x929) [0x55f9a13b0b59] -E #58 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyFunction_Vectorcall+0x594) [0x55f9a13b1a14] -E #59 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x71a) [0x55f9a13c4e2a] -E #60 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalCodeWithName+0x260) [0x55f9a13b0490] -E #61 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyFunction_Vectorcall+0x594) [0x55f9a13b1a14] -E #62 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(+0x168dbe) [0x55f9a1350dbe] -E #63 in /share/software/miniconda3/envs/cuml_017_101920/bin/python(_PyEval_EvalFrameDefault+0x15a6) [0x55f9a13c5cb6] - -cuml/neighbors/nearest_neighbors.pyx:544: RuntimeError -=============================== warnings summary =============================== -:219 -:219 - :219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject - -../../../software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/sklearn/utils/deprecation.py:143 - /share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/sklearn/utils/deprecation.py:143: FutureWarning: The sklearn.datasets.samples_generator module is deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.datasets. Anything that cannot be imported from sklearn.datasets is now part of the private API. - warnings.warn(message, FutureWarning) - --- Docs: https://docs.pytest.org/en/stable/warnings.html -=========================== short test summary info ============================ -FAILED cuml/test/test_nearest_neighbors.py::test_nearest_neighbors_sparse[20000-20000-4-0.3-10-1000-l1] -================== 1 failed, 3 warnings in 217.70s (0:03:37) =================== -on (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4bf) [0x1dcbcf] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x1e6a) [0x1de57a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x929) [0x1c8b59] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x15a6) [0x1ddcb6] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x1b7) [0x1c9637] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x168dbe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalFrameDefault + 0x4e73) [0x1e1583] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyEval_EvalCodeWithName + 0x260) [0x1c8490] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyFunction_Vectorcall + 0x594) [0x1c9a14] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (_PyObject_FastCallDict + 0xe7) [0x15e0d7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x19c5ab] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x452) [0x135d92] -========= -========= Program hit CUDA_ERROR_LAUNCH_FAILED (error 719) due to "unspecified launch failure" on CUDA API call to cuModuleUnload. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 (cuModuleUnload + 0x184) [0x2ab584] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/driver.cpython-38-x86_64-linux-gnu.so [0xc7fb] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/function.cpython-38-x86_64-linux-gnu.so [0xa4aa] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/function.cpython-38-x86_64-linux-gnu.so [0x91de] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyDict_Clear + 0x14b) [0x10f51b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10f5b9] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/util.cpython-38-x86_64-linux-gnu.so [0xa156] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x13902b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x230532] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_FinalizeEx + 0x18b) [0x24f57b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_Exit + 0x8) [0x24f598] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f5db] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f620] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyRun_SimpleFileExFlags + 0x263) [0x252593] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_RunMain + 0x39e) [0x25299e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_BytesMain + 0x39) [0x252b99] -========= Host Frame:/lib64/libc.so.6 (__libc_start_main + 0xf5) [0x22505] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1f6eb9] -========= -========= Program hit CUDA_ERROR_LAUNCH_FAILED (error 719) due to "unspecified launch failure" on CUDA API call to cuModuleUnload. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 (cuModuleUnload + 0x184) [0x2ab584] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/driver.cpython-38-x86_64-linux-gnu.so [0xc7fb] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/function.cpython-38-x86_64-linux-gnu.so [0xa4aa] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/function.cpython-38-x86_64-linux-gnu.so [0x91de] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyDict_Clear + 0x14b) [0x10f51b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10f5b9] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/util.cpython-38-x86_64-linux-gnu.so [0xa156] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x13902b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x230532] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_FinalizeEx + 0x18b) [0x24f57b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_Exit + 0x8) [0x24f598] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f5db] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f620] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyRun_SimpleFileExFlags + 0x263) [0x252593] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_RunMain + 0x39e) [0x25299e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_BytesMain + 0x39) [0x252b99] -========= Host Frame:/lib64/libc.so.6 (__libc_start_main + 0xf5) [0x22505] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1f6eb9] -========= -========= Program hit CUDA_ERROR_LAUNCH_FAILED (error 719) due to "unspecified launch failure" on CUDA API call to cuModuleUnload. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 (cuModuleUnload + 0x184) [0x2ab584] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/driver.cpython-38-x86_64-linux-gnu.so [0xc7fb] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/function.cpython-38-x86_64-linux-gnu.so [0xa4aa] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/function.cpython-38-x86_64-linux-gnu.so [0x91de] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyDict_Clear + 0x14b) [0x10f51b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10f5b9] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/util.cpython-38-x86_64-linux-gnu.so [0xa156] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x13902b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x230532] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_FinalizeEx + 0x18b) [0x24f57b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_Exit + 0x8) [0x24f598] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f5db] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f620] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyRun_SimpleFileExFlags + 0x263) [0x252593] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_RunMain + 0x39e) [0x25299e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_BytesMain + 0x39) [0x252b99] -========= Host Frame:/lib64/libc.so.6 (__libc_start_main + 0xf5) [0x22505] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1f6eb9] -========= -========= Program hit CUDA_ERROR_LAUNCH_FAILED (error 719) due to "unspecified launch failure" on CUDA API call to cuModuleUnload. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 (cuModuleUnload + 0x184) [0x2ab584] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/driver.cpython-38-x86_64-linux-gnu.so [0xc7fb] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/function.cpython-38-x86_64-linux-gnu.so [0xa4aa] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/function.cpython-38-x86_64-linux-gnu.so [0x91de] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyDict_Clear + 0x14b) [0x10f51b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10f5b9] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/util.cpython-38-x86_64-linux-gnu.so [0xa156] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x13902b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyObject_Call + 0x7d) [0x1359bd] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x230532] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_FinalizeEx + 0x18b) [0x24f57b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_Exit + 0x8) [0x24f598] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f5db] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f620] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyRun_SimpleFileExFlags + 0x263) [0x252593] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_RunMain + 0x39e) [0x25299e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_BytesMain + 0x39) [0x252b99] -========= Host Frame:/lib64/libc.so.6 (__libc_start_main + 0xf5) [0x22505] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1f6eb9] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaFree. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaFree + 0x130) [0x3e150] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/runtime.cpython-38-x86_64-linux-gnu.so [0x1348b] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/memory.cpython-38-x86_64-linux-gnu.so [0x1449a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/memory.cpython-38-x86_64-linux-gnu.so [0x10dfe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/core/core.cpython-38-x86_64-linux-gnu.so [0x18496] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10e248] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1b23c7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10b60c] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10e13c] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10f145] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1fba29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x12e294] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x12ed24] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyGC_Collect + 0x4f) [0x2360df] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_FinalizeEx + 0x74) [0x24f464] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_Exit + 0x8) [0x24f598] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f5db] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f620] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyRun_SimpleFileExFlags + 0x263) [0x252593] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_RunMain + 0x39e) [0x25299e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_BytesMain + 0x39) [0x252b99] -========= Host Frame:/lib64/libc.so.6 (__libc_start_main + 0xf5) [0x22505] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1f6eb9] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaGetLastError. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaGetLastError + 0x11f) [0x3773f] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/runtime.cpython-38-x86_64-linux-gnu.so [0x106e4] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/runtime.cpython-38-x86_64-linux-gnu.so [0x134df] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/memory.cpython-38-x86_64-linux-gnu.so [0x1449a] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/memory.cpython-38-x86_64-linux-gnu.so [0x10dfe] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/core/core.cpython-38-x86_64-linux-gnu.so [0x18496] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10e248] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1b23c7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10b60c] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10e13c] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10f145] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1fba29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x12e294] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x12ed24] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyGC_Collect + 0x4f) [0x2360df] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_FinalizeEx + 0x74) [0x24f464] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_Exit + 0x8) [0x24f598] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f5db] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f620] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyRun_SimpleFileExFlags + 0x263) [0x252593] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_RunMain + 0x39e) [0x25299e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_BytesMain + 0x39) [0x252b99] -========= Host Frame:/lib64/libc.so.6 (__libc_start_main + 0xf5) [0x22505] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1f6eb9] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaEventDestroy. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaEventDestroy + 0x133) [0x3b2f3] -========= Host Frame:/share/workspace/cuml/python/cuml/raft/common/handle.cpython-38-x86_64-linux-gnu.so (_ZN4raft8handle_t17destroy_resourcesEv + 0x81) [0x33b31] -========= Host Frame:/share/workspace/cuml/python/cuml/raft/common/handle.cpython-38-x86_64-linux-gnu.so (_ZN4raft8handle_tD1Ev + 0x20) [0x342c0] -========= Host Frame:/share/workspace/cuml/python/cuml/raft/common/handle.cpython-38-x86_64-linux-gnu.so [0x2de71] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10e248] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1b23c7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10b60c] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10e13c] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10f145] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1fba29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x12e294] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x12ed24] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyGC_Collect + 0x4f) [0x2360df] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_FinalizeEx + 0x74) [0x24f464] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_Exit + 0x8) [0x24f598] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f5db] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f620] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyRun_SimpleFileExFlags + 0x263) [0x252593] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_RunMain + 0x39e) [0x25299e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_BytesMain + 0x39) [0x252b99] -========= Host Frame:/lib64/libc.so.6 (__libc_start_main + 0xf5) [0x22505] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1f6eb9] -========= -========= Program hit cudaErrorLaunchFailure (error 719) due to "unspecified launch failure" on CUDA API call to cudaGetLastError. -========= Saved host backtrace up to driver entry point at error -========= Host Frame:/lib64/libcuda.so.1 [0x3eef63] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/lib/python3.8/site-packages/cupy/cuda/../../../../libcudart.so.11.0 (cudaGetLastError + 0x11f) [0x3773f] -========= Host Frame:/share/workspace/cuml/python/cuml/raft/common/handle.cpython-38-x86_64-linux-gnu.so (_ZN4raft8handle_t17destroy_resourcesEv + 0x588) [0x34038] -========= Host Frame:/share/workspace/cuml/python/cuml/raft/common/handle.cpython-38-x86_64-linux-gnu.so (_ZN4raft8handle_tD1Ev + 0x20) [0x342c0] -========= Host Frame:/share/workspace/cuml/python/cuml/raft/common/handle.cpython-38-x86_64-linux-gnu.so [0x2de71] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10e248] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1b23c7] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10b60c] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10e13c] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x10f145] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1fba29] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x12e294] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x12ed24] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyGC_Collect + 0x4f) [0x2360df] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_FinalizeEx + 0x74) [0x24f464] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_Exit + 0x8) [0x24f598] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f5db] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x24f620] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (PyRun_SimpleFileExFlags + 0x263) [0x252593] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_RunMain + 0x39e) [0x25299e] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python (Py_BytesMain + 0x39) [0x252b99] -========= Host Frame:/lib64/libc.so.6 (__libc_start_main + 0xf5) [0x22505] -========= Host Frame:/share/software/miniconda3/envs/cuml_017_101920/bin/python [0x1f6eb9] -========= -========= Error: process didn't terminate successfully -========= No CUDA-MEMCHECK results found