updater_gpu_hist.cu:908: Exception in gpu_hist.cu

[kyoungrok0517]

Hello, I'm seeing the following error. I'm using CUDA 10.0, 2 GPUs (1 Titan-V, 1 1080 TI)

xgboost.core.XGBoostError: b'[23:05:47] /workspace/src/tree/updater_gpu_hist.cu:908: Exception in gpu_hist: [23:05:47] /workspace/include/xgboost/./../../src/common/common.h:41: 
/workspace/src/tree/updater_gpu_hist.cu: 279: invalid argument\n\nStack trace returned 10 entries:\n[bt] (0) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(dmlc::StackTrace()+0x3d) [0x7fda1e32ab0d]\n[bt] 
(1) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x18) [0x7fda1e32af08]\n[bt] 
(2) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(+0x34faa0) [0x7fda1e57aaa0]\n[bt] 
(3) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(xgboost::tree::DeviceHistogram::AllocateHistogram(int)+0x185) [0x7fda1e591695]\n[bt] 
(4) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(xgboost::tree::GPUHistMaker::BuildHistLeftRight(int, int, int)+0x28e) [0x7fda1e5925de]\n[bt] 
(5) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(xgboost::tree::GPUHistMaker::UpdateTree(xgboost::HostDeviceVector<xgboost::detail::GradientPairInternal<float> >*, xgboost::DMatrix*, xgboost::RegTree*)+0x12bb) [0x7fda1e59882b]\n[bt] 
(6) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(xgboost::tree::GPUHistMaker::Update(xgboost::HostDeviceVector<xgboost::detail::GradientPairInternal<float> >*, xgboost::DMatrix*, std::vector<xgboost::RegTree*, std::allocator<xgboost::RegTree*> > const&)+0x182) [0x7fda1e599562]\n[bt] 
(7) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(xgboost::gbm::GBTree::BoostNewTrees(xgboost::HostDeviceVector<xgboost::detail::GradientPairInternal<float> >*, xgboost::DMatrix*, int, std::vector<std::unique_ptr<xgboost::RegTree, std::default_delete<xgboost::RegTree> >, std::allocator<std::unique_ptr<xgboost::RegTree, std::default_delete<xgboost::RegTree> > > >*)+0x5a8) [0x7fda1e3fd298]\n[bt]
(8) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(xgboost::gbm::GBTree::DoBoost(xgboost::DMatrix*, xgboost::HostDeviceVector<xgboost::detail::GradientPairInternal<float> >*, xgboost::ObjFunction*)+0x358) [0x7fda1e3fdaa8]\n[bt]
(9) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(xgboost::LearnerImpl::UpdateOneIter(int, xgboost::DMatrix*)+0x3b8) [0x7fda1e3a1778]\n\n\n\n\n
Stack trace returned 10 entries:\n[bt] 
(0) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(dmlc::StackTrace()+0x3d) [0x7fda1e32ab0d]\n[bt] 
(1) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x18) [0x7fda1e32af08]\n[bt] 
(2) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(xgboost::tree::GPUHistMaker::Update(xgboost::HostDeviceVector<xgboost::detail::GradientPairInternal<float> >*, xgboost::DMatrix*, std::vector<xgboost::RegTree*, std::allocator<xgboost::RegTree*> > const&)+0x33f) [0x7fda1e59971f]\n[bt] 
(3) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(xgboost::gbm::GBTree::BoostNewTrees(xgboost::HostDeviceVector<xgboost::detail::GradientPairInternal<float> >*, xgboost::DMatrix*, int, std::vector<std::unique_ptr<xgboost::RegTree, std::default_delete<xgboost::RegTree> >, std::allocator<std::unique_ptr<xgboost::RegTree, std::default_delete<xgboost::RegTree> > > >*)+0x5a8) [0x7fda1e3fd298]\n[bt] 
(4) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(xgboost::gbm::GBTree::DoBoost(xgboost::DMatrix*, xgboost::HostDeviceVector<xgboost::detail::GradientPairInternal<float> >*, xgboost::ObjFunction*)+0x358) [0x7fda1e3fdaa8]\n[bt] 
(5) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(xgboost::LearnerImpl::UpdateOneIter(int, xgboost::DMatrix*)+0x3b8) [0x7fda1e3a1778]\n[bt] 
(6) /home/kyoungrok/anaconda3/xgboost/libxgboost.so(XGBoosterUpdateOneIter+0x35) [0x7fda1e322ff5]\n[bt] 
(7) /home/kyoungrok/anaconda3/lib/python3.7/lib-dynload/../../libffi.so.6(ffi_call_unix64+0x4c) [0x7fda70e15ec0]\n[bt] 
(8) /home/kyoungrok/anaconda3/lib/python3.7/lib-dynload/../../libffi.so.6(ffi_call+0x22d) [0x7fda70e1587d]\n[bt] 
(9) /home/kyoungrok/anaconda3/lib/python3.7/lib-dynload/_ctypes.cpython-37m-x86_64-linux-gnu.so(_ctypes_callproc+0x2ce) [0x7fda72bf5f3e]\n\n'

[trivialfis]

Most likely you ran out of GPU memory.

[kyoungrok0517]

Oh really?? Then I'll try with smaller batch then re-post if the error persists. Thanks for the immediate response :)