Magma and cuBLAS incompatible in unit tests

[ndellingwood]

Cuda builds with Magma and CuBlas are currently incompatible, resulting in the following compilation error:

17:08:00 /home/jenkins/weaver/workspace/KokkosKernels_Weaver_Cuda_Serial_cuda_1180_gcc_850_cusparse_cublas_magma/kokkos-kernels/tpls/gtest/gtest/gtest.h(18865): error: operand types are incompatible ("magma_queue_t *" and "cublasHandle_t *")
17:08:00           detected during:
17:08:00             instantiation of "testing::AssertionResult testing::internal::CmpHelperEQ(const char *, const char *, const T1 &, const T2 &) [with T1=magma_queue_t *, T2=cublasHandle_t *]" 
17:08:00 (18902): here
17:08:00             instantiation of "testing::AssertionResult testing::internal::EqHelper<lhs_is_null_literal>::Compare(const char *, const char *, const T1 &, const T2 &) [with lhs_is_null_literal=false, T1=magma_queue_t *, T2=cublasHandle_t *]" 
17:08:00 /home/jenkins/weaver/workspace/KokkosKernels_Weaver_Cuda_Serial_cuda_1180_gcc_850_cusparse_cublas_magma/kokkos-kernels/batched/dense/unit_test/Test_Batched_BatchedGemm.hpp(236): here
17:08:00             instantiation of "void Test::impl_test_batched_gemm<DeviceType,ViewType,ScalarType,ParamTagType>(int, int, int, int, int, int, int) [with DeviceType=Kokkos::Serial, ViewType=Kokkos::View<double ***, Kokkos::LayoutLeft, Kokkos::Serial>, ScalarType=double, ParamTagType=Test::SharedParamTag<KokkosBlas::Trans::NoTranspose, KokkosBlas::Trans::NoTranspose, KokkosBatched::BatchLayout::Right>]" 
17:08:00 /home/jenkins/weaver/workspace/KokkosKernels_Weaver_Cuda_Serial_cuda_1180_gcc_850_cusparse_cublas_magma/kokkos-kernels/batched/dense/unit_test/Test_Batched_BatchedGemm.hpp(320): here
17:08:00             instantiation of "void test_batched_gemm_with_layout<ViewType,DeviceType,ValueType,ScalarType,ParamTagType>(int) [with ViewType=Kokkos::View<double ***, Kokkos::LayoutLeft, Kokkos::Serial>, DeviceType=Kokkos::Serial, ValueType=double, ScalarType=double, ParamTagType=Test::SharedParamTag<KokkosBlas::Trans::NoTranspose, KokkosBlas::Trans::NoTranspose, KokkosBatched::BatchLayout::Right>]" 
17:08:00 /home/jenkins/weaver/workspace/KokkosKernels_Weaver_Cuda_Serial_cuda_1180_gcc_850_cusparse_cublas_magma/kokkos-kernels/batched/dense/unit_test/Test_Batched_BatchedGemm.hpp(388): here
17:08:00             instantiation of "int test_batched_gemm<DeviceType,ValueType,ScalarType,ParamTagType>() [with DeviceType=Kokkos::Serial, ValueType=double, ScalarType=double, ParamTagType=Test::SharedParamTag<KokkosBlas::Trans::NoTranspose, KokkosBlas::Trans::NoTranspose, KokkosBatched::BatchLayout::Right>]" 
17:08:00 /home/jenkins/weaver/workspace/KokkosKernels_Weaver_Cuda_Serial_cuda_1180_gcc_850_cusparse_cublas_magma/kokkos-kernels/batched/dense/unit_test/Test_Batched_BatchedGemm_Real.hpp(260): here
17:08:00 
17:08:01 1 error detected in the compilation of "/home/jenkins/weaver/workspace

Reproducer (weaver):

module load gcc/8.5.0 cuda/11.8.0 cmake magma/2.7.0 openblas/0.3.23

${KOKKOSKERNELS_PATH}/cm_generate_makefile.bash --with-cuda --with-serial --compiler=${KOKKOS_PATH}/bin/nvcc_wrapper --arch=Volta70,Power9 --with-cuda-options=enable_lambda --kokkos-path=${KOKKOS_PATH} --kokkoskernels-path=${KOKKOSKERNELS_PATH} ${DEBUG} --cxxflags=\"${CXXFLAGS}\" --with-scalars='double,complex_double' --with-ordinals=int --with-offsets=int,size_t --with-layouts=LayoutLeft --with-tpls=cusparse,cublas,magma

[ndellingwood]

This is the culprit region in the BatchedGemmHandle for the unit tests:

kokkos-kernels/batched/dense/src/KokkosBatched_HostLevel_Gemm_Handle.hpp

Lines 99 to 152 in cb824ca

	BatchedGemmHandle(int kernelAlgoType = BaseHeuristicAlgos::SQUARE,
	int teamSize = 0, int vecLength = 0)
	: BatchedKernelHandle(kernelAlgoType, teamSize, vecLength) {
	#if defined(KOKKOSKERNELS_ENABLE_TPL_CUBLAS)
	if (!_tplParamsSet && kernelAlgoType == GemmTplAlgos::CUBLAS) {
	static cublasHandle_t cublas_handle;
	_tplParamsSingleton.cublas_handle = &cublas_handle;
	_tplParamsSet = true;
	}
	#endif // CUBLAS
	#if defined(KOKKOSKERNELS_ENABLE_TPL_MAGMA)
	if (!_tplParamsSet && kernelAlgoType == GemmTplAlgos::MAGMA) {
	static magma_queue_t magma_queue;
	_tplParamsSingleton.magma_queue = &magma_queue;
	_tplParamsSet = true;
	}
	#endif // MAGMA
	};
	BatchedGemmHandle(bool tplParamsSet,
	int kernelAlgoType = BaseHeuristicAlgos::SQUARE,
	int teamSize = 0, int vecLength = 0)
	: BatchedKernelHandle(kernelAlgoType, teamSize, vecLength) {
	_tplParamsSet = tplParamsSet;
	};
	#if defined(KOKKOSKERNELS_ENABLE_TPL_CUBLAS)
	BatchedGemmHandle(cublasHandle_t &cublas_handle,
	int kernelAlgoType = BaseHeuristicAlgos::SQUARE,
	int teamSize = 0, int vecLength = 0)
	: BatchedGemmHandle(true, kernelAlgoType, teamSize, vecLength) {
	_tplParamsSingleton.cublas_handle = &cublas_handle;
	};
	#endif // CUBLAS
	#if defined(KOKKOSKERNELS_ENABLE_TPL_MAGMA)
	BatchedGemmHandle(magma_queue_t &magma_queue,
	int kernelAlgoType = BaseHeuristicAlgos::SQUARE,
	int teamSize = 0, int vecLength = 0)
	: BatchedGemmHandle(true, kernelAlgoType, teamSize, vecLength) {
	_tplParamsSingleton.magma_queue = &magma_queue;
	};
	#endif // MAGMA
	decltype(auto) get_tpl_params() {
	#if _kernelAlgoType == CUBLAS && defined(KOKKOSKERNELS_ENABLE_TPL_CUBLAS)
	return _tplParamsSingleton.cublas_handle;
	#elif _kernelAlgoType == MAGMA && defined(KOKKOSKERNELS_ENABLE_TPL_MAGMA)
	return _tplParamsSingleton.magma_queue;
	#else
	return this->BatchedKernelHandle::get_tpl_params();
	#endif
	}

[lucbv]

Thanks for catching this, we should go clean it up.
Setting up the build with MAGMA is going to be very helpful!

[ndellingwood]

Adding a note that #2181 has a temporary workaround in the unit test to allow compiling tests with both magma and cublas enabled. I think the BatchedGemmHandle will need some refactoring in order to remove the workaround