Improve multiple issues, leaks, improve instrumentation with valgrind…

…, allow python to use debug mode for better debugging as a dependency.

cmake/CompileOptions.cmake

@@ -68,6 +68,15 @@ set(DEFAULT_INCLUDE_DIRECTORIES)
 # Libraries
 #
 
+# Valgrind
+if(OPTION_TEST_MEMORYCHECK)
+	set(MEMORYCHECK_COMPILE_DEFINITIONS
+		"__MEMORYCHECK__=1"
+	)
+else()
+	set(MEMORYCHECK_COMPILE_DEFINITIONS)
+endif()
+
 # ThreadSanitizer is incompatible with AddressSanitizer and LeakSanitizer
 if(OPTION_BUILD_THREAD_SANITIZER AND (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo"))
 	set(SANITIZER_LIBRARIES -ltsan)
@@ -136,6 +145,7 @@ set(DEFAULT_COMPILE_DEFINITIONS
 	LOG_POLICY_FORMAT_PRETTY=${LOG_POLICY_FORMAT_PRETTY_VALUE}
 	REFLECT_MEMORY_TRACKER=${REFLECT_MEMORY_TRACKER_VALUE}
 	SYSTEM_${SYSTEM_NAME_UPPER}
+	${MEMORYCHECK_COMPILE_DEFINITIONS}
 	${SANITIZER_COMPILE_DEFINITIONS}
 )
 

docker-compose.test.yml

@@ -24,6 +24,7 @@ services:
     image: metacall/core:deps
     build:
       args:
+        METACALL_BUILD_TYPE: ${METACALL_BUILD_TYPE}
         METACALL_INSTALL_OPTIONS: base python ruby netcore7 nodejs typescript file rpc wasm java c cobol go rust rapidjson funchook swig pack backtrace sandbox ${METACALL_BUILD_COVERAGE} # clangformat v8rep51
   dev:
     image: metacall/core:dev

docker-compose.yml

@@ -30,6 +30,7 @@ services:
         METACALL_BASE_IMAGE: $METACALL_BASE_IMAGE
         METACALL_PATH: $METACALL_PATH
         METACALL_TOOLS_PATH: $METACALL_PATH/tools
+        METACALL_BUILD_TYPE: $METACALL_BUILD_TYPE
         METACALL_INSTALL_OPTIONS: base python ruby nodejs typescript file rpc rapidjson funchook swig pack backtrace # clangformat v8rep51 coverage
     environment:
       DEBIAN_FRONTEND: noninteractive

source/cli/metacallcli/source/application.cpp

@@ -337,7 +337,7 @@ bool application::load_path(const char *path, void **handle)
 
 	if (plugin_path == NULL || plugin_extension_handle == NULL)
 	{
-		return NULL;
+		return false;
 	}
 
 	/* Define the cli plugin path as string (core plugin path plus the subpath) */

source/dynlink/source/dynlink_impl_beos.c

@@ -94,7 +94,7 @@ int dynlink_impl_interface_unload_beos(dynlink handle, dynlink_impl impl)
 {
 	(void)handle;
 
-#if defined(__ADDRESS_SANITIZER__) || defined(__THREAD_SANITIZER__) || defined(__MEMORY_SANITIZER__)
+#if defined(__MEMORYCHECK__) || defined(__ADDRESS_SANITIZER__) || defined(__THREAD_SANITIZER__) || defined(__MEMORY_SANITIZER__)
 	/* Disable dlclose when running with address sanitizer in order to maintain stacktraces */
 	(void)impl;
 	return 0;

source/dynlink/source/dynlink_impl_unix.c

@@ -114,8 +114,8 @@ int dynlink_impl_interface_unload_unix(dynlink handle, dynlink_impl impl)
 {
 	(void)handle;
 
-#if defined(__ADDRESS_SANITIZER__) || defined(__THREAD_SANITIZER__) || defined(__MEMORY_SANITIZER__)
-	/* Disable dlclose when running with address sanitizer in order to maintain stacktraces */
+#if defined(__MEMORYCHECK__) || defined(__ADDRESS_SANITIZER__) || defined(__THREAD_SANITIZER__) || defined(__MEMORY_SANITIZER__)
+	/* Disable dlclose when running with valgrind or sanitizers in order to maintain stacktraces */
 	(void)impl;
 	return 0;
 #else

source/loaders/py_loader/source/py_loader_impl.c

@@ -4206,6 +4206,7 @@ void py_loader_impl_sys_path_print(PyObject *sys_path_list)
 
 	log_write("metacall", LOG_LEVEL_DEBUG, sys_path_format_str, sys_path_str);
 
+	Py_XDECREF(sys_path_str_obj);
 	Py_XDECREF(separator);
 }
 #endif

source/ports/js_port/test/main.cpp

@@ -320,7 +320,7 @@ void ModulesClear()
 		 it != modules.end(); ++it)
 	{
 /* Disable dlclose when running with address sanitizer in order to maintain stacktraces */
-#if !defined(__ADDRESS_SANITIZER__) && !defined(__THREAD_SANITIZER__) && !defined(__MEMORY_SANITIZER__)
+#if !defined(__MEMORYCHECK__) && !defined(__ADDRESS_SANITIZER__) && !defined(__THREAD_SANITIZER__) && !defined(__MEMORY_SANITIZER__)
 	#if defined(JS_PORT_TEST_WIN)
 		FreeLibrary(it->second);
 	#elif defined(JS_PORT_TEST_UNIX)

source/serials/rapid_json_serial/source/rapid_json_serial_impl.cpp

@@ -12,6 +12,9 @@
 
 #include <log/log.h>
 
+// TODO: RapidJSON seems to be outdated, but we use it meanwhile there's a better solution.
+// Here's a patch for some of the bugs in the library: https://github.com/Tencent/rapidjson/issues/1928
+
 #include <rapidjson/document.h>
 #include <rapidjson/error/en.h>
 #include <rapidjson/stringbuffer.h>

source/tests/metacall_python_without_env_vars_test/CMakeLists.txt

@@ -115,6 +115,26 @@ target_link_libraries(${target}
 	${DEFAULT_LINKER_OPTIONS}
 )
 
+if(OPTION_BUILD_ADDRESS_SANITIZER OR OPTION_BUILD_THREAD_SANITIZER)
+	# TODO: This test leaks and I am not sure if it is a false positive or not:
+	#
+	# Direct leak of 18682 byte(s) in 12 object(s) allocated from:
+	#     #0 0x7fa978386bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
+	#     #1 0x7fa9734ddbb1 in _PyMem_RawMalloc ../Objects/obmalloc.c:101
+
+	# Indirect leak of 2775 byte(s) in 2 object(s) allocated from:
+	#     #0 0x7fa978386bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
+	#     #1 0x7fa9734ddbb1 in _PyMem_RawMalloc ../Objects/obmalloc.c:101
+
+	# SUMMARY: AddressSanitizer: 21457 byte(s) leaked in 14 allocation(s).
+	#
+	# Valgrind does not show anything:
+	# valgrind --tool=memcheck --leak-check=full --show-leak-kinds=possibly --track-origins=yes --num-callers=500 --suppressions=../source/tests/memcheck/valgrind-python.supp ./metacall-python-without-env-vars-testd
+	# For checking all the leaks (and including false positives), check:
+	# valgrind --tool=memcheck --leak-check=full --show-leak-kinds=all --track-origins=yes --num-callers=500 --suppressions=../source/tests/memcheck/valgrind-python.supp ./metacall-python-without-env-vars-testd
+	return()
+endif()
+
 #
 # Define test
 #

source/tests/metacall_test/source/metacall_test.cpp

@@ -280,7 +280,7 @@ TEST_F(metacall_test, DefaultConstructor)
 
 		metacall_value_destroy(ret);
 
-	#if !defined(__ADDRESS_SANITIZER__) && !defined(__THREAD_SANITIZER__) && !defined(__MEMORY_SANITIZER__)
+	#if !defined(__MEMORYCHECK__) && !defined(__ADDRESS_SANITIZER__) && !defined(__THREAD_SANITIZER__) && !defined(__MEMORY_SANITIZER__)
 		/* Testing corrupted value input */
 		struct
 		{

tools/deps/Dockerfile

@@ -48,8 +48,9 @@ WORKDIR $METACALL_PATH
 COPY tools/metacall-environment.sh tools/nobuildtest.patch $METACALL_TOOLS_PATH/
 
 # Install MetaCall and runtimes then build
+ARG METACALL_BUILD_TYPE
 ARG METACALL_INSTALL_OPTIONS
 
 RUN chmod 500 $METACALL_TOOLS_PATH/metacall-environment.sh \
-	&& $METACALL_TOOLS_PATH/metacall-environment.sh ${METACALL_INSTALL_OPTIONS} \
+	&& $METACALL_TOOLS_PATH/metacall-environment.sh ${METACALL_BUILD_TYPE} ${METACALL_INSTALL_OPTIONS} \
 	&& rm -rf $METACALL_PATH

tools/metacall-configure.sh

@@ -63,8 +63,8 @@ esac
 
 # Linux Distro detection
 if [ -f /etc/os-release ]; then # Either Debian or Ubuntu
-	# Cat file | Get the ID field | Remove 'ID=' | Remove leading and trailing spaces
-	LINUX_DISTRO=$(cat /etc/os-release | grep "^ID=" | cut -f2- -d= | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
+	# Cat file | Get the ID field | Remove 'ID=' | Remove leading and trailing spaces | Remove quotes
+	LINUX_DISTRO=$(cat /etc/os-release | grep "^ID=" | cut -f2- -d= | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -d '"')
 	# Cat file | Get the ID field | Remove 'ID=' | Remove leading and trailing spaces | Remove quotes
 	LINUX_VERSION_ID=$(cat /etc/os-release | grep "^VERSION_ID=" | cut -f2- -d= | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -d '"')
 else

tools/metacall-environment.sh

@@ -25,6 +25,7 @@ ROOT_DIR=$(pwd)
 
 APT_CACHE=0
 APT_CACHE_CMD=""
+BUILD_TYPE=Release
 INSTALL_BASE=1
 INSTALL_PYTHON=0
 INSTALL_RUBY=0
@@ -78,8 +79,8 @@ fi
 
 # Linux Distro detection
 if [ -f /etc/os-release ]; then # Either Debian or Ubuntu
-	# Cat file | Get the ID field | Remove 'ID=' | Remove leading and trailing spaces
-	LINUX_DISTRO=$(cat /etc/os-release | grep "^ID=" | cut -f2- -d= | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
+	# Cat file | Get the ID field | Remove 'ID=' | Remove leading and trailing spaces | Remove quotes
+	LINUX_DISTRO=$(cat /etc/os-release | grep "^ID=" | cut -f2- -d= | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -d '"')
 	# Cat file | Get the ID field | Remove 'ID=' | Remove leading and trailing spaces | Remove quotes
 	LINUX_VERSION_ID=$(cat /etc/os-release | grep "^VERSION_ID=" | cut -f2- -d= | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -d '"')
 else
@@ -146,7 +147,13 @@ sub_python(){
 
 	if [ "${OPERATIVE_SYSTEM}" = "Linux" ]; then
 		if [ "${LINUX_DISTRO}" = "debian" ] || [ "${LINUX_DISTRO}" = "ubuntu" ]; then
-			$SUDO_CMD apt-get $APT_CACHE_CMD install -y --no-install-recommends python3 python3-dev python3-pip
+			if [ "${BUILD_TYPE}" = "Debug" ]; then
+				PYTHON3_PKG=python3-dbg
+			else
+				PYTHON3_PKG=python3
+			fi
+
+			$SUDO_CMD apt-get $APT_CACHE_CMD install -y --no-install-recommends $PYTHON3_PKG python3-dev python3-pip
 
 			# Python test dependencies
 			$SUDO_CMD apt-get $APT_CACHE_CMD install -y --no-install-recommends \
@@ -926,136 +933,144 @@ sub_install(){
 
 # Configuration
 sub_options(){
-	for var in "$@"
+	for option in "$@"
 	do
-		if [ "$var" = 'cache' ]; then
+		if [ "$option" = 'cache' ]; then
 			if [ "${OPERATIVE_SYSTEM}" = "Linux" ]; then
 				echo "apt caching selected"
 				APT_CACHE=1
 			fi
 		fi
-		if [ "$var" = 'base' ]; then
+		if [ "$option" = 'debug' ]; then
+			echo "Install dependencies in debug mode"
+			BUILD_TYPE=Debug
+		fi
+		if [ "$option" = 'release' ] || [ "$option" = 'relwithdebinfo' ]; then
+			echo "Install dependencies release mode"
+			BUILD_TYPE=Release
+		fi
+		if [ "$option" = 'base' ]; then
 			echo "apt selected"
 			INSTALL_BASE=1
 		fi
-		if [ "$var" = 'python' ]; then
+		if [ "$option" = 'python' ]; then
 			echo "python selected"
 			INSTALL_PYTHON=1
 		fi
-		if [ "$var" = 'ruby' ]; then
+		if [ "$option" = 'ruby' ]; then
 			echo "ruby selected"
 			INSTALL_RUBY=1
 		fi
-		if [ "$var" = 'netcore' ]; then
+		if [ "$option" = 'netcore' ]; then
 			echo "netcore selected"
 			INSTALL_NETCORE=1
 		fi
-		if [ "$var" = 'netcore2' ]; then
+		if [ "$option" = 'netcore2' ]; then
 			echo "netcore 2 selected"
 			INSTALL_NETCORE2=1
 		fi
-		if [ "$var" = 'netcore5' ]; then
+		if [ "$option" = 'netcore5' ]; then
 			echo "netcore 5 selected"
 			INSTALL_NETCORE5=1
 		fi
-		if [ "$var" = 'netcore7' ]; then
+		if [ "$option" = 'netcore7' ]; then
 			echo "netcore 7 selected"
 			INSTALL_NETCORE7=1
 		fi
-		if [ "$var" = 'rapidjson' ]; then
+		if [ "$option" = 'rapidjson' ]; then
 			echo "rapidjson selected"
 			INSTALL_RAPIDJSON=1
 		fi
-		if [ "$var" = 'funchook' ]; then
+		if [ "$option" = 'funchook' ]; then
 			echo "funchook selected"
 			INSTALL_FUNCHOOK=1
 		fi
-		if [ "$var" = 'v8' ] || [ "$var" = 'v8rep54' ]; then
+		if [ "$option" = 'v8' ] || [ "$option" = 'v8rep54' ]; then
 			echo "v8 selected"
 			INSTALL_V8REPO=1
 			INSTALL_V8REPO54=1
 		fi
-		if [ "$var" = 'v8rep57' ]; then
+		if [ "$option" = 'v8rep57' ]; then
 			echo "v8 selected"
 			INSTALL_V8REPO=1
 			INSTALL_V8REPO57=1
 		fi
-		if [ "$var" = 'v8rep58' ]; then
+		if [ "$option" = 'v8rep58' ]; then
 			echo "v8 selected"
 			INSTALL_V8REPO=1
 			INSTALL_V8REPO58=1
 		fi
-		if [ "$var" = 'v8rep52' ]; then
+		if [ "$option" = 'v8rep52' ]; then
 			echo "v8 selected"
 			INSTALL_V8REPO=1
 			INSTALL_V8REPO52=1
 		fi
-		if [ "$var" = 'v8rep51' ]; then
+		if [ "$option" = 'v8rep51' ]; then
 			echo "v8 selected"
 			INSTALL_V8REPO=1
 			INSTALL_V8REPO51=1
 		fi
-		if [ "$var" = 'nodejs' ]; then
+		if [ "$option" = 'nodejs' ]; then
 			echo "nodejs selected"
 			INSTALL_NODEJS=1
 		fi
-		if [ "$var" = 'typescript' ]; then
+		if [ "$option" = 'typescript' ]; then
 			echo "typescript selected"
 			INSTALL_TYPESCRIPT=1
 		fi
-		if [ "$var" = 'file' ]; then
+		if [ "$option" = 'file' ]; then
 			echo "file selected"
 			INSTALL_FILE=1
 		fi
-		if [ "$var" = 'rpc' ]; then
+		if [ "$option" = 'rpc' ]; then
 			echo "rpc selected"
 			INSTALL_RPC=1
 		fi
-		if [ "$var" = 'wasm' ]; then
+		if [ "$option" = 'wasm' ]; then
 			echo "wasm selected"
 			INSTALL_WASM=1
 		fi
-		if [ "$var" = 'java' ]; then
+		if [ "$option" = 'java' ]; then
 			echo "java selected"
 			INSTALL_JAVA=1
 		fi
-		if [ "$var" = 'c' ]; then
+		if [ "$option" = 'c' ]; then
 			echo "c selected"
 			INSTALL_C=1
 		fi
-		if [ "$var" = 'cobol' ]; then
+		if [ "$option" = 'cobol' ]; then
 			echo "cobol selected"
 			INSTALL_COBOL=1
 		fi
-		if [ "$var" = 'go' ]; then
+		if [ "$option" = 'go' ]; then
 			echo "go selected"
 			INSTALL_GO=1
 		fi
-		if [ "$var" = 'rust' ]; then
+		if [ "$option" = 'rust' ]; then
 			echo "rust selected"
 			INSTALL_RUST=1
 		fi
-		if [ "$var" = 'swig' ]; then
+		if [ "$option" = 'swig' ]; then
 			echo "swig selected"
 			INSTALL_SWIG=1
 		fi
-		if [ "$var" = 'pack' ]; then
+		if [ "$option" = 'pack' ]; then
 			echo "pack selected"
 			INSTALL_PACK=1
 		fi
-		if [ "$var" = 'coverage' ]; then
+		if [ "$option" = 'coverage' ]; then
 			echo "coverage selected"
 			INSTALL_COVERAGE=1
 		fi
-		if [ "$var" = 'clangformat' ]; then
+		if [ "$option" = 'clangformat' ]; then
 			echo "clangformat selected"
 			INSTALL_CLANGFORMAT=1
 		fi
-		if [ "$var" = 'backtrace' ]; then
+		if [ "$option" = 'backtrace' ]; then
 			echo "backtrace selected"
 			INSTALL_BACKTRACE=1
 		fi
-		if [ "$var" = 'sandbox' ]; then
+		if [ "$option" = 'sandbox' ]; then
 			echo "sandbox selected"
 			INSTALL_SANDBOX=1
 		fi
@@ -1066,6 +1081,7 @@ sub_options(){
 sub_help() {
 	echo "Usage: `basename "$0"` list of component"
 	echo "Components:"
+	echo "	debug | release | relwithdebinfo"
 	echo "	cache"
 	echo "	base"
 	echo "	python"