Memory debugging

CMakeLists.txt

@@ -371,7 +371,8 @@ if(MEMORYCHECK_COMMAND)
     add_custom_command(
       OUTPUT ${logfile}
       COMMAND ${CMAKE_COMMAND}
-        -E env PYTHONMALLOC=malloc ${MEMORYCHECK_COMMAND}
+        -E env PYTHONMALLOC=malloc DLITE_ATEXIT_FREE=
+        -- ${MEMORYCHECK_COMMAND}
         --leak-check=yes
         --show-leak-kinds=all
         --keep-debuginfo=yes
@@ -405,11 +406,24 @@ if(MEMORYCHECK_COMMAND)
   set(MEMORYCHECK_SUPPRESSIONS_FILE
     ${suppfile}
     CACHE FILEPATH "File that contains suppressions for the memory checker"
-    )
+  )
+
+  # Generate a wrapper for ctest that allows to passing arguments from the
+  # environment to ctest.  Useful for running a single test, like
+  #
+  #     CTEST_ARGS="-R test_compat" make memcheck
+  #
+  configure_file(
+    ${dlite_SOURCE_DIR}/cmake/ctest.sh
+    ${dlite_BINARY_DIR}/ctest.sh
+    @ONLY
+    #USE_SOURCE_PERMISSIONS  # available from cmake 3.20
+  )
 
   add_custom_target(memcheck
     COMMAND ${CMAKE_COMMAND}
-        -E env PYTHONMALLOC=malloc ${CMAKE_CTEST_COMMAND}
+        -E env PYTHONMALLOC=malloc DLITE_ATEXIT_FREE=
+            -- ${dlite_BINARY_DIR}/ctest.sh
             --exclude-regex static-code-analysis
             --force-new-ctest-process
             --output-on-failure

cmake/ctest.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+@CMAKE_CTEST_COMMAND@ $CTEST_ARGS $@

cmake/valgrind-python.supp

@@ -634,6 +634,29 @@
    fun:Py_BytesMain
 }
 
+{
+   Leak in numpy random_bit_generator (py311, alvis)
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:calloc
+   obj:/usr/lib64/libgomp.so.1.0.0
+   obj:/usr/lib64/libgomp.so.1.0.0
+   obj:/usr/lib64/libgomp.so.1.0.0
+   fun:call_init
+   fun:call_init
+   fun:_dl_init
+   fun:_dl_catch_exception
+}
+
+{
+   Leak in pydantic initialisation (py311)
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:realloc
+   ...
+   fun:PyInit__pydantic_core
+}
+
 
 # ------------------------------------
 # Python plugins

doc/contributors_guide/tips_and_tricks.md

@@ -3,7 +3,19 @@ Tips and Tricks
 
 Setting up a virtual Python environment for building dlite
 ----------------------------------------------------------
-See the [Build and install to a virtual Python environment] in the installation instructions.
+See [Build against Python environment] in the installation instructions.
+
+
+Debugging Python storage plugins
+--------------------------------
+Exceptions occurring inside Python storage plugins are not propagated to the calling interpreter, and will therefor not be shown.
+However, it is possible to write them to stderr by setting the `DLITE_PYDEBUG` environment variable.
+
+For example, run:
+
+    DLITE_PYDEBUG= python bindings/python/tests/test_storage.py 1>/dev/null
+
+to see tracebacks of exceptions occurring inside any of the tested storage plugins.
 
 
 Debugging tests failing inside docker on GitHub
@@ -88,6 +100,76 @@ Debugging tests failing inside docker on GitHub
 
 
 
+Memory debugging
+----------------
+On Linux systems, one can use [valgrind] to check for and debug memory issues.
+
+
+### Finding memory issues
+To check for any memory issue, you can run valgrind on all tests with
+
+    make memcheck
+
+However, this usually takes a while. If you have localised an issue, and want
+to rerun valgrind on that issue, you can run
+
+    CTEST_ARGS="-R <test_name>" make memcheck
+
+Both of the above commands will write the output to `<BUILD_DIR>/Testing/Temporary/MemoryChecker.<#>.log`, where `<#>` is the test number.
+
+---
+
+Alternatively, you can run valgrind manually, with
+
+    valgrind <path/to/test-executable>
+
+if the test is written in C, or with
+
+    valgrind python <path/to/test_file.py>
+
+if the test is written in Python.
+This will write the output from valgrind to stderr.
+
+
+### Debugging segmentation faults
+If a test results in a segmentation fault, you can debug it with [gdb].
+See the [GDB Tutorial] for how to use `gdb`.
+
+Tests written in C can be debugged with
+
+    $ gdb <path/to/test-executable>
+    (gdb) run
+
+Test written in Python can be debugged with
+
+    $ gdb python
+    (gdb) run <path/to/test_file.py>
+
+Some useful gdb commands:
+- `where`: show the call stack when the test fails.
+- `up`, `down`: navigated up and down the call stack.
+- `print <expression>`: print the value of a variable or expression.
+
+
+### Debugging memory issues
+To debug other memory issues, invoke `gdb` as shown above, but set a breakpoint
+at the function before issuing the `run` command.  For example:
+
+    $ gdb python
+    (gdb) break <c_function_name>
+    (gdb) run <path/to/test_file.py>
+
+More useful gdb commands:
+- `break <function_name>`: set a break point at given function.
+- `break <file:lineno>`: set a break point at given source file position.
+- `continue`: continue to run the program after a break point.
+- `next`: execute next source line.
+- `step`: steps into the outermost subroutine on the next source line.
+  Behaves like `next` if the next source line has no subroutines.
+
 
 [virtualenvwrapper]: https://pypi.org/project/virtualenvwrapper/
 [Build against Python environment]: https://sintef.github.io/dlite/getting_started/build/build_against_python_env.html#build-against-python-environment
+[valgrind]: http://valgrind.org/
+[gdb]: https://sourceware.org/gdb/
+[GDB Tutorial]: https://www.gdbtutorial.com/

doc/getting_started/build/build.md

@@ -51,6 +51,8 @@ If you have [valgrind] installed, you can run the tests with memory checking tur
 
     cmake --build . --target memcheck
 
+For more info about checking and debugging memory issues, see [Memory debugging].
+
 
 Build documentation
 -------------------
@@ -138,3 +140,4 @@ Using [Visual Studio Code] (VS Code) it is possible to do development on the sys
 [configuration variables]: https://sintef.github.io/dlite/getting_started/build/cmake_variables.html
 [build against Python environment]: https://sintef.github.io/dlite/getting_started/build/build_against_python_env.html
 [build with Visual Studio]: https://sintef.github.io/dlite/getting_started/build/build_with_vs.html
+[Memory debugging]: https://sintef.github.io/dlite/contributors_guide/tips_and_tricks.html#memory-debugging

doc/getting_started/build/build_dependencies.md

@@ -50,6 +50,15 @@ On Ubuntu they can be installed with
     sudo apt install cppcheck valgrind
 
 
+Debugging (optional)
+--------------------
+Use [gdb] for debugging errors in the C code.  On Ubuntu it can be installed with
+
+    sudo apt install gdb
+
+See the [GDB Tutorial] for more information on how to use GDB.  Some DLite-specific tips can also be found in [Memory debugging].
+
+
 Building documentation (optional)
 ---------------------------------
 DLite uses [Sphinx] and [Doxygen] to generate documentation.
@@ -69,3 +78,6 @@ Ubuntu they can be installed with
 [Doxygen]: http://www.doxygen.org/
 [Sphinx]: https://www.sphinx-doc.org/
 [valgrind]: http://valgrind.org/
+[gdb]: https://sourceware.org/gdb/
+[GDB Tutorial]: https://www.gdbtutorial.com/
+[Memory debugging]: https://sintef.github.io/dlite/contributors_guidetips_and_tricks.html#memory-debugging

src/dlite-mapping.c

@@ -100,9 +100,20 @@ DLiteMapping *mapping_create_rec(const char *output_uri, Instances *inputs,
     }
     if (ignore) continue;
 
+    /*
+      FIXME: avoid memory leak
+      Implement dlite_mapping_plugin_free() and call it here.
+
+      Even better, implement caching in dlite_mapping_plugin_next()
+      such that it returns borrowed references to the api. Add a function
+      to clear the cache.
+    */
     if (!cheapest || cost < lowest_cost) {
+      //dlite_mapping_plugin_free(cheapest);
       cheapest = api;
       lowest_cost = cost;
+    } else {
+      //dlite_mapping_plugin_free(api);
     }
   }
   if (!(api = cheapest)) goto fail;

src/dlite-misc.c

@@ -465,6 +465,7 @@ int dlite_add_dll_path(void)
 /* Local state for this module. */
 typedef struct {
   int in_atexit;
+  int finalizing;
 } Locals;
 
 
@@ -496,7 +497,8 @@ static Locals *get_locals(void)
 static void _handle_atexit(void) {
 
   /* No extra finalisation is needed if we already are in an atexit handler */
-  if (dlite_globals_in_atexit()) return;
+  //if (dlite_globals_in_atexit()) return;
+  if (dlite_globals_in_atexit() && !getenv("DLITE_ATEXIT_FREE")) return;
 
   /* Mark that we are in an atexit handler */
   dlite_globals_set_atexit();
@@ -582,6 +584,11 @@ void dlite_init(void)
 void dlite_finalize(void)
 {
   Session *s = session_get_default();
+  Locals *locals = get_locals();
+
+  /* Ensure that we only finalize once. */
+  if (locals->finalizing) return;
+  locals->finalizing = 1;
 
   /* Don't free anything if we are in an atexit handler */
   if (dlite_globals_in_atexit() && !getenv("DLITE_ATEXIT_FREE")) return;

src/pyembed/dlite-python-storage.c

@@ -46,7 +46,7 @@ static void free_globals(void *globals)
   if (g->initialised) fu_paths_deinit(&g->paths);
 
   /* Do not call Py_DECREF if we are in an atexit handler */
-  if (!dlite_globals_in_atexit() || getenv("DLITE_ATEXIT_FREE")) {
+  if (!dlite_globals_in_atexit()) {
     Py_XDECREF(g->loaded_storages);
     g->loaded_storages = NULL;
   }

storages/json/dlite-json-storage.c

@@ -318,9 +318,12 @@ DLiteInstance *json_memload(const DLiteStoragePlugin *api,
                             const unsigned char *buf, size_t size,
                             const char *id, const char *options)
 {
+  DLiteInstance *inst;
   DLiteStorage *s = json_loader(api, NULL, buf, size, options);
-  DLiteInstance *inst = json_load(s, id);
+  if (!s) return NULL;
+  inst = json_load(s, id);
   json_close(s);
+  free(s);
   return inst;
 }