Move to new t8code version 3.0.0

.github/workflows/ci.yml

@@ -37,13 +37,13 @@ jobs:
         julia_version:
           - '1.10'
         t8code_version:
-          - '2.0.0'
+          - '3.0.0'
         include:
           - os: ubuntu-latest
             test_type: package-compiler
             arch: x64
             julia_version: '1.9.3'  # 1.9.4: missing nghttp2 symbols in libcurl
-            t8code_version: '2.0.0'
+            t8code_version: '3.0.0'
     env:
       # Necessary for HDF5 to play nice with Julia
       LD_PRELOAD: /lib/x86_64-linux-gnu/libcurl.so.4
@@ -92,12 +92,22 @@ jobs:
           T8CODE_RELEASE=${{ matrix.t8code_version }}
           mkdir t8code-local
           cd t8code-local
-          wget https://github.com/DLR-AMR/t8code/releases/download/v${T8CODE_RELEASE}/t8-${T8CODE_RELEASE}.tar.gz
-          tar xf t8-${T8CODE_RELEASE}.tar.gz
+          wget https://github.com/DLR-AMR/t8code/releases/download/v${T8CODE_RELEASE}/T8CODE-${T8CODE_RELEASE}-Source.tar.gz
+          tar xf T8CODE-${T8CODE_RELEASE}-Source.tar.gz
           mkdir build
           cd build
-          CC=mpicc CXX=mpicxx ../t8-${T8CODE_RELEASE}/configure \
-              --prefix=$PWD/../prefix --enable-mpi
+          cmake \
+              -DCMAKE_C_COMPILER=mpicc \
+              -DCMAKE_CXX_COMPILER=mpicxx \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DCMAKE_INSTALL_PREFIX="$PWD/../prefix" \
+              -DT8CODE_BUILD_TESTS=OFF \
+              -DT8CODE_BUILD_TUTORIALS=OFF \
+              -DT8CODE_BUILD_EXAMPLES=OFF \
+              -DT8CODE_BUILD_BENCHMARKS=OFF \
+              -DT8CODE_ENABLE_MPI=ON \
+              -DT8CODE_BUILD_FORTRAN_INTERFACE=ON \
+              ../T8CODE-${T8CODE_RELEASE}-Source
           make -j 2
           make install
 
@@ -108,7 +118,7 @@ jobs:
           cd build
           cmake .. -DCMAKE_INSTALL_PREFIX=../install \
                    -DCMAKE_BUILD_TYPE=Release \
-                   -DT8CODE_PREFIX=$PWD/../t8code-local/prefix \
+                   -DT8CODE_ROOT=$PWD/../t8code-local/prefix \
                    -DENABLE_TESTING=ON -DJULIA_PROJECT_PATH=../libtrixi-julia
 
       - name: Configure (test_type == 'coverage')
@@ -118,7 +128,7 @@ jobs:
           cd build
           cmake .. -DCMAKE_INSTALL_PREFIX=../install \
                    -DCMAKE_BUILD_TYPE=Debug \
-                   -DT8CODE_PREFIX=$PWD/../t8code-local/prefix \
+                   -DT8CODE_ROOT=$PWD/../t8code-local/prefix \
                    -DCMAKE_C_FLAGS="-cpp --coverage -O0" \
                    -DCMAKE_Fortran_FLAGS="-cpp --coverage -O0" \
                    -DCMAKE_EXE_LINKER_FLAGS="--coverage" \
@@ -235,6 +245,7 @@ jobs:
           ../build/examples/trixi_controller_data_c . libelixir_t8code_2d_dgsem_advection_amr.jl
           ../build/examples/trixi_controller_data_f . libelixir_t8code_2d_dgsem_advection_amr.jl
           ../build/examples/trixi_controller_t8code_c . libelixir_t8code_2d_dgsem_advection_amr.jl
+          ../build/examples/trixi_controller_t8code_f . libelixir_t8code_2d_dgsem_advection_amr.jl
         env:
           LIBTRIXI_DEBUG: all
 
@@ -261,8 +272,10 @@ jobs:
                          "../build/examples/trixi_controller_data_c ."   \
                          "../build/examples/trixi_controller_data_f"     \
                          "../build/examples/trixi_controller_data_f ."   \
-                         "../build/examples/trixi_controller_t8code_c"             \
-                         "../build/examples/trixi_controller_t8code_c ."
+                         "../build/examples/trixi_controller_t8code_c"   \
+                         "../build/examples/trixi_controller_t8code_c ." \
+                         "../build/examples/trixi_controller_t8code_f"   \
+                         "../build/examples/trixi_controller_t8code_f ."
             do
               $command
               if [ $? -ne 2 ]; then

CMakeLists.txt

@@ -1,5 +1,7 @@
-# Specify the minimum version (3.9 required for regex submatches).
-cmake_minimum_required ( VERSION 3.9 )
+# Specify the minimum version
+#   3.9  required for regex submatches
+#   3.12 required for policy CMP0074 (using *_ROOT variables)
+cmake_minimum_required ( VERSION 3.12 )
 
 # Reconfigure if Project.toml has changed
 set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${CMAKE_SOURCE_DIR}/LibTrixi.jl/Project.toml")
@@ -36,7 +38,7 @@ list ( APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/" )
 find_package( Julia REQUIRED )
 
 # Find t8code
-find_package( T8code )
+find_package( T8CODE CONFIG )
 if ( NOT T8CODE_FOUND )
     message( NOTICE "t8code not found: t8code examples will NOT be built")
 endif()
@@ -140,9 +142,6 @@ else()
 
     # Include directories
     target_include_directories( ${PROJECT_NAME} PRIVATE src ${JULIA_INCLUDE_DIRS} )
-    if ( T8CODE_FOUND )
-        target_include_directories( ${PROJECT_NAME} PRIVATE ${T8CODE_INCLUDE_DIR} )
-    endif()
 
     # Libraries to link
     target_link_libraries( ${PROJECT_NAME} PRIVATE ${JULIA_LIBRARY} )

LibTrixi.jl/src/api_jl.jl

@@ -46,6 +46,16 @@ function trixi_finalize_simulation_jl(simstate)
         end
     end
 
+    # In course of garbage collection, MPI might get finalized before t8code related objects.
+    # This can lead to crashes because t8code allocates MPI related objects, e.g. shared
+    # memory arrays. The workaround is to finalize T8codeMesh explicitly in advance.
+    # x-ref: https://github.com/DLR-AMR/t8code/issues/1295
+    # x-ref: https://github.com/trixi-framework/libtrixi/pull/215#discussion_r1843676330
+    mesh, _, _, _ = mesh_equations_solver_cache(simstate.semi)
+    if mesh isa Trixi.T8codeMesh
+        finalize(mesh)
+    end
+
     if show_debug_output()
         println("Simulation state finalized")
     end
@@ -132,7 +142,7 @@ end
 
 
 function trixi_get_t8code_forest_jl(simstate)
-    mesh, _, _, _ = Trixi.mesh_equations_solver_cache(simstate.semi)
+    mesh, _, _, _ = mesh_equations_solver_cache(simstate.semi)
     return mesh.forest
 end
 

README.md

@@ -24,7 +24,7 @@ software packages need to be made available locally before installing libtrixi:
 * [CMake](https://cmake.org/)
 * MPI (e.g., [OpenMPI](https://www.open-mpi.org/) or [MPICH](https://www.mpich.org/))
 * [HDF5](https://www.hdfgroup.org/solutions/hdf5/)
-* [t8code](https://github.com/DLR-AMR/t8code)
+* [t8code](https://github.com/DLR-AMR/t8code) v3.0.0
 
 ### Get the sources
 
@@ -57,7 +57,7 @@ For building, `cmake` and its typical workflow is used.
     - Optional specification of build type sets some default compiler options for optimized
       or debug code.
     - Building with t8code support is optional. It requires to pass
-      `-DT8CODE_PREFIX=<t8code_install_directory>`.
+      `-DT8CODE_ROOT=<t8code_install_directory>`.
 
 3. Call make
 
@@ -214,8 +214,7 @@ aspects on how to use the C and Fortran APIs of libtrixi:
 - `trixi_controller_simple.(c|f90)`: basic usage
 - `trixi_controller_mpi.(c|f90)`: usage in the presence of MPI
 - `trixi_controller_data.(c|f90)`: simulation data access
-- `trixi_controller_t8code.c`: interacting with t8code
-  (there is no Fortran example yet as the Fortran interface of t8code is still under development)
+- `trixi_controller_t8code.(c|f90)`: interacting with t8code
 
 If you just want to test the Julia part of libtrixi, i.e., LibTrixi.jl, you can also run
 `trixi_controller_simple.jl` from Julia.

examples/CMakeLists.txt

@@ -5,7 +5,8 @@ set ( EXAMPLES
       trixi_controller_mpi.f90
       trixi_controller_data.c
       trixi_controller_data.f90
-      trixi_controller_t8code.c )
+      trixi_controller_t8code.c
+      trixi_controller_t8code.f90 )
 
 if ( NOT T8CODE_FOUND )
     list( FILTER EXAMPLES EXCLUDE REGEX ".*(t|T)8(c|C)(o|O)(d|D)(e|E).*" )
@@ -39,7 +40,7 @@ foreach ( EXAMPLE ${EXAMPLES} )
         target_link_libraries( ${TARGET_NAME} PRIVATE ${PROJECT_NAME}_tls )
     endif()
     if ( T8CODE_FOUND )
-        target_link_libraries( ${TARGET_NAME} PRIVATE ${T8CODE_LIBRARIES} )
+        target_link_libraries( ${TARGET_NAME} PRIVATE T8CODE::T8 )
     endif()
 
     # set include directories
@@ -48,7 +49,7 @@ foreach ( EXAMPLE ${EXAMPLES} )
         PRIVATE ${CMAKE_SOURCE_DIR}/src
     )
     if ( T8CODE_FOUND )
-        target_include_directories( ${TARGET_NAME} PRIVATE ${T8CODE_INCLUDE_DIR} )
+        target_include_directories( ${TARGET_NAME} PRIVATE ${T8CODE_ROOT}/include/t8_fortran_interface/ )
     endif()
 
     # set runtime path for installed binaries

examples/trixi_controller_t8code.f90

@@ -0,0 +1,82 @@
+! Print the local and global number of elements of a forest.
+subroutine t8_print_forest_information(forest)
+    use t8_fortran_interface_mod
+    use, intrinsic :: iso_c_binding, only: c_ptr, c_int
+
+    implicit none
+
+    type(c_ptr) :: forest
+    integer(c_int) :: local_num_elements, global_num_elements
+
+    ! Check that forest is a committed, that is valid and usable, forest.
+    ! T8_ASSERT (t8_forest_is_committed (forest));
+
+    ! Get the local number of elements.
+    local_num_elements = t8_forest_get_local_num_elements (forest)
+
+    ! Get the global number of elements.
+    global_num_elements = t8_forest_get_global_num_elements (forest)
+
+    write(*, '(a,i6)') "*** T8code ***  Local number of elements:  ", local_num_elements
+    write(*, '(a,i6)') "*** T8code ***  Global number of elements: ", global_num_elements
+end subroutine
+
+
+program trixi_controller_simple_f
+  use LibTrixi
+  use, intrinsic :: iso_fortran_env, only: error_unit
+  use, intrinsic :: iso_c_binding, only: c_int, c_ptr
+
+  implicit none
+
+  integer(c_int) :: handle, nelements
+  character(len=256) :: argument
+  type(c_ptr) :: forest
+
+  if (command_argument_count() < 1) then
+    call get_command_argument(0, argument)
+    write(error_unit, '(a)') "ERROR: missing arguments: PROJECT_DIR LIBELIXIR_PATH"
+    write(error_unit, '(a)') ""
+    write(error_unit, '(3a)') "usage: ", trim(argument), " PROJECT_DIR LIBELIXIR_PATH"
+    call exit(2)
+  else if (command_argument_count() < 2) then
+    call get_command_argument(0, argument)
+    write(error_unit, '(a)') "ERROR: missing argument: LIBELIXIR_PATH"
+    write(error_unit, '(a)') ""
+    write(error_unit, '(3a)') "usage: ", trim(argument), " PROJECT_DIR LIBELIXIR_PATH"
+    call exit(2)
+  end if
+
+  ! Initialize Trixi
+  call get_command_argument(1, argument)
+  call trixi_initialize(argument)
+
+  ! Set up the Trixi simulation
+  ! We get a handle to use subsequently
+  call get_command_argument(2, argument)
+  handle = trixi_initialize_simulation(argument)
+
+  ! Main loop
+  do
+    ! Exit loop once simulation is completed
+    if ( trixi_is_finished(handle) ) exit
+
+    call trixi_step(handle)
+  end do
+
+  ! get number of elements
+  nelements = trixi_nelements( handle );
+  write(*, '(a)') ""
+  write(*, '(a,i6)') "*** Trixi controller ***   nelements ", nelements
+  write(*, '(a)') ""
+
+  ! get t8code forest
+  forest = trixi_get_t8code_forest( handle )
+  call t8_print_forest_information ( forest )
+
+  ! Finalize Trixi simulation
+  call trixi_finalize_simulation(handle)
+
+  ! Finalize Trixi
+  call trixi_finalize()
+end program

src/api.f90

@@ -301,6 +301,26 @@ subroutine trixi_load_cell_averages(data, handle) bind(c)
 
 
 
+    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+    !! t8code                                                                             !!
+    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+    !>
+    !! @fn LibTrixi::trixi_get_t8code_forest::trixi_get_t8code_forest(handle)
+    !!
+    !! @brief Get t8code forest
+    !!
+    !! @param[in]  handle       simulation handle
+    !!
+    !! @return t8code forest
+    !!
+    !! @see @ref trixi_get_t8code_forest_api_c "trixi_get_t8code_forest (C API)"
+    type (c_ptr) function trixi_get_t8code_forest(handle) bind(c)
+      use, intrinsic :: iso_c_binding, only: c_int, c_ptr
+      integer(c_int), value, intent(in) :: handle
+    end function
+
+
+
     !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
     !! Misc                                                                               !!
     !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

test/c/CMakeLists.txt

@@ -23,17 +23,14 @@ foreach ( TEST ${TESTS} )
         PRIVATE MPI::MPI_CXX ${PROJECT_NAME} ${PROJECT_NAME}_tls GTest::gtest_main
     )
     if ( T8CODE_FOUND )
-        target_link_libraries( ${TARGET_NAME} PRIVATE ${T8CODE_LIBRARIES} )
+        target_link_libraries( ${TARGET_NAME} PRIVATE T8CODE::T8 )
     endif()
 
     # set include directories
     target_include_directories(
         ${TARGET_NAME}
         PRIVATE ${CMAKE_SOURCE_DIR}/src
     )
-    if ( T8CODE_FOUND )
-        target_include_directories( ${TARGET_NAME} PRIVATE ${T8CODE_INCLUDE_DIR} )
-    endif()
 
     # enable warnings
     target_compile_options( ${TARGET_NAME} PRIVATE -Wall -Wextra -Werror )

test/fortran/CMakeLists.txt

@@ -3,6 +3,10 @@ set ( TESTS
       simulationRun_suite
       versionInfo_suite )
 
+if ( T8CODE_FOUND )
+      list( APPEND TESTS t8code_suite )
+endif()
+
 set ( TEST_SRCS
       "main.f90"
 )
@@ -20,12 +24,18 @@ target_link_libraries(
     ${TARGET_NAME}
     PRIVATE MPI::MPI_Fortran ${PROJECT_NAME} ${PROJECT_NAME}_tls "test-drive::test-drive"
 )
+if ( T8CODE_FOUND )
+    target_link_libraries( ${TARGET_NAME} PRIVATE T8CODE::T8 )
+endif()
 
 # set include directories
 target_include_directories(
     ${TARGET_NAME}
     PRIVATE ${CMAKE_SOURCE_DIR}/src
 )
+if ( T8CODE_FOUND )
+    target_include_directories( ${TARGET_NAME} PRIVATE ${T8CODE_ROOT}/include/t8_fortran_interface/ )
+endif()
 
 # enable warnings
 target_compile_options( ${TARGET_NAME} PRIVATE -cpp -Wall -Wextra -Werror -Wno-uninitialized )