Merge branch '94-problems-using-milhoja-with-a-new-taskfunction' into…

… 'master'

Resolve "Problems using Milhoja with a new taskfunction"

Closes #94

See merge request joneal/OrchestrationRuntime!77

src/Milhoja_TileFlashxr.cpp

@@ -34,13 +34,13 @@ TileFlashxr::TileFlashxr(const FlashxrTileRawPtrs tP,
                          const FlashxTileRawInts tI,
                          const FlashxTileRawReals tR)
     : Tile{},
-      level_{static_cast<unsigned int>(tI.level - 1)}, // 1-based Flash-X convention -> 0-based
+      level_{static_cast<unsigned int>(tI.level - 1)},     // 1-based Flash-X convention -> 0-based
       gridIdxOrBlkId_{tI.gridIdxOrBlkId},
       tileIdx_{tI.tileIdx},
-      lo_{LIST_NDIM(tI.loX,tI.loY,tI.loZ)},
-      hi_{LIST_NDIM(tI.hiX,tI.hiY,tI.hiZ)},
-      loGC_{LIST_NDIM(tI.loGCX,tI.loGCY,tI.loGCZ)},
-      hiGC_{LIST_NDIM(tI.hiGCX,tI.hiGCY,tI.hiGCZ)},
+      lo_{LIST_NDIM(tI.loX-1,tI.loY-1,tI.loZ-1)},          // 1-based Flash-X convention -> 0-based
+      hi_{LIST_NDIM(tI.hiX-1,tI.hiY-1,tI.hiZ-1)},          // 1-based Flash-X convention -> 0-based
+      loGC_{LIST_NDIM(tI.loGCX-1,tI.loGCY-1,tI.loGCZ-1)},  // 1-based Flash-X convention -> 0-based
+      hiGC_{LIST_NDIM(tI.hiGCX-1,tI.hiGCY-1,tI.hiGCZ-1)},  // 1-based Flash-X convention -> 0-based
       nCcComp_{tI.nCcComp},
       nFluxComp_{tI.nFluxComp},
       deltas_{LIST_NDIM(tR.deltaX,tR.deltaY,tR.deltaZ)},

tools/milhoja_pypkg/VERSION

@@ -1 +1 @@
-0.0.5
+0.0.8

tools/milhoja_pypkg/src/milhoja/DataPacketGenerator.py

@@ -347,9 +347,9 @@ def cpp_sort(kv_pair):
             )
 
         def fortran_sort(x):
-            return self._sizes.get(
-                VECTOR_ARRAY_EQUIVALENT[SOURCE_DATATYPES[x[1]['source']]], 0
-            )
+            dtype = SOURCE_DATATYPES[x[1]['source']]
+            dtype = VECTOR_ARRAY_EQUIVALENT.get(dtype, dtype)
+            return self._sizes.get(dtype, 0)
 
         if lang == 'c++':
             sort_func = cpp_sort
@@ -372,6 +372,10 @@ def fortran_sort(x):
             mdata[key]['type'] = SOURCE_DATATYPES[mdata[key]["source"]]
             if lang == "fortran":
                 dtype = mdata[key]['type']
+                # Fortran has no equivalent of unsigned int.
+                if dtype == 'unsigned int':
+                    dtype = 'int'
+
                 mdata[key]['type'] = \
                     VECTOR_ARRAY_EQUIVALENT.get(dtype, dtype)
 

tools/milhoja_pypkg/src/milhoja/FortranTemplateUtility.py

@@ -12,7 +12,8 @@
     EXTERNAL_ARGUMENT, GRID_DATA_ARGUMENT, TILE_UBOUND_ARGUMENT,
     TILE_LO_ARGUMENT, TILE_HI_ARGUMENT, TILE_INTERIOR_ARGUMENT,
     TILE_ARRAY_BOUNDS_ARGUMENT, TILE_LBOUND_ARGUMENT, LBOUND_ARGUMENT,
-    GRID_DATA_LBOUNDS, SCRATCH_ARGUMENT, F2C_TYPE_MAPPING
+    GRID_DATA_LBOUNDS, SCRATCH_ARGUMENT, F2C_TYPE_MAPPING,
+    TILE_LEVEL_ARGUMENT, SOURCE_DATATYPES
 )
 
 
@@ -104,8 +105,13 @@ def iterate_tilemetadata(
             if source == interior or source == TILE_ARRAY_BOUNDS_ARGUMENT:
                 bound_size_modifier = "2 * "
 
+            # check if the tile source is an array.
+            mdim_modifier = "MILHOJA_MDIM"
+            if SOURCE_DATATYPES[source] not in ["IntVect", "RealVect"]:
+                mdim_modifier = "1"
+
             size_eq = \
-                f"{bound_size_modifier}MILHOJA_MDIM * sizeof({item_type})"
+                f"{bound_size_modifier}{mdim_modifier} * sizeof({item_type})"
 
             lbound = []
 
@@ -162,6 +168,7 @@ def iterate_tilemetadata(
 
             construct_host = ""
             if pure_source != LBOUND_ARGUMENT:
+                # adjusting index base
                 fix_index = '+1' if pure_source in bounds_data else ''
                 info.dtype = F2C_TYPE_MAPPING.get(info.dtype, info.dtype)
 
@@ -170,27 +177,36 @@ def iterate_tilemetadata(
                 # not have knowledge of the other
                 if source == TILE_INTERIOR_ARGUMENT:
                     construct_host = "[MILHOJA_MDIM * 2] = {" \
-                        "tileDesc_h->lo().I()+1,tileDesc_h->hi().I()+1, " \
-                        "tileDesc_h->lo().J()+1,tileDesc_h->hi().J()+1, " \
-                        "tileDesc_h->lo().K()+1,tileDesc_h->hi().K()+1 }"
+                        f"tileDesc_h->lo().I(){fix_index}, tileDesc_h->hi().I(){fix_index}, " \
+                        f"tileDesc_h->lo().J(){fix_index}, tileDesc_h->hi().J(){fix_index}, " \
+                        f"tileDesc_h->lo().K(){fix_index}, tileDesc_h->hi().K(){fix_index} " \
+                        "}"
+
                 elif source == TILE_ARRAY_BOUNDS_ARGUMENT:
                     construct_host = "[MILHOJA_MDIM * 2] = {" \
-                        "tileDesc_h->loGC().I()+1,tileDesc_h->hiGC().I()+1, "\
-                        "tileDesc_h->loGC().J()+1,tileDesc_h->hiGC().J()+1, "\
-                        "tileDesc_h->loGC().K()+1,tileDesc_h->hiGC().K()+1 }"
+                        f"tileDesc_h->loGC().I(){fix_index}, tileDesc_h->hiGC().I(){fix_index}, "\
+                        f"tileDesc_h->loGC().J(){fix_index}, tileDesc_h->hiGC().J(){fix_index}, "\
+                        f"tileDesc_h->loGC().K(){fix_index}, tileDesc_h->hiGC().K(){fix_index} " \
+                        "}"
+
+                elif source == TILE_LEVEL_ARGUMENT:
+                    one_time_mdata[item] = data
+                    construct_host = f"[1] = {{(int){short_source}}}"
+
                 else:
                     one_time_mdata[item] = data
                     construct_host = "[MILHOJA_MDIM] = { " \
                         f"{short_source}.I(){fix_index}, " \
                         f"{short_source}.J(){fix_index}, " \
                         f"{short_source}.K(){fix_index} }}"
+
             # we can write very specific code if we know that the variable
             # is an lbound argument.
             else:
                 # info.dtype = VECTOR_ARRAY_EQUIVALENT[info.dtype]
                 # intvect i,j,k start at 0 so we need to add 1 to the
-                # index. However, anything that's just integers needs to be
-                # untouched.
+                # index as it is a Fortran.
+                # However, anything that's just integers needs to be untouched.
                 for idx, value in enumerate(lbound):
                     found = re.search('[a-zA-z]', value)
                     if found:

tools/milhoja_pypkg/src/milhoja/TaskFunctionC2FGenerator_OpenACC_F.py

@@ -183,8 +183,8 @@ def _get_metadata_info(self, arg, spec):
         :param arg: The name of the variable
         :param spec: The arg spec of the variable
         """
-        dtype = SOURCE_DATATYPES[spec["source"]]
-        dtype = VECTOR_ARRAY_EQUIVALENT[dtype]
+        original_dtype = SOURCE_DATATYPES[spec["source"]]
+        dtype = VECTOR_ARRAY_EQUIVALENT.get(original_dtype, original_dtype)
         dtype = C2F_TYPE_MAPPING.get(dtype, dtype)
         shape = []
         info = C2FInfo(
@@ -213,8 +213,10 @@ def _get_metadata_info(self, arg, spec):
             arrayBound = TILE_ARRAY_BOUNDS_ARGUMENT
             if arg == interior or arg == arrayBound:
                 info.shape = ['2', 'MILHOJA_MDIM', 'F_nTiles_h']
-            else:
+            elif original_dtype in VECTOR_ARRAY_EQUIVALENT:
                 info.shape = ['MILHOJA_MDIM', 'F_nTiles_h']
+            else:
+                info.shape = ["1", "F_nTiles_h"]
             info.conversion_eq = info.conversion_eq.format(
                 info.cname, info.fname, ', '.join(info.shape)
             )

tools/milhoja_pypkg/src/milhoja/TaskFunctionC2FGenerator_cpu_F.py

@@ -12,7 +12,7 @@
     EXTERNAL_ARGUMENT, LBOUND_ARGUMENT, GRID_DATA_ARGUMENT,
     SCRATCH_ARGUMENT, TILE_INTERIOR_ARGUMENT, TILE_ARRAY_BOUNDS_ARGUMENT,
     TILE_ARGUMENTS_ALL, VECTOR_ARRAY_EQUIVALENT, SOURCE_DATATYPES,
-    GRID_DATA_EXTENTS, GRID_DATA_LBOUNDS, C2F_TYPE_MAPPING
+    GRID_DATA_EXTENTS, GRID_DATA_LBOUNDS, C2F_TYPE_MAPPING, TILE_LEVEL_ARGUMENT
 )
 
 
@@ -104,6 +104,9 @@ def _get_tmdata_info(self, arg, arg_spec) -> ConversionData:
             name_key = src
             shape.insert(0, "2")
 
+        if src == TILE_LEVEL_ARGUMENT:
+            shape = ["1"]
+
         return ConversionData(
             cname=f"C_{name_key}",
             fname=f"F_{name_key}",

tools/milhoja_pypkg/src/milhoja/TaskFunctionCpp2CGenerator_cpu_F.py

@@ -7,13 +7,17 @@
 from . import LogicError
 from . import TaskFunction
 from .generate_packet_file import generate_packet_file
-from .parse_helpers import parse_lbound_f
+from .parse_helpers import (
+    get_initial_index,
+    parse_lbound_f
+)
 from . import (
     EXTERNAL_ARGUMENT, LBOUND_ARGUMENT, TILE_LBOUND_ARGUMENT,
     TILE_UBOUND_ARGUMENT, SCRATCH_ARGUMENT, F2C_TYPE_MAPPING,
     THREAD_INDEX_VAR_NAME, GRID_DATA_ARGUMENT, TILE_INTERIOR_ARGUMENT,
     TILE_ARRAY_BOUNDS_ARGUMENT, GRID_DATA_PTRS, SOURCE_DATATYPES,
     VECTOR_ARRAY_EQUIVALENT, TILE_ARGUMENTS_ALL, GRID_DATA_LBOUNDS,
+    TILE_INDEX_DATA,
 )
 
 
@@ -146,14 +150,16 @@ def _fill_mdata_connectors(self, arg, spec, connectors: dict, saved):
         if src == TILE_INTERIOR_ARGUMENT or src == TILE_ARRAY_BOUNDS_ARGUMENT:
             lo = 'tile_lo' if src == TILE_INTERIOR_ARGUMENT else "tile_loGC"
             hi = 'tile_hi' if src == TILE_INTERIOR_ARGUMENT else "tile_hiGC"
+            # Fortran assumes 1-based index
+            offs = "+1"
             connectors[self.C2F_ARG_LIST].append(f"const void* {src}")
             connectors[self.REAL_ARGS].append(f"static_cast<void*>({src})")
             lo_data = lo.replace("tile_", "") + "()"
             hi_data = hi.replace("tile_", "") + "()"
             combined = f"int {src}[] = {{\n{self.INDENT}"
             combined += f',\n{self.INDENT}'.join(
-                '{0}->{1}.{3}(),{0}->{2}.{3}()'.format(
-                    self.tile_desc_name, lo_data, hi_data, char
+                '{0}->{1}.{3}(){4}, {0}->{2}.{3}(){4}'.format(
+                    self.tile_desc_name, lo_data, hi_data, char, offs
                 )
                 for char in ['I', 'J', 'K']
             ) + "\n}"
@@ -166,6 +172,7 @@ def _fill_mdata_connectors(self, arg, spec, connectors: dict, saved):
                 alt_src = "tile_loGC"
             elif src == TILE_UBOUND_ARGUMENT:
                 alt_src = "tile_hiGC"
+
             tile_desc_func = alt_src.replace("tile_", '')
             tile_desc_name = self.tile_desc_name
             if alt_src not in saved:
@@ -182,11 +189,22 @@ def _fill_mdata_connectors(self, arg, spec, connectors: dict, saved):
 
             dtype = SOURCE_DATATYPES[src]
             if dtype in VECTOR_ARRAY_EQUIVALENT:
+                offs = ""
+                if src in TILE_INDEX_DATA:
+                    # Fortran assumes 1-based index
+                    offs = "+1"
                 raw = VECTOR_ARRAY_EQUIVALENT[dtype]
                 connectors[self.CONSOLIDATE_TILE_DATA].append(
-                    f"{raw} {arg}_array[] = {{\n{self.INDENT}{arg}.I(),\n"
-                    f"{self.INDENT}{arg}.J(),\n"
-                    f"{self.INDENT}{arg}.K()\n}}"
+                    f"{raw} {arg}_array[] = {{\n{self.INDENT}{arg}.I(){offs},\n"
+                    f"{self.INDENT}{arg}.J(){offs},\n"
+                    f"{self.INDENT}{arg}.K(){offs}\n}}"
+                )
+
+            else:
+                if 'unsigned' in dtype:
+                    dtype = dtype.replace("unsigned ", "")
+                connectors[self.CONSOLIDATE_TILE_DATA].append(
+                    f"{dtype} {arg}_array[] = {{{arg}}}"
                 )
 
     def _fill_lbound_connectors(self, arg, spec, connectors, saved):
@@ -208,8 +226,11 @@ def _fill_lbound_connectors(self, arg, spec, connectors, saved):
         words = None
         if var_spec["source"] == GRID_DATA_ARGUMENT:
             st_idx = var_spec["structure_index"][0].upper()
-            gcells = self._tf_spec.n_guardcells
-            lb, words = parse_lbound_f(GRID_DATA_LBOUNDS[st_idx].format(gcells))
+            # get starting array value
+            vars_in = var_spec.get("variables_in", None)
+            vars_out = var_spec.get("variables_out", None)
+            init = get_initial_index(vars_in, vars_out)
+            lb, words = parse_lbound_f(GRID_DATA_LBOUNDS[st_idx].format(init))
         else:
             lb, words = parse_lbound_f(var_spec["lbound"])
 
@@ -227,6 +248,13 @@ def _fill_lbound_connectors(self, arg, spec, connectors, saved):
                 )
                 saved.add(word)
 
+        # adjusting the base index
+        # because it is a Fortran
+        for i, bound in enumerate(lb):
+            for keyword in TILE_INDEX_DATA:
+                if keyword in bound:
+                    lb[i] = bound + "+1"
+
         lb = f"{{\n{self.INDENT}" + f',\n{self.INDENT}'.join(lb) + "\n}"
         lb = lb.replace(" ", "")
         connectors[self.CONSOLIDATE_TILE_DATA].append(

tools/milhoja_pypkg/src/milhoja/TaskFunctionGenerator_OpenACC_F.py

@@ -8,7 +8,8 @@
     LOG_LEVEL_BASIC, LOG_LEVEL_BASIC_DEBUG, EXTERNAL_ARGUMENT, TILE_LO_ARGUMENT,
     TILE_HI_ARGUMENT, TILE_LBOUND_ARGUMENT, TILE_UBOUND_ARGUMENT,
     TILE_DELTAS_ARGUMENT, GRID_DATA_ARGUMENT, SCRATCH_ARGUMENT, LBOUND_ARGUMENT,
-    C2F_TYPE_MAPPING, TILE_INTERIOR_ARGUMENT, TILE_ARRAY_BOUNDS_ARGUMENT
+    C2F_TYPE_MAPPING, TILE_INTERIOR_ARGUMENT, TILE_ARRAY_BOUNDS_ARGUMENT,
+    TILE_LEVEL_ARGUMENT
 )
 
 
@@ -192,6 +193,9 @@ def generate_source_code(self, destination, overwrite):
                 elif src in bounds:
                     fptr.write(f"{INDENT*2}integer, intent(IN) :: {arg}_d(:, :, :)\n")
 
+                elif src == TILE_LEVEL_ARGUMENT:
+                    fptr.write(f"{INDENT*2}integer, intent(IN) :: {arg}_d(:, :)\n")
+
                 elif src == GRID_DATA_ARGUMENT:
                     if arg in self._tf_spec.tile_in_arguments:
                         intent = "IN"
@@ -292,10 +296,14 @@ def generate_source_code(self, destination, overwrite):
                     for argument in actual_args:
                         spec = self._tf_spec.argument_specification(argument)
                         extents = ""
+                        offs = ""
                         if spec["source"] in points:
                             extents = "(:, n)"
                         elif spec["source"] == TILE_DELTAS_ARGUMENT:
                             extents = "(:, n)"
+                        elif spec["source"] == TILE_LEVEL_ARGUMENT:
+                            extents = "(1, n)"
+                            offs = " + 1"
                         elif spec["source"] in bounds:
                             extents = "(:, :, n)"
                         elif spec["source"] == GRID_DATA_ARGUMENT:
@@ -304,7 +312,7 @@ def generate_source_code(self, destination, overwrite):
                             dimension = len(parse_extents(spec["extents"]))
                             tmp = [":" for _ in range(dimension)]
                             extents = "(" + ", ".join(tmp) + ", n)"
-                        arg_list.append(f"{INDENT*5}{argument}_d{extents}")
+                        arg_list.append(f"{INDENT*5}{argument}_d{extents}{offs}")
                     fptr.write(", &\n".join(arg_list) + " &\n")
                     fptr.write(f"{INDENT*5})\n")
                     fptr.write(f"{INDENT*2}end do\n")

tools/milhoja_pypkg/src/milhoja/TaskFunctionGenerator_cpu_F.py

@@ -9,7 +9,7 @@
     TILE_LO_ARGUMENT, TILE_HI_ARGUMENT, TILE_LBOUND_ARGUMENT,
     TILE_UBOUND_ARGUMENT, TILE_DELTAS_ARGUMENT, GRID_DATA_ARGUMENT,
     SCRATCH_ARGUMENT, LBOUND_ARGUMENT, C2F_TYPE_MAPPING,
-    TILE_INTERIOR_ARGUMENT, TILE_ARRAY_BOUNDS_ARGUMENT
+    TILE_INTERIOR_ARGUMENT, TILE_ARRAY_BOUNDS_ARGUMENT, TILE_LEVEL_ARGUMENT
 )
 
 
@@ -167,7 +167,7 @@ def generate_source_code(self, destination, overwrite):
             # Generation-time argument definitions
             points = {
                 TILE_LO_ARGUMENT, TILE_HI_ARGUMENT, TILE_LBOUND_ARGUMENT,
-                TILE_UBOUND_ARGUMENT
+                TILE_UBOUND_ARGUMENT, TILE_LEVEL_ARGUMENT
             }
             bounds = {TILE_INTERIOR_ARGUMENT, TILE_ARRAY_BOUNDS_ARGUMENT}
 
@@ -220,6 +220,9 @@ def generate_source_code(self, destination, overwrite):
                         f"{INDENT*2}integer, intent(IN) :: {arg}(:, :)\n"
                     )
 
+                elif src == TILE_LEVEL_ARGUMENT:
+                    fptr.write(f"{INDENT*2}integer, intent(IN) :: {arg}\n")
+
                 elif src == GRID_DATA_ARGUMENT:
                     if arg in self._tf_spec.tile_in_arguments:
                         intent = "IN"
@@ -276,6 +279,10 @@ def generate_source_code(self, destination, overwrite):
                         ):
                             arg += "_ptr"
 
+                        # get the first argument in the tile level array?
+                        if argument == TILE_LEVEL_ARGUMENT:
+                            arg += "(1) + 1"
+
                         arg_list.append(arg)
                     fptr.write(", &\n".join(arg_list) + " &\n")
                     fptr.write(f"{INDENT*2})\n")

tools/milhoja_pypkg/src/milhoja/__init__.py

@@ -34,6 +34,7 @@
     TILE_FACE_AREAS_ARGUMENT,
     TILE_CELL_VOLUMES_ARGUMENT,
     TILE_ARGUMENTS_ALL,
+    TILE_INDEX_DATA,
     GRID_DATA_LBOUNDS, GRID_DATA_EXTENTS,
     C2F_TYPE_MAPPING, VECTOR_ARRAY_EQUIVALENT,
     GRID_DATA_PTRS, SOURCE_DATATYPES, F2C_TYPE_MAPPING,

tools/milhoja_pypkg/src/milhoja/constants.py

@@ -66,6 +66,16 @@
     TILE_CELL_VOLUMES_ARGUMENT
 }
 
+# The following index data must be handled carefully
+# with a proper starting index, e.g., 1-based for Fortran language,
+# during the code generations.
+TILE_INDEX_DATA = [
+    TILE_LO_ARGUMENT,
+    TILE_HI_ARGUMENT,
+    TILE_LBOUND_ARGUMENT,
+    TILE_UBOUND_ARGUMENT,
+]
+
 # ----- DATA ACCESS KEYWORDS
 READ = "r"
 WRITE = "w"
@@ -99,8 +109,8 @@
 # to insert lbounds for grid data variables inside of the tf spec that need
 # an lbound array.
 GRID_DATA_LBOUNDS = {
-    "CENTER": "(tile_lbound, {0})",  # CC data does use guard cells.
-    "FLUXX": "(tile_lo, {0})",  # assume that flux arrays use 0 guard cells.
+    "CENTER": "(tile_lbound, {0})",  # CC data (including guard cells)
+    "FLUXX": "(tile_lo, {0})",  # flux arrays (without guard cells)
     "FLUXY": "(tile_lo, {0})",
     "FLUXZ": "(tile_lo, {0})"
 }