Feature/new modifier implementation

modifiers/caliper/modifier.py

@@ -6,6 +6,21 @@
 from ramble.modkit import *
 
 
+def add_mode(mode_name, mode_option, description):
+    mode(
+        name=mode_name,
+        description=description,
+    )
+
+    env_var_modification(
+        "CALI_CONFIG_MODE",
+        mode_option,
+        method="append",
+        separator=",",
+        modes=[mode_name],
+    )
+
+
 class Caliper(SpackModifier):
     """Define a modifier for Caliper"""
 
@@ -15,15 +30,57 @@ class Caliper(SpackModifier):
 
     maintainers("pearce8")
 
-    mode("time", description="Platform-independent collection of time")
-
     _cali_datafile = "{experiment_run_dir}/{experiment_name}.cali"
 
+    _default_mode = "time"
+
+    add_mode(
+        mode_name=_default_mode,
+        mode_option="time.exclusive",
+        description="Platform-independent collection of time (default mode)",
+    )
+
     env_var_modification(
         "CALI_CONFIG",
-        "spot(output={})".format(_cali_datafile),
+        "spot(output={}{})".format(_cali_datafile, "${CALI_CONFIG_MODE}"),
         method="set",
-        modes=["time"],
+        modes=[_default_mode],
+    )
+
+    add_mode(
+        mode_name="mpi",
+        mode_option="profile.mpi",
+        description="Profile MPI functions",
+    )
+
+    add_mode(
+        mode_name="cuda",
+        mode_option="profile.cuda",
+        description="Profile CUDA API functions",
+    )
+
+    add_mode(
+        mode_name="topdown-counters-all",
+        mode_option="topdown-counters.all",
+        description="Raw counter values for Intel top-down analysis (all levels)",
+    )
+
+    add_mode(
+        mode_name="topdown-counters-toplevel",
+        mode_option="topdown-counters.toplevel",
+        description="Raw counter values for Intel top-down analysis (top level)",
+    )
+
+    add_mode(
+        mode_name="topdown-all",
+        mode_option="topdown.all",
+        description="Top-down analysis for Intel CPUs (all levels)",
+    )
+
+    add_mode(
+        mode_name="topdown-toplevel",
+        mode_option="topdown.toplevel",
+        description="Top-down analysis for Intel CPUs (top level)",
     )
 
     archive_pattern(_cali_datafile)

var/exp_repo/experiments/amg2023/experiment.py

@@ -0,0 +1,226 @@
+from benchpark.directives import variant
+from benchpark.experiment import Experiment
+from benchpark.expr.builtin.caliper import Caliper
+
+
+class Amg2023(Caliper, Experiment):
+    variant(
+        "programming_model",
+        default="openmp",
+        values=("openmp", "cuda", "rocm"),
+        description="on-node parallelism model",
+    )
+
+    variant(
+        "workload",
+        default="problem1",
+        description="problem1 or problem2",
+    )
+
+    variant(
+        "experiment",
+        default="example",
+        values=("strong", "weak", "example"),
+        description="type of experiment",
+    )
+
+    # TODO: Support list of 3-tuples
+    # variant(
+    #     "p",
+    #     description="value of p",
+    # )
+
+    # TODO: Support list of 3-tuples
+    # variant(
+    #     "n",
+    #     description="value of n",
+    # )
+
+    def make_experiment_example(self):
+        app_name = self.spec.name
+
+        variables = {}
+        matrices = []
+        zips = {}
+
+        if self.spec.satisfies("programming_model=openmp"):
+            # TODO: Support variants
+            n = ["55", "110"]
+            variables["n_nodes"] = ["1", "2"]
+            variables["n_ranks"] = "8"
+            variables["n_threads_per_proc"] = ["4", "6", "12"]
+            exp_name = f"{app_name}_example_omp_{{n_nodes}}_{{n_ranks}}_{{n_threads_per_proc}}_{{px}}_{{py}}_{{pz}}_{{nx}}_{{ny}}_{{nz}}"
+        elif self.spec.satisfies("programming_model=cuda"):
+            # TODO: Support variants
+            n = ["10", "20"]
+            variables["n_gpus"] = "8"
+            exp_name = f"{app_name}_example_cuda_{{n_gpus}}_{{px}}_{{py}}_{{pz}}_{{nx}}_{{ny}}_{{nz}}"
+        elif self.spec.satisfies("programming_model=rocm"):
+            # TODO: Support variants
+            n = ["110", "220"]
+            variables["n_gpus"] = "8"
+            exp_name = f"{app_name}_example_rocm_{{n_gpus}}_{{px}}_{{py}}_{{pz}}_{{nx}}_{{ny}}_{{nz}}"
+        else:
+            raise NotImplementedError(
+                "Unsupported programming_model. Only openmp, cuda and rocm are supported"
+            )
+
+        # TODO: Support variant
+        p = "2"
+        variables["px"] = p
+        variables["py"] = p
+        variables["pz"] = p
+
+        variables["nx"] = n
+        variables["ny"] = n
+        variables["nz"] = n
+        zips["size"] = ["nx", "ny", "nz"]
+
+        m_tag = (
+            "matrices" if self.spec.satisfies("programming_model=openmp") else "matrix"
+        )
+        if self.spec.satisfies("programming_model=openmp"):
+            matrices.append(
+                {"size_nodes_threads": ["size", "n_nodes", "n_threads_per_proc"]}
+            )
+        elif self.spec.satisfies("programming_model=cuda") or self.spec.satisfies(
+            "programming_model=rocm"
+        ):
+            matrices.append("size")
+        else:
+            pass
+
+        excludes = {}
+        if self.spec.satisfies("programming_model=openmp"):
+            excludes["where"] = [
+                "{n_threads_per_proc} * {n_ranks} > {n_nodes} * {sys_cores_per_node}"
+            ]
+
+        return {
+            app_name: {
+                "workloads": {
+                    f"{self.workload}": {
+                        "experiments": {
+                            exp_name: {
+                                "variants": {"package_manager": "spack"},
+                                "variables": variables,
+                                "zips": zips,
+                                "exclude": excludes,
+                                m_tag: matrices,
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+    def compute_modifiers_section(self):
+        return Experiment.compute_modifiers_section(
+            self
+        ) + Caliper.compute_modifiers_section(self)
+
+    def compute_applications_section(self):
+        if self.spec.satisfies("workload=problem1"):
+            self.workload = "problem1"
+        else:
+            self.workload = "problem2"
+
+        if self.spec.satisfies("experiment=example"):
+            return self.make_experiment_example()
+        elif self.spec.satisfies("experiment=strong"):
+            return self.make_experiment_strong()
+        elif self.spec.satisfies("experiment=weak"):
+            return self.make_experiment_weak()
+        else:
+            raise NotImplementedError(
+                "Unsupported experiment. Only strong, weak and example experiments are supported"
+            )
+
+    def compute_spack_section(self):
+        app_name = self.spec.name
+
+        # set package versions
+        app_version = "develop"
+        hypre_version = "2.31.0"
+
+        # get system config options
+        # TODO: Get compiler/mpi/package handles directly from system.py
+        system_specs = {}
+        system_specs["compiler"] = "default-compiler"
+        system_specs["mpi"] = "default-mpi"
+        system_specs["lapack"] = "lapack"
+        if self.spec.satisfies("programming_model=cuda"):
+            system_specs["cuda_version"] = "{default_cuda_version}"
+            system_specs["cuda_arch"] = "{cuda_arch}"
+            system_specs["blas"] = "cublas-cuda"
+        if self.spec.satisfies("programming_model=rocm"):
+            system_specs["rocm_arch"] = "{rocm_arch}"
+            system_specs["blas"] = "blas-rocm"
+
+        # set package spack specs
+        package_specs = {}
+        if self.spec.satisfies("programming_model=cuda"):
+            package_specs["cuda"] = {
+                "pkg_spec": "cuda@{}+allow-unsupported-compilers".format(
+                    system_specs["cuda_version"]
+                ),
+                "compiler": system_specs["compiler"],
+            }
+            package_specs[system_specs["blas"]] = (
+                {}
+            )  # empty package_specs value implies external package
+        if self.spec.satisfies("programming_model=rocm"):
+            package_specs[system_specs["blas"]] = (
+                {}
+            )  # empty package_specs value implies external package
+        package_specs[system_specs["mpi"]] = (
+            {}
+        )  # empty package_specs value implies external package
+        package_specs[system_specs["lapack"]] = (
+            {}
+        )  # empty package_specs value implies external package
+        package_specs["hypre"] = {
+            "pkg_spec": f"hypre@{hypre_version} +mpi+mixedint~fortran",
+            "compiler": system_specs["compiler"],
+        }
+        package_specs[app_name] = {
+            "pkg_spec": f"amg2023@{app_version} +mpi",
+            "compiler": system_specs["compiler"],
+        }
+
+        caliper_package_specs = Caliper.compute_spack_section(self)
+        if Caliper.is_enabled(self):
+            package_specs["hypre"]["pkg_spec"] += "+caliper"
+            package_specs[app_name]["pkg_spec"] += "+caliper"
+        else:
+            package_specs["hypre"]["pkg_spec"] += "~caliper"
+            package_specs[app_name]["pkg_spec"] += "~caliper"
+
+        if self.spec.satisfies("programming_model=openmp"):
+            package_specs["hypre"]["pkg_spec"] += "+openmp"
+            package_specs[app_name]["pkg_spec"] += "+openmp"
+        elif self.spec.satisfies("programming_model=cuda"):
+            package_specs["hypre"]["pkg_spec"] += "+cuda cuda_arch={}".format(
+                system_specs["cuda_arch"]
+            )
+            package_specs[app_name]["pkg_spec"] += "+cuda cuda_arch={}".format(
+                system_specs["cuda_arch"]
+            )
+        elif self.spec.satisfies("programming_model=rocm"):
+            package_specs["hypre"]["pkg_spec"] += "+rocm amdgpu_target={}".format(
+                system_specs["rocm_arch"]
+            )
+            package_specs[app_name]["pkg_spec"] += "+rocm amdgpu_target={}".format(
+                system_specs["rocm_arch"]
+            )
+
+        return {
+            "packages": {k: v for k, v in package_specs.items() if v}
+            | caliper_package_specs["packages"],
+            "environments": {
+                app_name: {
+                    "packages": list(package_specs.keys())
+                    + list(caliper_package_specs["packages"].keys())
+                }
+            },
+        }