Skip to content

Commit

Permalink
(#18980) openmpi: migrate to Conan v2
Browse files Browse the repository at this point in the history 
* openmpi: migrate to Conan v2

* openmpi: bump deps

* openmpi: do not print created env vars

* openmpi: add v4.1.6

* openmpi: improve options coverage

* openmpi: fix path env vars

* openmpi: fix a PMIX error

* openmpi: add MPI::MPI_CXX only if `cxx` is enabled

* openmpi: fix static build issues, add external_hwloc option

* openmpi: fix libltdl issue, replace AutotoolsDeps with PkgConfigDeps

* openmpi: disable buildign of docs

* openmpi: drop libltdl dependency

* openmpi: external libevent does not really work

* openmpi: model components correctly

* openmpi: fix pkg-config names

* openmpi: disable armv8 for v4.1.0

* openmpi: bump libnl

* openmpi: enable with_verbs

* openmpi: use REQUIRED CONFIG in test_package for legacy generators

* bump rdma-core

* openmpi: fix libibverbs.so not being found

* openmpi: drop test_v1_package

* openmpi: fix apple-clang cross-compilation

* openmpi: run autoreconf

* openmpi: drop v4.1.0

* openmpi: Revert "run autoreconf"

This reverts commit 6f678da.

* openmpi: disable Clang on Conan v1

* openmpi: restore v4.1.0

This reverts commit 0461166.

* openmpi: add --allow-run-as-root to test_package

Co-authored-by: Maksim Petukhov <6967052+maksim-petukhov@users.noreply.github.com>

* openmpi: adjust deps

* openmpi: enable cxx by default

* openmpi: drop external_hwloc option

* openmpi: fix an invalid require name

* openmpi: add missing requires

* openmpi: add VirtualRunEnv to fix ./configure

* openmpi: workaround for macOS build failure

* openmpi: re-enable Clang/AppleClang for Conan v1

* openmpi: fix_apple_shared_install_name()

* openmpi: apply review suggestions

* openmpi: drop --with-libevent=internal

* openmpi: drop --allow-run-as-root in test_package

* openmpi: add a comment for pkg_config_name

Co-authored-by: Uilian Ries <uilianries@gmail.com>

* openmpi: get_safe("with_verbs")

* openmpi: Revert "re-enable Clang/AppleClang for Conan v1"

This reverts commit 40d407e.

* openmpi: VirtualRunEnv is required to find libhwloc.so during ./configure

* openmpi: macOS armv8 fails when shared=True

* Simplify test package to avoid networking usage

The current openmpi test package tries to use networking,
and in OS like Mac, the GUI asks about permission to
execute the test package. It's too much for a simple
package validation. It would be better keep a version
check for library linkage validation.

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* Revert "openmpi: macOS armv8 fails when shared=True"

Enable again dynamic library build for Mac. With the simplified
test package, I see no errors locally for Mac M1.

This reverts commit d79c757.

---------

Signed-off-by: Uilian Ries <uilianries@gmail.com>
Co-authored-by: Maksim Petukhov <6967052+maksim-petukhov@users.noreply.github.com>
Co-authored-by: Uilian Ries <uilianries@gmail.com>
  • Loading branch information
@valgur @maksim-petukhov @uilianries
3 people authored Sep 5, 2024
1 parent 18c96ad commit 30a5ab7
Show file tree
Hide file tree
Showing 7 changed files with 265 additions and 101 deletions.
7 changes: 5 additions & 2 deletions recipes/openmpi/all/conandata.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
sources:
"4.1.6":
url: "https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.6.tar.bz2"
sha256: "f740994485516deb63b5311af122c265179f5328a0d857a567b85db00b11e415"
"4.1.0":
sha256: 73866fb77090819b6a8c85cb8539638d37d6877455825b74e289d647a39fd5b5
url: https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.0.tar.bz2
url: "https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.0.tar.bz2"
sha256: "73866fb77090819b6a8c85cb8539638d37d6877455825b74e289d647a39fd5b5"
284 changes: 223 additions & 61 deletions recipes/openmpi/all/conanfile.py
View file
Original file line number Diff line number Diff line change
@@ -1,97 +1,259 @@
from conans import ConanFile, tools, AutoToolsBuildEnvironment
from conans.errors import ConanInvalidConfiguration
import os

required_conan_version = ">=1.29.1"
from conan import ConanFile
from conan.errors import ConanInvalidConfiguration
from conan.tools.apple import is_apple_os, fix_apple_shared_install_name
from conan.tools.env import VirtualRunEnv
from conan.tools.files import copy, get, rm, rmdir, save, replace_in_file
from conan.tools.gnu import Autotools, AutotoolsToolchain, AutotoolsDeps
from conan.tools.layout import basic_layout
from conan.tools.microsoft import unix_path

required_conan_version = ">=1.53.0"


class OpenMPIConan(ConanFile):
name = "openmpi"
homepage = "https://www.open-mpi.org"
url = "https://github.com/conan-io/conan-center-index"
topics = ("conan", "mpi", "openmpi")
description = "A High Performance Message Passing Library"
license = "BSD-3-Clause"
url = "https://github.com/conan-io/conan-center-index"
homepage = "https://www.open-mpi.org"
topics = ("mpi", "openmpi")
provides = "mpi"
package_type = "library"
settings = "os", "arch", "compiler", "build_type"
options = {
"shared": [True, False],
"fPIC": [True, False],
"fortran": ["yes", "mpifh", "usempi", "usempi80", "no"]
"fortran": ["yes", "mpifh", "usempi", "usempi80", "no"],
"enable_cxx": [True, False],
"enable_cxx_exceptions": [True, False],
"with_verbs": [True, False],
}
default_options = {
"shared": False,
"fPIC": True,
"fortran": "no"
"fortran": "no",
"enable_cxx": False,
"enable_cxx_exceptions": False,
"with_verbs": False,
}

_autotools = None

@property
def _source_subfolder(self):
return "source_subfolder"
def config_options(self):
if self.settings.os == "Windows":
del self.options.fPIC

def configure(self):
if self.options.shared:
del self.options.fPIC
del self.settings.compiler.libcxx
del self.settings.compiler.cppstd
self.options.rm_safe("fPIC")
if not self.options.enable_cxx:
self.settings.rm_safe("compiler.libcxx")
self.settings.rm_safe("compiler.cppstd")
del self.options.enable_cxx_exceptions
if is_apple_os(self):
# Unavailable due to dependency on libnl
del self.options.with_verbs

def layout(self):
basic_layout(self, src_folder="src")

def requirements(self):
# OpenMPI public headers don't include anything besides stddef.h.
# transitive_headers=True is not needed for any dependencies.
self.requires("hwloc/2.10.0")
self.requires("zlib/[>=1.2.11 <2]")
if self.settings.os == "Linux":
self.requires("libnl/3.8.0")
if self.options.get_safe("with_verbs"):
self.requires("rdma-core/52.0")

def validate(self):
if self.settings.os == "Windows":
# Requires Cygwin or WSL
raise ConanInvalidConfiguration("OpenMPI doesn't support Windows")

def requirements(self):
# FIXME : self.requires("libevent/2.1.12") - try to use libevent from conan
self.requires("zlib/1.2.11")
if self.version == "4.1.0" and is_apple_os(self) and self.settings.arch == "armv8":
# INFO: https://github.com/open-mpi/ompi/issues/8410
raise ConanInvalidConfiguration(f"{self.ref} is not supported in Mac M1. Use a newer version.")

def source(self):
tools.get(**self.conan_data["sources"][self.version])
extracted_dir = self.name + "-" + self.version
os.rename(extracted_dir, self._source_subfolder)

def _configure_autotools(self):
if self._autotools:
return self._autotools
self._autotools = AutoToolsBuildEnvironment(self)
args = ["--disable-wrapper-rpath", "--disable-wrapper-runpath"]
if self.settings.build_type == "Debug":
args.append("--enable-debug")
if self.options.shared:
args.extend(["--enable-shared", "--disable-static"])
else:
args.extend(["--enable-static", "--disable-shared"])
args.append("--with-pic" if self.options.get_safe("fPIC", True) else "--without-pic")
args.append("--enable-mpi-fortran={}".format(str(self.options.fortran)))
args.append("--with-zlib={}".format(self.deps_cpp_info["zlib"].rootpath))
args.append("--with-zlib-libdir={}".format(self.deps_cpp_info["zlib"].lib_paths[0]))
args.append("--datarootdir=${prefix}/res")
self._autotools.configure(args=args)
return self._autotools
get(self, **self.conan_data["sources"][self.version], strip_root=True)

def generate(self):
def root(pkg):
return unix_path(self, self.dependencies[pkg].package_folder)

def yes_no(v):
return "yes" if v else "no"

tc = AutotoolsToolchain(self)
tc.configure_args += [
f"--enable-mpi-fortran={self.options.fortran}",
f"--enable-mpi-cxx={yes_no(self.options.enable_cxx)}",
f"--enable-cxx-exceptions={yes_no(self.options.get_safe('enable_cxx_exceptions'))}",
f"--with-hwloc={root('hwloc')}",
f"--with-libnl={root('libnl') if not is_apple_os(self) else 'no'}",
f"--with-verbs={root('rdma-core') if self.options.get_safe('with_verbs') else 'no'}",
f"--with-zlib={root('zlib')}",
"--with-pic" if self.options.get_safe("fPIC", True) else "--without-pic",
"--disable-wrapper-rpath",
"--disable-wrapper-runpath",
"--exec-prefix=/",
"--datarootdir=${prefix}/res",
# Disable other external libraries explicitly
"--with-alps=no", # ALPS
"--with-cuda=no", # CUDA
"--with-fca=no", # FCA
"--with-gpfs=no", # Gpfs
"--with-hcoll=no", # hcoll
"--with-ime=no", # IME
"--with-lsf=no", # LSF
"--with-lustre=no", # Lustre
"--with-memkind=no", # memkind
"--with-moab=no", # Moab
"--with-mxm=no", # Mellanox MXM
"--with-ofi=no", # libfabric, TODO: enable once libfabric is available
"--with-pmi=no", # PMI
"--with-pmix=internal", # PMIx
"--with-portals4=no", # Portals4
"--with-psm2=no", # PSM2
"--with-psm=no", # PSM
"--with-pvfs2=no", # Pvfs2
"--with-treematch=no", # TreeMatch
"--with-ucx=no", # UCX
"--with-valgrind=no", # Valgrind
"--with-x=no", # X11
"--with-xpmem=no", # XPMEM
]
if is_apple_os(self):
if self.settings.arch == "armv8":
tc.configure_args.append("--host=aarch64-apple-darwin")
tc.extra_ldflags.append("-arch arm64")
# macOS has no libnl
tc.configure_args.append("--enable-mca-no-build=reachable-netlink")
# libtool's libltdl is not really needed, OpenMPI provides its own equivalent.
# Not adding it as it fails to be detected by ./configure in some cases.
# https://github.com/open-mpi/ompi/blob/v4.1.6/opal/mca/dl/dl.h#L20-L25
tc.configure_args.append("--with-libltdl=no")
tc.generate()

deps = AutotoolsDeps(self)
deps.generate()

# Needed for ./configure to find libhwloc.so and libibnetdisc.so
VirtualRunEnv(self).generate(scope="build")

# TODO: might want to enable reproducible builds by setting
# $SOURCE_DATE_EPOCH, $USER and $HOSTNAME

def _patch_sources(self):
# Not needed and fails with v5.0 due to additional Python dependencies
save(self, os.path.join(self.source_folder, "docs", "Makefile.in"), "all:\ninstall:\n")
# Workaround for <cstddef> trying to include VERSION from source dir due to a case-insensitive filesystem on macOS
# Based on https://github.com/macports/macports-ports/blob/22dded99ae76a287f04a9685bbc820ecaa397fea/science/openmpi/files/patch-configure.diff
replace_in_file(self, os.path.join(self.source_folder, "configure"),
"-I$(top_srcdir) ", "-idirafter$(top_srcdir) ")

def build(self):
with tools.chdir(self._source_subfolder):
autotools = self._configure_autotools()
autotools.make()
self._patch_sources()
autotools = Autotools(self)
autotools.configure()
autotools.make()

def package(self):
self.copy(pattern="LICENSE", src=self._source_subfolder, dst="licenses")
with tools.chdir(self._source_subfolder):
autotools = self._configure_autotools()
autotools.install()
tools.rmdir(os.path.join(self.package_folder, "lib", "pkgconfig"))
tools.rmdir(os.path.join(self.package_folder, "share"))
tools.rmdir(os.path.join(self.package_folder, "etc"))
tools.remove_files_by_mask(os.path.join(self.package_folder, "lib"), "*.la")
copy(self, "LICENSE", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses"))
autotools = Autotools(self)
autotools.install()
rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig"))
rmdir(self, os.path.join(self.package_folder, "etc"))
rmdir(self, os.path.join(self.package_folder, "res", "man"))
rm(self, "*.la", self.package_folder, recursive=True)
fix_apple_shared_install_name(self)

def package_info(self):
self.cpp_info.libs = ['mpi', 'open-rte', 'open-pal']
# Based on https://cmake.org/cmake/help/latest/module/FindMPI.html#variables-for-using-mpi
self.cpp_info.set_property("cmake_find_mode", "both")
self.cpp_info.set_property("cmake_file_name", "MPI")
# TODO: Use None when available as Conan feature.
self.cpp_info.set_property("pkg_config_name", "_ompi-do-not-use")
# TODO: export a .cmake module to correctly set all variables set by CMake's FindMPI.cmake

requires = [
"hwloc::hwloc",
"zlib::zlib",
]
if self.settings.os == "Linux":
self.cpp_info.system_libs = ["dl", "pthread", "rt", "util"]
requires.append("libnl::libnl")
if self.options.get_safe("with_verbs"):
requires.extend(["rdma-core::libibverbs", "rdma-core::librdmacm"])

# The components are modelled based on OpenMPI's pkg-config files

self.output.info("Creating MPI_HOME environment variable: {}".format(self.package_folder))
# Run-time environment library
self.cpp_info.components["orte"].set_property("pkg_config_name", "orte")
self.cpp_info.components["orte"].libs = ["open-rte", "open-pal"]
self.cpp_info.components["orte"].includedirs.append(os.path.join("include", "openmpi"))
if self.settings.os in ["Linux", "FreeBSD"]:
self.cpp_info.components["orte"].system_libs = ["dl", "pthread", "rt", "util"]
self.cpp_info.components["orte"].cflags = ["-pthread"]
if self.options.get_safe("enable_cxx_exceptions"):
self.cpp_info.components["orte"].cflags.append("-fexceptions")
self.cpp_info.components["orte"].requires = requires

self.cpp_info.components["ompi"].set_property("pkg_config_name", "ompi")
self.cpp_info.components["ompi"].libs = ["mpi"]
self.cpp_info.components["ompi"].requires = ["orte"]

self.cpp_info.components["ompi-c"].set_property("pkg_config_name", "ompi-c")
self.cpp_info.components["ompi-c"].set_property("cmake_target_name", "MPI::MPI_C")
self.cpp_info.components["ompi-c"].requires = ["ompi"]

self.cpp_info.components["ompitrace"].set_property("pkg_config_name", "ompitrace")
self.cpp_info.components["ompitrace"].libs = ["ompitrace"]
self.cpp_info.components["ompitrace"].requires = ["ompi"]

if self.options.enable_cxx:
self.cpp_info.components["ompi-cxx"].set_property("pkg_config_name", "ompi-cxx")
self.cpp_info.components["ompi-cxx"].set_property("cmake_target_name", "MPI::MPI_CXX")
self.cpp_info.components["ompi-cxx"].libs = ["mpi_cxx"]
self.cpp_info.components["ompi-cxx"].requires = ["ompi"]

if self.options.fortran != "no":
self.cpp_info.components["ompi-fort"].set_property("pkg_config_name", "ompi-fort")
self.cpp_info.components["ompi-fort"].set_property("cmake_target_name", "MPI::MPI_Fortran")
self.cpp_info.components["ompi-fort"].libs = ["mpi_mpifh"]
self.cpp_info.components["ompi-fort"].requires = ["ompi"]
# Aliases
self.cpp_info.components["ompi-f77"].set_property("pkg_config_name", "ompi-f77")
self.cpp_info.components["ompi-f77"].requires = ["ompi-fort"]
self.cpp_info.components["ompi-f90"].set_property("pkg_config_name", "ompi-f90")
self.cpp_info.components["ompi-f90"].requires = ["ompi-fort"]

bin_folder = os.path.join(self.package_folder, "bin")
# Prepend to PATH to avoid a conflict with system MPI
self.runenv_info.prepend_path("PATH", bin_folder)
self.runenv_info.define_path("MPI_BIN", bin_folder)
self.runenv_info.define_path("MPI_HOME", self.package_folder)
self.runenv_info.define_path("OPAL_PREFIX", self.package_folder)
self.runenv_info.define_path("OPAL_EXEC_PREFIX", self.package_folder)
self.runenv_info.define_path("OPAL_LIBDIR", os.path.join(self.package_folder, "lib"))
self.runenv_info.define_path("OPAL_DATADIR", os.path.join(self.package_folder, "res"))
self.runenv_info.define_path("OPAL_DATAROOTDIR", os.path.join(self.package_folder, "res"))

# TODO: Legacy, to be removed on Conan 2.0
self.env_info.PATH.append(bin_folder)
self.env_info.MPI_BIN = bin_folder
self.env_info.MPI_HOME = self.package_folder
self.output.info("Creating OPAL_PREFIX environment variable: {}".format(self.package_folder))
self.env_info.OPAL_PREFIX = self.package_folder
mpi_bin = os.path.join(self.package_folder, 'bin')
self.output.info("Creating MPI_BIN environment variable: {}".format(mpi_bin))
self.env_info.MPI_BIN = mpi_bin
self.output.info("Appending PATH environment variable: {}".format(mpi_bin))
self.env_info.PATH.append(mpi_bin)
self.env_info.OPAL_EXEC_PREFIX = self.package_folder
self.env_info.OPAL_LIBDIR = os.path.join(self.package_folder, "lib")
self.env_info.OPAL_DATADIR = os.path.join(self.package_folder, "res")
self.env_info.OPAL_DATAROOTDIR = os.path.join(self.package_folder, "res")

self.cpp_info.names["cmake_find_package"] = "MPI"
self.cpp_info.names["cmake_find_package_multi"] = "MPI"
self.cpp_info.components["ompi-c"].names["cmake_find_package"] = "MPI_C"
if self.options.enable_cxx:
self.cpp_info.components["ompi-cxx"].names["cmake_find_package"] = "MPI_CXX"
if self.options.fortran != "no":
self.cpp_info.components["ompi-fort"].names["cmake_find_package"] = "MPI_Fortran"
13 changes: 5 additions & 8 deletions recipes/openmpi/all/test_package/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
cmake_minimum_required(VERSION 3.1)
project(test_package)
cmake_minimum_required(VERSION 3.15)
project(test_package LANGUAGES C)

include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
conan_basic_setup()
find_package(MPI REQUIRED CONFIG)

find_package(MPI REQUIRED)

add_executable(${PROJECT_NAME} test_package.cpp)
target_link_libraries(${PROJECT_NAME} ${CONAN_LIBS})
add_executable(${PROJECT_NAME} test_package.c)
target_link_libraries(${PROJECT_NAME} PRIVATE MPI::MPI_C)
24 changes: 17 additions & 7 deletions recipes/openmpi/all/test_package/conanfile.py
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,28 @@
from conans import ConanFile, CMake, tools
import os

from conan import ConanFile
from conan.tools.build import can_run
from conan.tools.cmake import cmake_layout, CMake


class TestPackageConan(ConanFile):
settings = "os", "compiler", "build_type", "arch"
generators = "cmake"
settings = "os", "arch", "compiler", "build_type"
generators = "CMakeDeps", "CMakeToolchain", "VirtualRunEnv"
test_type = "explicit"

def requirements(self):
self.requires(self.tested_reference_str)

def layout(self):
cmake_layout(self)

def build(self):
cmake = CMake(self)
cmake.configure()
cmake.build()

def test(self):
if not tools.cross_building(self.settings):
mpiexec = os.path.join(os.environ['MPI_BIN'], 'mpiexec')
command = '%s -mca plm_rsh_agent yes -np 2 %s' % (mpiexec, os.path.join("bin", "test_package"))
self.run(command, run_environment=True)
if can_run(self):
bin_path = os.path.join(self.cpp.build.bindir, "test_package")
command = f"mpiexec -mca plm_rsh_agent yes {bin_path}"
self.run(command, env="conanrun")
13 changes: 13 additions & 0 deletions recipes/openmpi/all/test_package/test_package.c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#include <stdio.h>
#include <stdlib.h>

#include <mpi.h>

int main(int argc, char* argv[])
{
char version[MPI_MAX_LIBRARY_VERSION_STRING] = {0};
int len = 0;
MPI_Get_library_version(version, &len);
printf("MPI version: %s\n", version);
return EXIT_SUCCESS;
}
Loading

0 comments on commit 30a5ab7

Please sign in to comment.