Skip to content

Commit

Permalink
Merge pull request #3 from ecmwf/fix/hpc-ci
Browse files Browse the repository at this point in the history 
Update ci.yml - fix hpc downstream branch
  • Loading branch information
@dsarmany
dsarmany authored Jul 4, 2024
2 parents a9c5b96 + bd38d00 commit ba4d2e2
Show file tree
Hide file tree
Showing 3 changed files with 133 additions and 129 deletions.
243 changes: 125 additions & 118 deletions .github/ci-hpc-config.yml
View file
Original file line number Diff line number Diff line change
@@ -1,123 +1,130 @@
matrix:
- openmpi
- hpcx_openmpi
- intel_mpi
- mpi_off
# !!! Turned off the matrix due to connection issues to anaconda when two many runners are connecting? (just a guess). Just one runner works


intel_mpi:
build:
python: '3.10'
modules:
- ninja
- intel-mpi
modules_package:
- eckit:intel-mpi
- atlas:intel-mpi
- multio:intel-mpi
dependency_cmake_options:
- "ecmwf/mir: '-DENABLE_BUILD_TOOLS=ON'"
- "ecmwf/fdb: '-DENABLE_LUSTRE=OFF'"
dependencies:
- ecmwf/ecbuild@develop
- ecmwf/eccodes@develop
- ecmwf/eckit@develop
- ecmwf/metkit@develop
- ecmwf/fckit@develop
- ecmwf/fdb@develop
- ecmwf/atlas@develop
- ecmwf/atlas-orca@develop
- ecmwf/mir@develop
parallel: 64
ntasks: 16
env:
- CTEST_PARALLEL_LEVEL=1
- OMPI_MCA_rmaps_base_oversubscribe=1
- ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
- ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions
# matrix:
# - openmpi
# - hpcx_openmpi
# - intel_mpi
# - mpi_off

hpcx_openmpi:
build:
python: '3.10'
modules:
- ninja
- hpcx-openmpi
modules_package:
- eckit:hpcx-openmpi
- atlas:hpcx-openmpi
- multio:hpcx-openmpi
dependency_cmake_options:
- "ecmwf/mir: '-DENABLE_BUILD_TOOLS=ON'"
- "ecmwf/fdb: '-DENABLE_LUSTRE=OFF'"
dependencies:
- ecmwf/ecbuild@develop
- ecmwf/eccodes@develop
- ecmwf/eckit@develop
- ecmwf/metkit@develop
- ecmwf/fckit@develop
- ecmwf/fdb@develop
- ecmwf/atlas@develop
- ecmwf/atlas-orca@develop
- ecmwf/mir@develop
parallel: 64
ntasks: 16
env:
- CTEST_PARALLEL_LEVEL=1
- OMPI_MCA_rmaps_base_oversubscribe=1
- ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
- ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions

openmpi:
build:
python: '3.10'
modules:
- ninja
- openmpi
modules_package:
- eckit:openmpi
- atlas:openmpi
- multio:openmpi
dependency_cmake_options:
- "ecmwf/mir: '-DENABLE_BUILD_TOOLS=ON'"
- "ecmwf/fdb: '-DENABLE_LUSTRE=OFF'"
dependencies:
- ecmwf/ecbuild@develop
- ecmwf/eccodes@develop
- ecmwf/eckit@develop
- ecmwf/metkit@develop
- ecmwf/fckit@develop
- ecmwf/fdb@develop
- ecmwf/atlas@develop
- ecmwf/atlas-orca@develop
- ecmwf/mir@develop
parallel: 64
ntasks: 16
env:
- CTEST_PARALLEL_LEVEL=1
- OMPI_MCA_rmaps_base_oversubscribe=1
- ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
- ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions
# intel_mpi:
# build:
# python: '3.10'
# modules:
# - ninja
# - intel-mpi
# modules_package:
# - eckit:intel-mpi
# - atlas:intel-mpi
# - multio:intel-mpi
# dependency_cmake_options:
# - "ecmwf/mir: '-DENABLE_BUILD_TOOLS=ON'"
# - "ecmwf/fdb: '-DENABLE_LUSTRE=OFF'"
# dependencies:
# - ecmwf/ecbuild@develop
# - ecmwf/eccodes@develop
# - ecmwf/eckit@develop
# - ecmwf/metkit@develop
# - ecmwf/fckit@develop
# - ecmwf/fdb@develop
# - ecmwf/atlas@develop
# - ecmwf/atlas-orca@develop
# - ecmwf/mir@develop
# - ecmwf/multio@develop
# parallel: 64
# ntasks: 16
# env:
# - CTEST_PARALLEL_LEVEL=1
# - OMPI_MCA_rmaps_base_oversubscribe=1
# - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
# - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions

mpi_off:
build:
python: '3.10'
modules:
- ninja
dependency_cmake_options:
- "ecmwf/mir: '-DENABLE_BUILD_TOOLS=ON'"
- "ecmwf/fdb: '-DENABLE_LUSTRE=OFF'"
dependencies:
- ecmwf/ecbuild@develop
- ecmwf/eccodes@develop
- ecmwf/eckit@develop
- ecmwf/metkit@develop
- ecmwf/fckit@develop
- ecmwf/fdb@develop
- ecmwf/atlas@develop
- ecmwf/atlas-orca@develop
- ecmwf/mir@develop
parallel: 64
env:
- CTEST_PARALLEL_LEVEL=1
- ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
- ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions
# hpcx_openmpi:
# build:
# python: '3.10'
# modules:
# - ninja
# - hpcx-openmpi
# modules_package:
# - eckit:hpcx-openmpi
# - atlas:hpcx-openmpi
# - multio:hpcx-openmpi
# dependency_cmake_options:
# - "ecmwf/mir: '-DENABLE_BUILD_TOOLS=ON'"
# - "ecmwf/fdb: '-DENABLE_LUSTRE=OFF'"
# dependencies:
# - ecmwf/ecbuild@develop
# - ecmwf/eccodes@develop
# - ecmwf/eckit@develop
# - ecmwf/metkit@develop
# - ecmwf/fckit@develop
# - ecmwf/fdb@develop
# - ecmwf/atlas@develop
# - ecmwf/atlas-orca@develop
# - ecmwf/mir@develop
# - ecmwf/multio@develop
# parallel: 64
# ntasks: 16
# env:
# - CTEST_PARALLEL_LEVEL=1
# - OMPI_MCA_rmaps_base_oversubscribe=1
# - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
# - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions

# openmpi:
# build:
# python: '3.10'
# modules:
# - ninja
# - openmpi
# modules_package:
# - eckit:openmpi
# - atlas:openmpi
# - multio:openmpi
# dependency_cmake_options:
# - "ecmwf/mir: '-DENABLE_BUILD_TOOLS=ON'"
# - "ecmwf/fdb: '-DENABLE_LUSTRE=OFF'"
# dependencies:
# - ecmwf/ecbuild@develop
# - ecmwf/eccodes@develop
# - ecmwf/eckit@develop
# - ecmwf/metkit@develop
# - ecmwf/fckit@develop
# - ecmwf/fdb@develop
# - ecmwf/atlas@develop
# - ecmwf/atlas-orca@develop
# - ecmwf/mir@develop
# - ecmwf/multio@develop
# parallel: 64
# ntasks: 16
# env:
# - CTEST_PARALLEL_LEVEL=1
# - OMPI_MCA_rmaps_base_oversubscribe=1
# - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
# - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions

# mpi_off:
build:
python: '3.10'
modules:
- ninja
dependency_cmake_options:
- "ecmwf/mir: '-DENABLE_BUILD_TOOLS=ON'"
- "ecmwf/fdb: '-DENABLE_LUSTRE=OFF'"
dependencies:
- ecmwf/ecbuild@develop
- ecmwf/eccodes@develop
- ecmwf/eckit@develop
- ecmwf/metkit@develop
- ecmwf/fckit@develop
- ecmwf/fdb@develop
- ecmwf/atlas@develop
- ecmwf/atlas-orca@develop
- ecmwf/mir@develop
- ecmwf/multio@develop
parallel: 64
env:
- CTEST_PARALLEL_LEVEL=1
- ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
- ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions
4 changes: 1 addition & 3 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ jobs:
downstream-ci:
name: downstream-ci
if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
# uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
with:
multio-python: ecmwf/multio-python@${{ github.event.pull_request.head.sha || github.sha }}
Expand All @@ -36,8 +35,7 @@ jobs:
downstream-ci-hpc:
name: downstream-ci-hpc
if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
# uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
with:
multio-python: ecmwf/multio-python@${{ github.event.pull_request.head.sha || github.sha }}
secrets: inherit
Expand Down
15 changes: 7 additions & 8 deletions multiopython/lib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,17 +43,16 @@ class PatchedLib:
def __init__(self):
ffi.cdef(self.__read_header())

libnames = [findlibs.find("multio-api")]
libname = findlibs.find("multio-api")
self.__lib = None

if libnames is None:
if libname is None:
raise RuntimeError("Multio is not found")

for libname in [libname for libname in libnames if libname is not None]:
try:
self.__lib = ffi.dlopen(libname)
break
except Exception:
pass
try:
self.__lib = ffi.dlopen(libname)
except Exception as e:
raise RuntimeError("Error loading the following library: {}".format(libname)) from e

# All of the executable members of the CFFI-loaded library are functions in the multio
# C API. These should be wrapped with the correct error handling. Otherwise forward
Expand Down

0 comments on commit ba4d2e2

Please sign in to comment.