diff --git a/pyproject.toml b/pyproject.toml index 631cf48..b663de8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ urls.Homepage = "https://github.com/ecmwf/pyfdb" urls.Issues = "https://github.com/ecmwf/pyfdb/issues" urls.Repository = "https://github.com/ecmwf/pyfdb" -requires-python = ">=3.9" +requires-python = ">=3.8" dependencies = [ "cffi", diff --git a/src/pyfdb/pyfdb.py b/src/pyfdb/pyfdb.py index 3f22169..e8b25c5 100644 --- a/src/pyfdb/pyfdb.py +++ b/src/pyfdb/pyfdb.py @@ -16,6 +16,8 @@ import json import os from functools import wraps +from pathlib import Path +from typing import Optional, Union import cffi import findlibs @@ -290,15 +292,48 @@ class FDB: fdb = pyfdb.FDB() # call fdb.archive, fdb.list, fdb.retrieve, fdb.flush as needed. + # Construct with a particular FDB_HOME + fdb = pyfdb.FDB(fdb_home="/path/to/fdb_home") + + # Construct from a config dictionary + config = dict( + type="local", + engine="toc", + schema="path/to/schema", + spaces=[ + dict( + handler="Default", + roots=[ + {"path": "/path/to/root"}, + ], + ) + ], + ) + fdb = pyfdb.FDB(config) + See the module level pyfdb.list, pyfdb.retrieve, and pyfdb.archive docstrings for more information on these functions. """ __fdb = None - def __init__(self, config=None, user_config=None): + def __init__( + self, + config: Optional[dict] = None, + user_config: Optional[dict] = None, + fdb_home: Union[str, Path, None] = None, + ): + """ + Args: + """ fdb = ffi.new("fdb_handle_t**") + if fdb_home is not None and config is not None: + raise ValueError("Cannot specify both fdb_home and config") + + if fdb_home is not None: + config = dict(fdb_home=str(fdb_home)) + if config is not None or user_config is not None: def prepare_config(c): diff --git a/tests/test_passing_config_directly.py b/tests/test_passing_config_directly.py index fcb947e..7c221f7 100644 --- a/tests/test_passing_config_directly.py +++ b/tests/test_passing_config_directly.py @@ -1,11 +1,74 @@ +import os +import shutil from pathlib import Path from tempfile import TemporaryDirectory +from pytest import raises + import pyfdb +IS_ATOS = os.environ.get("SCRATCHDIR", None) + + +def test_conflicting_arguments(): + "You can't pass both fdb_home and config because the former overrides the latter!" + assert raises(ValueError, pyfdb.FDB, fdb_home="/tmp", config={}) + + +def test_fdb_home(): + with TemporaryDirectory(dir=IS_ATOS) as tmp_home: + tests_dir = Path(__file__).parent + schema_path = tests_dir / "default_fdb_schema" + + home = Path(tmp_home) + etc = home / "etc" / "fdb" + root_path = home / "root" + root_path.mkdir(parents=True) + etc.mkdir(parents=True) + + shutil.copy(schema_path, etc / "schema") + + with open(etc / "config.yaml", "w") as f: + f.write( + f""" +--- +type: local +engine: toc +schema: {etc / "schema"} +spaces: +- handler: Default + roots: + - path: "{root_path}" + writable: true + visit: true +""" + ) + + with open(etc / "config.yaml", "r") as f: + print(f.read()) + + fdb = pyfdb.FDB(fdb_home=home) + data = open(tests_dir / "x138-300.grib", "rb").read() + fdb.archive(data) + fdb.flush() + + list_output = list(fdb.list(keys=True)) + assert len(list_output) == 1 + + # Check that the archive path is in the tmp directory + # On OSX tmp file paths look like /private/var/folders/.../T/tmp.../x138-300.grib + # While the tmp directory looks like /var/folders/.../T/tmp.../ hence why this check is not "startwith" + + # Disabled for now because the HPC has a different convention and I don't know how to deal with all cases + # On the HPC + # tmp_home looks like '/etc/ecmwf/ssd/ssd1/tmpdirs/***.31103395/tmpg137a4ml' + # while the archive path looks like '/etc/ecmwf/ssd/ssd1/tmpdirs/***.31103395/data/fdb/...' + + assert os.path.realpath(tmp_home) in os.path.realpath(list_output[0]["path"]) + def test_direct_config(): - with TemporaryDirectory() as tmp_root: + with TemporaryDirectory(dir=IS_ATOS) as tmp_root: tests_dir = Path(__file__).parent config = dict( @@ -33,11 +96,13 @@ def test_direct_config(): # Check that the archive path is in the tmp directory # On OSX tmp file paths look like /private/var/folders/.../T/tmp.../x138-300.grib # While the tmp directory looks like /var/folders/.../T/tmp.../ hence why this check is not "startwith" - assert tmp_root in list_output[0]["path"] + + # Disabled for now because the HPC has a different convention and I don't know how to deal with all cases + assert os.path.realpath(tmp_root) in os.path.realpath(list_output[0]["path"]) def test_opening_two_fdbs(): - with TemporaryDirectory() as tmp_root1, TemporaryDirectory() as tmp_root2: + with TemporaryDirectory(dir=IS_ATOS) as tmp_root1, TemporaryDirectory(dir=IS_ATOS) as tmp_root2: tests_dir = Path(__file__).parent fdb1 = pyfdb.FDB( @@ -80,4 +145,4 @@ def test_opening_two_fdbs(): for fdb, root in [(fdb1, tmp_root1), (fdb2, tmp_root2)]: list_output = list(fdb.list(keys=True)) assert len(list_output) == 1 - assert root in list_output[0]["path"] + assert os.path.realpath(root) in os.path.realpath(list_output[0]["path"]) diff --git a/tox.ini b/tox.ini index d93e522..337019d 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,10 @@ wheel_build_env = .pkg deps = pytest>=6 pass_env = - FDB_HOME + FDB_HOME # (optional) Tells fdb where to look for config files + FDB5_HOME # (optional) Tell findlibs where to find lib/libfdb5.dylib + ECCODES_HOME # (optional) Tell findlibs where to find lib/libeccodes.dylib + ECCODES_PYTHON_TRACE_LIB_SEARCH # (optional) Print debug info about eccodes search for libeccodes.dylib + ECCODES_PYTHON_USE_FINDLIBS # (optional) Force eccodes to use findlibs to find libeccodes.dylib rather that using the binary installed by pip commands = pytest {tty:--color=yes} {posargs}