From b89d31314bf1821841ed8ce3c9d40f0582dfc5b5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Apr 2023 13:41:02 +0545 Subject: [PATCH] update template (#17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update template * Apply suggestions from code review * Apply suggestions from code review * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * black preview * fix pylint issues --------- Co-authored-by: skshetry Co-authored-by: skshetry <18718008+skshetry@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Saugat Pachhai (सौगात) --- .cruft.json | 2 +- .pre-commit-config.yaml | 16 ++++++++-------- pyproject.toml | 7 +++++-- setup.cfg | 16 ++++++++++------ src/morefs/asyn_local.py | 4 +--- src/morefs/dict.py | 21 ++++++--------------- src/morefs/memory.py | 16 +++++++--------- src/morefs/overlay.py | 35 ++++++++++++++++------------------- tests/test_asyn_local.py | 20 +++++--------------- tests/test_morefs.py | 4 +--- 10 files changed, 60 insertions(+), 81 deletions(-) diff --git a/.cruft.json b/.cruft.json index f5638d3..d17bbb3 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/iterative/py-template", - "commit": "5fb8af99d07678287dfc10ad1394452b57d23970", + "commit": "c4e24f909659b6ce9c34a1da631290f0c70ff2f2", "checkout": null, "context": { "cookiecutter": { diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6a80404..576f1d8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,11 +2,11 @@ default_language_version: python: python3 repos: - repo: https://github.com/psf/black - rev: 22.8.0 + rev: 23.1.0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -29,21 +29,21 @@ repos: - id: codespell additional_dependencies: ["tomli"] - repo: https://github.com/asottile/pyupgrade - rev: v2.38.0 + rev: v3.3.1 hooks: - id: pyupgrade + args: [--py38-plus] - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/pycqa/flake8 - rev: 5.0.4 + rev: 6.0.0 hooks: - id: flake8 additional_dependencies: - - flake8-broken-line==0.5.0 - - flake8-bugbear==22.9.11 - - flake8-comprehensions==3.10.0 + - flake8-bugbear==23.1.20 + - flake8-comprehensions==3.10.1 - flake8-debugger==4.1.2 - flake8-string-format==0.3.0 - repo: https://github.com/pycqa/bandit diff --git a/pyproject.toml b/pyproject.toml index c39c847..aaa9a98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] [tool.black] -line-length = 79 +line-length = 88 include = '\.pyi?$' exclude = ''' /( @@ -25,7 +25,7 @@ exclude = ''' [tool.isort] profile = "black" known_first_party = ["morefs"] -line_length = 79 +line_length = 88 [tool.pytest.ini_options] addopts = "-ra" @@ -64,6 +64,9 @@ warn_unreachable = true ignore_missing_imports = true files = ["src", "tests"] +[tool.pylint.format] +max-line-length = 88 + [tool.pylint.message_control] disable = [ "format", "refactoring", "spelling", "design", "invalid-name", diff --git a/setup.cfg b/setup.cfg index 91f75aa..95fa9b8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,7 +39,7 @@ all = %(memfs)s %(asynclocalfs)s tests = - pytest==7.1.2 + pytest==7.2.0 pytest-sugar==0.9.5 pytest-cov==3.0.0 pytest-mock==3.8.2 @@ -67,12 +67,16 @@ where=src [flake8] ignore= - E203, # Whitespace before ':' - E266, # Too many leading '#' for block comment - W503, # Line break occurred before a binary operator - P1, # unindexed parameters in the str.format, see: + # Whitespace before ':' + E203 + # Too many leading '#' for block comment + E266 + # Line break occurred before a binary operator + W503 + # unindexed parameters in the str.format, see: # https://pypi.org/project/flake8-string-format/ -max_line_length = 79 + P1 +max_line_length = 88 max-complexity = 15 select = B,C,E,F,W,T4,B902,T,P show_source = true diff --git a/src/morefs/asyn_local.py b/src/morefs/asyn_local.py index f531643..238972b 100644 --- a/src/morefs/asyn_local.py +++ b/src/morefs/asyn_local.py @@ -59,9 +59,7 @@ class AsyncLocalFileSystem(AsyncFileSystem, LocalFileSystem): _write_text = wrap(LocalFileSystem.write_text) sign = LocalFileSystem.sign - async def _get_file( - self, src, dst, **kwargs - ): # pylint: disable=arguments-renamed + async def _get_file(self, src, dst, **kwargs): # pylint: disable=arguments-renamed if not iscoroutinefunction(getattr(dst, "write", None)): src = self._strip_protocol(src) return await self._get_file_async(src, dst) diff --git a/src/morefs/dict.py b/src/morefs/dict.py index 8c3629a..5ac03b5 100644 --- a/src/morefs/dict.py +++ b/src/morefs/dict.py @@ -18,9 +18,7 @@ def __init__(self, paths: Iterable[str] = ()) -> None: def new_child(self, paths: Iterable[str]) -> None: self.set(paths, type(self)(paths=paths)) - def set( - self, paths: Iterable[str], value: Any, overwrite: bool = False - ) -> None: + def set(self, paths: Iterable[str], value: Any, overwrite: bool = False) -> None: if not paths: raise ValueError("no path supplied") @@ -31,9 +29,7 @@ def set( raise ValueError("cannot overwrite - item exists") child[key] = value - def get( # type: ignore[override] - self, paths: Iterable[str] - ) -> "ContainerOrFile": + def get(self, paths: Iterable[str]) -> "ContainerOrFile": # type: ignore[override] child = self for path in paths: child = child[path] @@ -67,18 +63,14 @@ def _strip_protocol(cls, path: str) -> str: path = path.lstrip("/").rstrip("/") return "/" + path if path else cls.root_marker - def __init__(self, store: Store = None) -> None: + def __init__(self, store: Optional[Store] = None) -> None: super().__init__() if store is None: store = Store() self.store = store def _info( - self, - path: str, - item: ContainerOrFile, - file: bool = False, - **kwargs: Any + self, path: str, item: ContainerOrFile, file: bool = False, **kwargs: Any ) -> Dict[str, Any]: if isinstance(item, dict): return {"name": path, "size": 0, "type": "directory"} @@ -135,8 +127,7 @@ def ls(self, path: str, detail: bool = False, **kwargs: Any): if not detail: return [self.join_paths((*paths, key)) for key, _ in entries] return [ - self._info(self.join_paths((*paths, key)), value) - for key, value in entries + self._info(self.join_paths((*paths, key)), value) for key, value in entries ] def _rm(self, path: str) -> None: @@ -260,7 +251,7 @@ def rm( self, path: Union[str, List[str]], recursive: bool = False, - maxdepth: int = None, + maxdepth: Optional[int] = None, ) -> None: if isinstance(path, str): paths = [path] diff --git a/src/morefs/memory.py b/src/morefs/memory.py index 13c4dc7..4cc1ddb 100644 --- a/src/morefs/memory.py +++ b/src/morefs/memory.py @@ -28,9 +28,11 @@ def _info(path, filelike=None, **kwargs): if filelike: return { "name": path, - "size": filelike.size - if hasattr(filelike, "size") - else filelike.getbuffer().nbytes, + "size": ( + filelike.size + if hasattr(filelike, "size") + else filelike.getbuffer().nbytes + ), "type": "file", "created": getattr(filelike, "created", None), } @@ -54,9 +56,7 @@ def node_factory(path_conv, parts, children, filelike=None): except KeyError as exc: if path in ("", "/"): return [] - raise FileNotFoundError( - errno.ENOENT, "No such file", path - ) from exc + raise FileNotFoundError(errno.ENOENT, "No such file", path) from exc return out @@ -87,9 +87,7 @@ def rm(self, path, recursive=False, maxdepth=None): for p in paths: self.store.pop(p, None) - def _open( # pylint: disable=arguments-differ - self, path, mode="rb", **kwargs - ): + def _open(self, path, mode="rb", **kwargs): # pylint: disable=arguments-differ path = self._strip_protocol(path) try: info = self.info(path) diff --git a/src/morefs/overlay.py b/src/morefs/overlay.py index 4c8e7b9..7569a29 100644 --- a/src/morefs/overlay.py +++ b/src/morefs/overlay.py @@ -9,13 +9,12 @@ class OverlayFileSystem(fsspec.AbstractFileSystem): cachable = False - def __init__(self, *fses, **kwargs): + def __init__(self, *fses: fsspec.AbstractFileSystem, **kwargs): storage_options = { - key: value - for key, value in kwargs.items() - if key.startswith("fs_") + key: value for key, value in kwargs.items() if key.startswith("fs_") } self.fses: List[fsspec.AbstractFileSystem] = list(fses) + self.fses.extend(kwargs.pop("filesystems", [])) for proto, options in kwargs.items(): if proto.startswith("fs_"): continue @@ -79,21 +78,17 @@ def inner(self, path, *args, **kwargs): def _raise_readonly(path, *args, **kwargs): raise OSError(errno.EROFS, os.strerror(errno.EROFS), path) - info = _iterate_fs_with("info") - created = _iterate_fs_with("created") - modified = _iterate_fs_with("modified") + info = _iterate_fs_with("info") # pylint: disable=no-value-for-parameter + created = _iterate_fs_with("created") # pylint: disable=no-value-for-parameter + modified = _iterate_fs_with("modified") # pylint: disable=no-value-for-parameter def mkdir(self, path, create_parents=True, **kwargs): # if create_parents is False: if self.exists(path): - raise FileExistsError( - errno.EEXIST, os.strerror(errno.EEXIST), path - ) + raise FileExistsError(errno.EEXIST, os.strerror(errno.EEXIST), path) parent = self._parent(path) if not create_parents and not self.isdir(parent): - raise FileNotFoundError( - errno.ENOENT, os.strerror(errno.ENOENT), path - ) + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) self.upper_fs.mkdir(path, create_parents=True, **kwargs) def makedirs(self, path, exist_ok=False): @@ -113,18 +108,14 @@ def cp_file(self, path1, path2, **kwargs): break if not src_fs: - raise FileNotFoundError( - errno.ENOENT, os.strerror(errno.ENOENT), path1 - ) + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path1) if src_fs == self.upper_fs: return src_fs.cp_file(path1, path2) with src_fs.open(path1) as src, self.upper_fs.open(path2, "wb") as dst: shutil.copyfileobj(src, dst) - def _open( - self, path, mode="rb", **kwargs - ): # pylint: disable=arguments-differ + def _open(self, path, mode="rb", **kwargs): # pylint: disable=arguments-differ if "rb" in mode: for fs in self.fses: try: @@ -165,3 +156,9 @@ def _open( def sign(self, path, expiration=100, **kwargs): return self.upper_fs.sign(path, expiration, **kwargs) + + if hasattr(fsspec.AbstractFileSystem, "fsid"): + + @property + def fsid(self): + return "overlay_" + "+".join(fs.fsid for fs in self.fses) diff --git a/tests/test_asyn_local.py b/tests/test_asyn_local.py index a61c7b4..3c220f7 100644 --- a/tests/test_asyn_local.py +++ b/tests/test_asyn_local.py @@ -29,23 +29,17 @@ async def test_ls(tmp_path, localfs, fs): assert set(await fs._ls(tmp_path, detail=False)) == { localfs._strip_protocol(tmp_path / f) for f in ["foo", "bar", "dir"] } - assert await fs._ls(tmp_path, detail=False) == localfs.ls( - tmp_path, detail=False - ) + assert await fs._ls(tmp_path, detail=False) == localfs.ls(tmp_path, detail=False) assert await fs._info(tmp_path / "foo") == localfs.info(tmp_path / "foo") assert await fs._info(tmp_path / "dir") == localfs.info(tmp_path / "dir") - assert await fs._ls(tmp_path, detail=True) == localfs.ls( - tmp_path, detail=True - ) + assert await fs._ls(tmp_path, detail=True) == localfs.ls(tmp_path, detail=True) assert await fs._find(tmp_path, detail=False) == localfs.find( tmp_path, detail=False ) - assert await fs._find(tmp_path, detail=True) == localfs.find( - tmp_path, detail=True - ) + assert await fs._find(tmp_path, detail=True) == localfs.find(tmp_path, detail=True) assert await fs._isfile(tmp_path / "foo") assert await fs._isdir(tmp_path / "dir") @@ -72,12 +66,8 @@ def test_sync_methods(tmp_path, localfs, fs): assert fs.info(tmp_path / "dir") == localfs.info(tmp_path / "dir") assert fs.ls(tmp_path, detail=True) == localfs.ls(tmp_path, detail=True) - assert fs.find(tmp_path, detail=False) == localfs.find( - tmp_path, detail=False - ) - assert fs.find(tmp_path, detail=True) == localfs.find( - tmp_path, detail=True - ) + assert fs.find(tmp_path, detail=False) == localfs.find(tmp_path, detail=False) + assert fs.find(tmp_path, detail=True) == localfs.find(tmp_path, detail=True) assert fs.isfile(tmp_path / "foo") assert fs.isdir(tmp_path / "dir") diff --git a/tests/test_morefs.py b/tests/test_morefs.py index cf57424..94119c1 100644 --- a/tests/test_morefs.py +++ b/tests/test_morefs.py @@ -6,9 +6,7 @@ from morefs.memory import MemFS -@pytest.mark.parametrize( - "proto, fs_cls", [("dictfs", DictFS), ("memfs", MemFS)] -) +@pytest.mark.parametrize("proto, fs_cls", [("dictfs", DictFS), ("memfs", MemFS)]) def test_fsspec(proto, fs_cls): fs = fsspec.filesystem(proto) assert isinstance(fs, fs_cls)