diff --git a/features/jupyter.feature b/features/jupyter.feature index 188d07f2f4..65b5173442 100644 --- a/features/jupyter.feature +++ b/features/jupyter.feature @@ -22,6 +22,6 @@ Feature: Jupyter targets in new project Given I have added a test jupyter notebook When I execute the test jupyter notebook and save changes And I execute the kedro jupyter command "convert --all" - And Wait until the process is finished + And Wait until the process is finished for up to "120" seconds Then I should get a successful exit code And Code cell with node tag should be converted into kedro node diff --git a/features/steps/cli_steps.py b/features/steps/cli_steps.py index 76bb0d2722..0008841de4 100644 --- a/features/steps/cli_steps.py +++ b/features/steps/cli_steps.py @@ -352,10 +352,10 @@ def wait_for_notebook_to_run(context, timeout): raise TimeoutError("Failed to run Jupyter server in time") -@when("Wait until the process is finished") -def wait(context): +@when('Wait until the process is finished for up to "{timeout:d}" seconds') +def wait(context, timeout): """Wait for child process to terminate.""" - context.result.wait() + context.result.wait(timeout) @when("I execute the test jupyter notebook and save changes") diff --git a/kedro/io/__init__.py b/kedro/io/__init__.py index 5899f904c6..2888b9f79c 100644 --- a/kedro/io/__init__.py +++ b/kedro/io/__init__.py @@ -3,7 +3,9 @@ """ from __future__ import annotations -from .cached_dataset import CachedDataSet, CachedDataset +import warnings + +from .cached_dataset import CachedDataset from .core import ( AbstractDataSet, AbstractVersionedDataSet, @@ -13,14 +15,17 @@ Version, ) from .data_catalog import DataCatalog -from .lambda_dataset import LambdaDataSet, LambdaDataset -from .memory_dataset import MemoryDataSet, MemoryDataset -from .partitioned_dataset import ( - IncrementalDataSet, - IncrementalDataset, - PartitionedDataSet, - PartitionedDataset, -) +from .lambda_dataset import LambdaDataset +from .memory_dataset import MemoryDataset +from .partitioned_dataset import IncrementalDataset, PartitionedDataset + +with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + from .cached_dataset import CachedDataSet + from .lambda_dataset import LambdaDataSet + from .memory_dataset import MemoryDataSet + from .partitioned_dataset import IncrementalDataSet, PartitionedDataSet + # https://github.com/pylint-dev/pylint/issues/4300#issuecomment-1043601901 DataSetError: type[Exception] diff --git a/kedro/io/cached_dataset.py b/kedro/io/cached_dataset.py index 21e60ab3ef..ed03983b37 100644 --- a/kedro/io/cached_dataset.py +++ b/kedro/io/cached_dataset.py @@ -5,11 +5,14 @@ from __future__ import annotations import logging +import warnings from typing import Any from kedro.io.core import VERSIONED_FLAG_KEY, AbstractDataSet, Version from kedro.io.memory_dataset import MemoryDataset -from kedro.utils import DeprecatedClassMeta + +# https://github.com/pylint-dev/pylint/issues/4300#issuecomment-1043601901 +CachedDataSet: AbstractDataSet class CachedDataset(AbstractDataSet): @@ -120,7 +123,14 @@ def __getstate__(self): return self.__dict__ -class CachedDataSet(metaclass=DeprecatedClassMeta): - # pylint: disable=missing-class-docstring, too-few-public-methods - - _DeprecatedClassMeta__alias = CachedDataset +def __getattr__(name): + if name == "CachedDataSet": + alias = CachedDataset + warnings.warn( + f"{repr(name)} has been renamed to {repr(alias.__name__)}, " + f"and the alias will be removed in Kedro 0.19.0", + DeprecationWarning, + stacklevel=2, + ) + return alias + raise AttributeError(f"module {repr(__name__)} has no attribute {repr(name)}") diff --git a/kedro/io/lambda_dataset.py b/kedro/io/lambda_dataset.py index 7dc96df75e..70378eac92 100644 --- a/kedro/io/lambda_dataset.py +++ b/kedro/io/lambda_dataset.py @@ -4,10 +4,13 @@ """ from __future__ import annotations +import warnings from typing import Any, Callable from kedro.io.core import AbstractDataSet, DatasetError -from kedro.utils import DeprecatedClassMeta + +# https://github.com/pylint-dev/pylint/issues/4300#issuecomment-1043601901 +LambdaDataSet: AbstractDataSet class LambdaDataset(AbstractDataSet): @@ -121,7 +124,14 @@ def __init__( self.metadata = metadata -class LambdaDataSet(metaclass=DeprecatedClassMeta): - # pylint: disable=missing-class-docstring, too-few-public-methods - - _DeprecatedClassMeta__alias = LambdaDataset +def __getattr__(name): + if name == "LambdaDataSet": + alias = LambdaDataset + warnings.warn( + f"{repr(name)} has been renamed to {repr(alias.__name__)}, " + f"and the alias will be removed in Kedro 0.19.0", + DeprecationWarning, + stacklevel=2, + ) + return alias + raise AttributeError(f"module {repr(__name__)} has no attribute {repr(name)}") diff --git a/kedro/io/memory_dataset.py b/kedro/io/memory_dataset.py index 349523c5d3..808fac8f81 100644 --- a/kedro/io/memory_dataset.py +++ b/kedro/io/memory_dataset.py @@ -3,13 +3,16 @@ from __future__ import annotations import copy +import warnings from typing import Any from kedro.io.core import AbstractDataSet, DatasetError -from kedro.utils import DeprecatedClassMeta _EMPTY = object() +# https://github.com/pylint-dev/pylint/issues/4300#issuecomment-1043601901 +MemoryDataSet: AbstractDataSet + class MemoryDataset(AbstractDataSet): """``MemoryDataset`` loads and saves data from/to an in-memory @@ -139,7 +142,14 @@ def _copy_with_mode(data: Any, copy_mode: str) -> Any: return copied_data -class MemoryDataSet(metaclass=DeprecatedClassMeta): - # pylint: disable=missing-class-docstring, too-few-public-methods - - _DeprecatedClassMeta__alias = MemoryDataset +def __getattr__(name): + if name == "MemoryDataSet": + alias = MemoryDataset + warnings.warn( + f"{repr(name)} has been renamed to {repr(alias.__name__)}, " + f"and the alias will be removed in Kedro 0.19.0", + DeprecationWarning, + stacklevel=2, + ) + return alias + raise AttributeError(f"module {repr(__name__)} has no attribute {repr(name)}") diff --git a/kedro/io/partitioned_dataset.py b/kedro/io/partitioned_dataset.py index 683abac283..67ab51df3d 100644 --- a/kedro/io/partitioned_dataset.py +++ b/kedro/io/partitioned_dataset.py @@ -4,10 +4,10 @@ from __future__ import annotations import operator +import warnings from copy import deepcopy from typing import Any, Callable from urllib.parse import urlparse -from warnings import warn from cachetools import Cache, cachedmethod @@ -19,7 +19,7 @@ parse_dataset_definition, ) from kedro.io.data_catalog import CREDENTIALS_KEY -from kedro.utils import DeprecatedClassMeta, load_obj +from kedro.utils import load_obj DATASET_CREDENTIALS_KEY = "dataset_credentials" CHECKPOINT_CREDENTIALS_KEY = "checkpoint_credentials" @@ -31,6 +31,10 @@ S3_PROTOCOLS = ("s3", "s3a", "s3n") +# https://github.com/pylint-dev/pylint/issues/4300#issuecomment-1043601901 +PartitionedDataSet: AbstractDataSet +IncrementalDataSet: AbstractDataSet + class PartitionedDataset(AbstractDataSet): # pylint: disable=too-many-instance-attributes,protected-access @@ -230,7 +234,7 @@ def __init__( # pylint: disable=too-many-arguments self._filepath_arg = filepath_arg if self._filepath_arg in self._dataset_config: - warn( + warnings.warn( f"'{self._filepath_arg}' key must not be specified in the dataset " f"definition as it will be overwritten by partition path" ) @@ -338,12 +342,6 @@ def _release(self) -> None: self._invalidate_caches() -class PartitionedDataSet(metaclass=DeprecatedClassMeta): - # pylint: disable=missing-class-docstring, too-few-public-methods - - _DeprecatedClassMeta__alias = PartitionedDataset - - class IncrementalDataset(PartitionedDataset): """``IncrementalDataset`` inherits from ``PartitionedDataset``, which loads and saves partitioned file-like data using the underlying dataset @@ -560,8 +558,20 @@ def confirm(self) -> None: self._checkpoint.save(partition_ids[-1]) # checkpoint to last partition -class IncrementalDataSet(metaclass=DeprecatedClassMeta): - # pylint: disable=missing-class-docstring, too-few-public-methods +_DEPRECATED_ERROR_CLASSES = { + "PartitionedDataSet": PartitionedDataset, + "IncrementalDataSet": IncrementalDataset, +} + - # pylint: disable=unused-private-member - __DeprecatedClassMeta__alias = IncrementalDataset +def __getattr__(name): + if name in _DEPRECATED_ERROR_CLASSES: + alias = _DEPRECATED_ERROR_CLASSES[name] + warnings.warn( + f"{repr(name)} has been renamed to {repr(alias.__name__)}, " + f"and the alias will be removed in Kedro 0.19.0", + DeprecationWarning, + stacklevel=2, + ) + return alias + raise AttributeError(f"module {repr(__name__)} has no attribute {repr(name)}") diff --git a/kedro/runner/parallel_runner.py b/kedro/runner/parallel_runner.py index fd0fa7d097..542dc74d63 100644 --- a/kedro/runner/parallel_runner.py +++ b/kedro/runner/parallel_runner.py @@ -7,6 +7,7 @@ import os import pickle import sys +import warnings from collections import Counter from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, wait from itertools import chain @@ -27,11 +28,13 @@ from kedro.pipeline import Pipeline from kedro.pipeline.node import Node from kedro.runner.runner import AbstractRunner, run_node -from kedro.utils import DeprecatedClassMeta # see https://github.com/python/cpython/blob/master/Lib/concurrent/futures/process.py#L114 _MAX_WINDOWS_WORKERS = 61 +# https://github.com/pylint-dev/pylint/issues/4300#issuecomment-1043601901 +_SharedMemoryDataSet: Any + class _SharedMemoryDataset: """``_SharedMemoryDataset`` is a wrapper class for a shared MemoryDataset in SyncManager. @@ -70,10 +73,17 @@ def save(self, data: Any): raise exc -class _SharedMemoryDataSet(metaclass=DeprecatedClassMeta): - # pylint: disable=too-few-public-methods - - _DeprecatedClassMeta__alias = _SharedMemoryDataset +def __getattr__(name): + if name == "_SharedMemoryDataSet": + alias = _SharedMemoryDataset + warnings.warn( + f"{repr(name)} has been renamed to {repr(alias.__name__)}, " + f"and the alias will be removed in Kedro 0.19.0", + DeprecationWarning, + stacklevel=2, + ) + return alias + raise AttributeError(f"module {repr(__name__)} has no attribute {repr(name)}") class ParallelRunnerManager(SyncManager): diff --git a/kedro/utils.py b/kedro/utils.py index 380372853c..6067d96b6c 100644 --- a/kedro/utils.py +++ b/kedro/utils.py @@ -3,7 +3,6 @@ """ import importlib from typing import Any -from warnings import warn def load_obj(obj_path: str, default_obj_path: str = "") -> Any: @@ -27,67 +26,3 @@ def load_obj(obj_path: str, default_obj_path: str = "") -> Any: if not hasattr(module_obj, obj_name): raise AttributeError(f"Object '{obj_name}' cannot be loaded from '{obj_path}'.") return getattr(module_obj, obj_name) - - -class DeprecatedClassMeta(type): - """Metaclass for constructing deprecated aliases of renamed classes. - - Code implementation copied from https://stackoverflow.com/a/52087847 - """ - - def __new__(mcs, name, bases, classdict, *args, **kwargs): - alias = classdict.get("_DeprecatedClassMeta__alias") - - if alias is not None: - - def new(mcs, *args, **kwargs): - alias = getattr(mcs, "_DeprecatedClassMeta__alias") - - if alias is not None: - warn( - f"{repr(mcs.__name__)} has been renamed to " - f"{repr(alias.__name__)}, and the alias will " - f"be removed in Kedro 0.19.0", - DeprecationWarning, - stacklevel=2, - ) - - return alias(*args, **kwargs) - - classdict["__new__"] = new - classdict["_DeprecatedClassMeta__alias"] = alias - - fixed_bases = [] - - for base in bases: - alias = getattr(base, "_DeprecatedClassMeta__alias", None) - - if alias is not None: - warn( - f"{repr(base.__name__)} has been renamed to " - f"{repr(alias.__name__)}, and the alias will be " - f"removed in Kedro 0.19.0", - DeprecationWarning, - stacklevel=2, - ) - - # Avoid duplicate base classes. - base = alias or base - if base not in fixed_bases: - fixed_bases.append(base) - - fixed_bases = tuple(fixed_bases) - - return super().__new__(mcs, name, fixed_bases, classdict, *args, **kwargs) - - def __instancecheck__(cls, instance): - return any( - # pylint: disable=no-value-for-parameter - cls.__subclasscheck__(c) - for c in [type(instance), instance.__class__] - ) - - def __subclasscheck__(cls, subclass): - return subclass is cls or issubclass( - subclass, getattr(cls, "_DeprecatedClassMeta__alias") - ) diff --git a/pyproject.toml b/pyproject.toml index 8cdf924f8e..ec06b75d5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ [build-system] # Minimum requirements for the build system to execute. requires = ["setuptools>=65.5.1"] # PEP 518 specifications +build-backend = "setuptools.build_meta" [project] name = "kedro" diff --git a/tests/runner/test_parallel_runner.py b/tests/runner/test_parallel_runner.py index 5b5d37c98a..6870561d8d 100644 --- a/tests/runner/test_parallel_runner.py +++ b/tests/runner/test_parallel_runner.py @@ -1,5 +1,6 @@ from __future__ import annotations +import importlib import sys from concurrent.futures.process import ProcessPoolExecutor from typing import Any @@ -33,6 +34,12 @@ ) +def test_deprecation(): + class_name = "_SharedMemoryDataSet" + with pytest.warns(DeprecationWarning, match=f"{repr(class_name)} has been renamed"): + getattr(importlib.import_module("kedro.runner.parallel_runner"), class_name) + + @pytest.mark.skipif( sys.platform.startswith("win"), reason="Due to bug in parallel runner" ) diff --git a/tests/test_utils.py b/tests/test_utils.py index b6ebe9e95a..4e99f3f726 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,7 +2,7 @@ import pytest -from kedro.utils import DeprecatedClassMeta, load_obj +from kedro.utils import load_obj # pylint: disable=too-few-public-methods @@ -28,53 +28,3 @@ def test_load_obj_invalid_attribute(self): def test_load_obj_invalid_module(self): with pytest.raises(ImportError, match=r"No module named 'missing_path'"): load_obj("InvalidClass", "missing_path") - - -class NewClass: - value = 1 - - -class NewClassSubclass(NewClass): - pass - - -class DeprecatedClass(metaclass=DeprecatedClassMeta): - _DeprecatedClassMeta__alias = NewClass - - -class DeprecatedClassSubclass(DeprecatedClass): - value = 2 - - -class DeprecatedClassSubSubclass(DeprecatedClassSubclass): - value = 3 - - -class TestDeprecatedClassMeta: - def test_is_subclass_of_deprecated_class(self): - assert issubclass(DeprecatedClass, DeprecatedClass) - assert issubclass(DeprecatedClassSubclass, DeprecatedClass) - assert issubclass(DeprecatedClassSubSubclass, DeprecatedClass) - assert issubclass(NewClass, DeprecatedClass) - assert issubclass(NewClassSubclass, DeprecatedClass) - - def test_is_subclass_of_new_class(self): - assert issubclass(DeprecatedClassSubclass, NewClass) - assert issubclass(DeprecatedClassSubSubclass, NewClass) - - def test_is_instance_of_deprecated_class(self): - assert isinstance(DeprecatedClass(), DeprecatedClass) - assert isinstance(DeprecatedClassSubclass(), DeprecatedClass) - assert isinstance(DeprecatedClassSubSubclass(), DeprecatedClass) - assert isinstance(NewClass(), DeprecatedClass) - assert isinstance(NewClassSubclass(), DeprecatedClass) - - def test_is_instance_of_new_class(self): - assert isinstance(DeprecatedClassSubclass(), NewClass) - assert isinstance(DeprecatedClassSubSubclass(), NewClass) - - def test_inheritance(self): - assert NewClass().value == 1 - assert DeprecatedClass().value == 1 # pylint: disable=no-member - assert DeprecatedClassSubclass().value == 2 - assert DeprecatedClassSubSubclass().value == 3