Remove DeprecatedClassMeta in favor of getattr (#2724)

* Complete build requirements

Signed-off-by: Juan Luis Cano Rodríguez <juan_luis_cano@mckinsey.com>

* Add timeout for e2e process waiting

Signed-off-by: Juan Luis Cano Rodríguez <juan_luis_cano@mckinsey.com>

* Remove DeprecatedClassMeta in favor of getattr

Fix kedro-org/kedro-starters#137

Signed-off-by: Juan Luis Cano Rodríguez <juan_luis_cano@mckinsey.com>

* Ignore deprecation warnings in our own code

Signed-off-by: Juan Luis Cano Rodríguez <juan_luis_cano@mckinsey.com>

---------

Signed-off-by: Juan Luis Cano Rodríguez <juan_luis_cano@mckinsey.com>
Co-authored-by: Juan Luis Cano Rodríguez <juan_luis_cano@mckinsey.com>
Co-authored-by: Deepyaman Datta <deepyaman.datta@utexas.edu>

features/jupyter.feature

@@ -22,6 +22,6 @@ Feature: Jupyter targets in new project
     Given I have added a test jupyter notebook
     When I execute the test jupyter notebook and save changes
     And I execute the kedro jupyter command "convert --all"
-    And Wait until the process is finished
+    And Wait until the process is finished for up to "120" seconds
     Then I should get a successful exit code
     And Code cell with node tag should be converted into kedro node

features/steps/cli_steps.py

@@ -352,10 +352,10 @@ def wait_for_notebook_to_run(context, timeout):
         raise TimeoutError("Failed to run Jupyter server in time")
 
 
-@when("Wait until the process is finished")
-def wait(context):
+@when('Wait until the process is finished for up to "{timeout:d}" seconds')
+def wait(context, timeout):
     """Wait for child process to terminate."""
-    context.result.wait()
+    context.result.wait(timeout)
 
 
 @when("I execute the test jupyter notebook and save changes")

kedro/io/__init__.py

@@ -3,7 +3,9 @@
 """
 from __future__ import annotations
 
-from .cached_dataset import CachedDataSet, CachedDataset
+import warnings
+
+from .cached_dataset import CachedDataset
 from .core import (
     AbstractDataSet,
     AbstractVersionedDataSet,
@@ -13,14 +15,17 @@
     Version,
 )
 from .data_catalog import DataCatalog
-from .lambda_dataset import LambdaDataSet, LambdaDataset
-from .memory_dataset import MemoryDataSet, MemoryDataset
-from .partitioned_dataset import (
-    IncrementalDataSet,
-    IncrementalDataset,
-    PartitionedDataSet,
-    PartitionedDataset,
-)
+from .lambda_dataset import LambdaDataset
+from .memory_dataset import MemoryDataset
+from .partitioned_dataset import IncrementalDataset, PartitionedDataset
+
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", DeprecationWarning)
+    from .cached_dataset import CachedDataSet
+    from .lambda_dataset import LambdaDataSet
+    from .memory_dataset import MemoryDataSet
+    from .partitioned_dataset import IncrementalDataSet, PartitionedDataSet
+
 
 # https://github.com/pylint-dev/pylint/issues/4300#issuecomment-1043601901
 DataSetError: type[Exception]

kedro/io/cached_dataset.py

@@ -5,11 +5,14 @@
 from __future__ import annotations
 
 import logging
+import warnings
 from typing import Any
 
 from kedro.io.core import VERSIONED_FLAG_KEY, AbstractDataSet, Version
 from kedro.io.memory_dataset import MemoryDataset
-from kedro.utils import DeprecatedClassMeta
+
+# https://github.com/pylint-dev/pylint/issues/4300#issuecomment-1043601901
+CachedDataSet: AbstractDataSet
 
 
 class CachedDataset(AbstractDataSet):
@@ -120,7 +123,14 @@ def __getstate__(self):
         return self.__dict__
 
 
-class CachedDataSet(metaclass=DeprecatedClassMeta):
-    # pylint: disable=missing-class-docstring, too-few-public-methods
-
-    _DeprecatedClassMeta__alias = CachedDataset
+def __getattr__(name):
+    if name == "CachedDataSet":
+        alias = CachedDataset
+        warnings.warn(
+            f"{repr(name)} has been renamed to {repr(alias.__name__)}, "
+            f"and the alias will be removed in Kedro 0.19.0",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return alias
+    raise AttributeError(f"module {repr(__name__)} has no attribute {repr(name)}")

kedro/io/lambda_dataset.py

@@ -4,10 +4,13 @@
 """
 from __future__ import annotations
 
+import warnings
 from typing import Any, Callable
 
 from kedro.io.core import AbstractDataSet, DatasetError
-from kedro.utils import DeprecatedClassMeta
+
+# https://github.com/pylint-dev/pylint/issues/4300#issuecomment-1043601901
+LambdaDataSet: AbstractDataSet
 
 
 class LambdaDataset(AbstractDataSet):
@@ -121,7 +124,14 @@ def __init__(
         self.metadata = metadata
 
 
-class LambdaDataSet(metaclass=DeprecatedClassMeta):
-    # pylint: disable=missing-class-docstring, too-few-public-methods
-
-    _DeprecatedClassMeta__alias = LambdaDataset
+def __getattr__(name):
+    if name == "LambdaDataSet":
+        alias = LambdaDataset
+        warnings.warn(
+            f"{repr(name)} has been renamed to {repr(alias.__name__)}, "
+            f"and the alias will be removed in Kedro 0.19.0",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return alias
+    raise AttributeError(f"module {repr(__name__)} has no attribute {repr(name)}")

kedro/io/memory_dataset.py

@@ -3,13 +3,16 @@
 from __future__ import annotations
 
 import copy
+import warnings
 from typing import Any
 
 from kedro.io.core import AbstractDataSet, DatasetError
-from kedro.utils import DeprecatedClassMeta
 
 _EMPTY = object()
 
+# https://github.com/pylint-dev/pylint/issues/4300#issuecomment-1043601901
+MemoryDataSet: AbstractDataSet
+
 
 class MemoryDataset(AbstractDataSet):
     """``MemoryDataset`` loads and saves data from/to an in-memory
@@ -139,7 +142,14 @@ def _copy_with_mode(data: Any, copy_mode: str) -> Any:
     return copied_data
 
 
-class MemoryDataSet(metaclass=DeprecatedClassMeta):
-    # pylint: disable=missing-class-docstring, too-few-public-methods
-
-    _DeprecatedClassMeta__alias = MemoryDataset
+def __getattr__(name):
+    if name == "MemoryDataSet":
+        alias = MemoryDataset
+        warnings.warn(
+            f"{repr(name)} has been renamed to {repr(alias.__name__)}, "
+            f"and the alias will be removed in Kedro 0.19.0",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return alias
+    raise AttributeError(f"module {repr(__name__)} has no attribute {repr(name)}")

kedro/io/partitioned_dataset.py

@@ -4,10 +4,10 @@
 from __future__ import annotations
 
 import operator
+import warnings
 from copy import deepcopy
 from typing import Any, Callable
 from urllib.parse import urlparse
-from warnings import warn
 
 from cachetools import Cache, cachedmethod
 
@@ -19,7 +19,7 @@
     parse_dataset_definition,
 )
 from kedro.io.data_catalog import CREDENTIALS_KEY
-from kedro.utils import DeprecatedClassMeta, load_obj
+from kedro.utils import load_obj
 
 DATASET_CREDENTIALS_KEY = "dataset_credentials"
 CHECKPOINT_CREDENTIALS_KEY = "checkpoint_credentials"
@@ -31,6 +31,10 @@
 
 S3_PROTOCOLS = ("s3", "s3a", "s3n")
 
+# https://github.com/pylint-dev/pylint/issues/4300#issuecomment-1043601901
+PartitionedDataSet: AbstractDataSet
+IncrementalDataSet: AbstractDataSet
+
 
 class PartitionedDataset(AbstractDataSet):
     # pylint: disable=too-many-instance-attributes,protected-access
@@ -230,7 +234,7 @@ def __init__(  # pylint: disable=too-many-arguments
 
         self._filepath_arg = filepath_arg
         if self._filepath_arg in self._dataset_config:
-            warn(
+            warnings.warn(
                 f"'{self._filepath_arg}' key must not be specified in the dataset "
                 f"definition as it will be overwritten by partition path"
             )
@@ -338,12 +342,6 @@ def _release(self) -> None:
         self._invalidate_caches()
 
 
-class PartitionedDataSet(metaclass=DeprecatedClassMeta):
-    # pylint: disable=missing-class-docstring, too-few-public-methods
-
-    _DeprecatedClassMeta__alias = PartitionedDataset
-
-
 class IncrementalDataset(PartitionedDataset):
     """``IncrementalDataset`` inherits from ``PartitionedDataset``, which loads
     and saves partitioned file-like data using the underlying dataset
@@ -560,8 +558,20 @@ def confirm(self) -> None:
             self._checkpoint.save(partition_ids[-1])  # checkpoint to last partition
 
 
-class IncrementalDataSet(metaclass=DeprecatedClassMeta):
-    # pylint: disable=missing-class-docstring, too-few-public-methods
+_DEPRECATED_ERROR_CLASSES = {
+    "PartitionedDataSet": PartitionedDataset,
+    "IncrementalDataSet": IncrementalDataset,
+}
+
 
-    # pylint: disable=unused-private-member
-    __DeprecatedClassMeta__alias = IncrementalDataset
+def __getattr__(name):
+    if name in _DEPRECATED_ERROR_CLASSES:
+        alias = _DEPRECATED_ERROR_CLASSES[name]
+        warnings.warn(
+            f"{repr(name)} has been renamed to {repr(alias.__name__)}, "
+            f"and the alias will be removed in Kedro 0.19.0",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return alias
+    raise AttributeError(f"module {repr(__name__)} has no attribute {repr(name)}")

kedro/runner/parallel_runner.py

@@ -7,6 +7,7 @@
 import os
 import pickle
 import sys
+import warnings
 from collections import Counter
 from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, wait
 from itertools import chain
@@ -27,11 +28,13 @@
 from kedro.pipeline import Pipeline
 from kedro.pipeline.node import Node
 from kedro.runner.runner import AbstractRunner, run_node
-from kedro.utils import DeprecatedClassMeta
 
 # see https://github.com/python/cpython/blob/master/Lib/concurrent/futures/process.py#L114
 _MAX_WINDOWS_WORKERS = 61
 
+# https://github.com/pylint-dev/pylint/issues/4300#issuecomment-1043601901
+_SharedMemoryDataSet: Any
+
 
 class _SharedMemoryDataset:
     """``_SharedMemoryDataset`` is a wrapper class for a shared MemoryDataset in SyncManager.
@@ -70,10 +73,17 @@ def save(self, data: Any):
             raise exc
 
 
-class _SharedMemoryDataSet(metaclass=DeprecatedClassMeta):
-    # pylint: disable=too-few-public-methods
-
-    _DeprecatedClassMeta__alias = _SharedMemoryDataset
+def __getattr__(name):
+    if name == "_SharedMemoryDataSet":
+        alias = _SharedMemoryDataset
+        warnings.warn(
+            f"{repr(name)} has been renamed to {repr(alias.__name__)}, "
+            f"and the alias will be removed in Kedro 0.19.0",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return alias
+    raise AttributeError(f"module {repr(__name__)} has no attribute {repr(name)}")
 
 
 class ParallelRunnerManager(SyncManager):

kedro/utils.py

@@ -3,7 +3,6 @@
 """
 import importlib
 from typing import Any
-from warnings import warn
 
 
 def load_obj(obj_path: str, default_obj_path: str = "") -> Any:
@@ -27,67 +26,3 @@ def load_obj(obj_path: str, default_obj_path: str = "") -> Any:
     if not hasattr(module_obj, obj_name):
         raise AttributeError(f"Object '{obj_name}' cannot be loaded from '{obj_path}'.")
     return getattr(module_obj, obj_name)
-
-
-class DeprecatedClassMeta(type):
-    """Metaclass for constructing deprecated aliases of renamed classes.
-
-    Code implementation copied from https://stackoverflow.com/a/52087847
-    """
-
-    def __new__(mcs, name, bases, classdict, *args, **kwargs):
-        alias = classdict.get("_DeprecatedClassMeta__alias")
-
-        if alias is not None:
-
-            def new(mcs, *args, **kwargs):
-                alias = getattr(mcs, "_DeprecatedClassMeta__alias")
-
-                if alias is not None:
-                    warn(
-                        f"{repr(mcs.__name__)} has been renamed to "
-                        f"{repr(alias.__name__)}, and the alias will "
-                        f"be removed in Kedro 0.19.0",
-                        DeprecationWarning,
-                        stacklevel=2,
-                    )
-
-                return alias(*args, **kwargs)
-
-            classdict["__new__"] = new
-            classdict["_DeprecatedClassMeta__alias"] = alias
-
-        fixed_bases = []
-
-        for base in bases:
-            alias = getattr(base, "_DeprecatedClassMeta__alias", None)
-
-            if alias is not None:
-                warn(
-                    f"{repr(base.__name__)} has been renamed to "
-                    f"{repr(alias.__name__)}, and the alias will be "
-                    f"removed in Kedro 0.19.0",
-                    DeprecationWarning,
-                    stacklevel=2,
-                )
-
-            # Avoid duplicate base classes.
-            base = alias or base
-            if base not in fixed_bases:
-                fixed_bases.append(base)
-
-        fixed_bases = tuple(fixed_bases)
-
-        return super().__new__(mcs, name, fixed_bases, classdict, *args, **kwargs)
-
-    def __instancecheck__(cls, instance):
-        return any(
-            # pylint: disable=no-value-for-parameter
-            cls.__subclasscheck__(c)
-            for c in [type(instance), instance.__class__]
-        )
-
-    def __subclasscheck__(cls, subclass):
-        return subclass is cls or issubclass(
-            subclass, getattr(cls, "_DeprecatedClassMeta__alias")
-        )

pyproject.toml

@@ -2,6 +2,7 @@
 [build-system]
 # Minimum requirements for the build system to execute.
 requires = ["setuptools>=65.5.1"]  # PEP 518 specifications
+build-backend = "setuptools.build_meta"
 
 [project]
 name = "kedro"

tests/runner/test_parallel_runner.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import importlib
 import sys
 from concurrent.futures.process import ProcessPoolExecutor
 from typing import Any
@@ -33,6 +34,12 @@
 )
 
 
+def test_deprecation():
+    class_name = "_SharedMemoryDataSet"
+    with pytest.warns(DeprecationWarning, match=f"{repr(class_name)} has been renamed"):
+        getattr(importlib.import_module("kedro.runner.parallel_runner"), class_name)
+
+
 @pytest.mark.skipif(
     sys.platform.startswith("win"), reason="Due to bug in parallel runner"
 )