diff --git a/docs/release_notes/release_notes-0.16.0.rst b/docs/release_notes/release_notes-0.16.0.rst index dee5bd05f85..ce9c481baba 100644 --- a/docs/release_notes/release_notes-0.16.0.rst +++ b/docs/release_notes/release_notes-0.16.0.rst @@ -36,6 +36,7 @@ Key Features and Updates * REFACTOR-#4530: Standardize access to physical data in partitions (#4563) * REFACTOR-#4534: Replace logging meta class with class decorator (#4535) * REFACTOR-#4708: Delete combine dtypes (#4709) + * REFACTOR-#4629: Add type annotations to modin/config (#4685) * Pandas API implementations and improvements * FEAT-#4670: Implement convert_dtypes by mapping across partitions (#4671) * OmniSci enhancements diff --git a/modin/config/__main__.py b/modin/config/__main__.py index fa17599de4a..17bb07ebac0 100644 --- a/modin/config/__main__.py +++ b/modin/config/__main__.py @@ -27,7 +27,7 @@ import argparse -def print_config_help(): +def print_config_help() -> None: """Print configs help messages.""" for objname in sorted(globals()): obj = globals()[objname] @@ -35,7 +35,7 @@ def print_config_help(): print(f"{obj.get_help()}\n\tCurrent value: {obj.get()}") # noqa: T201 -def export_config_help(filename: str): +def export_config_help(filename: str) -> None: """ Export all configs help messages to the CSV file. @@ -57,7 +57,9 @@ def export_config_help(filename: str): if obj.__name__ != "RayRedisPassword" else "random string", # `Notes` `-` underlining can't be correctly parsed inside csv table by sphinx - "Description": dedent(obj.__doc__).replace("Notes\n-----", "Notes:\n"), + "Description": dedent(obj.__doc__ or "").replace( + "Notes\n-----", "Notes:\n" + ), "Options": obj.choices, } configs_data.append(data) diff --git a/modin/config/envvars.py b/modin/config/envvars.py index db6b86b3d8e..e1d39d699b1 100644 --- a/modin/config/envvars.py +++ b/modin/config/envvars.py @@ -21,12 +21,13 @@ import secrets from .pubsub import Parameter, _TYPE_PARAMS, ExactStr, ValueSource +from typing import Optional class EnvironmentVariable(Parameter, type=str, abstract=True): """Base class for environment variables-based configuration.""" - varname: str = None + varname: Optional[str] = None @classmethod def _get_raw_from_config(cls) -> str: @@ -40,9 +41,13 @@ def _get_raw_from_config(cls) -> str: Raises ------ + TypeError + If `varname` is None. KeyError If value is absent. """ + if cls.varname is None: + raise TypeError("varname should not be None") return os.environ[cls.varname] @classmethod @@ -73,7 +78,7 @@ class Engine(EnvironmentVariable, type=str): choices = ("Ray", "Dask", "Python", "Native") @classmethod - def _get_default(cls): + def _get_default(cls) -> str: """ Get default value of the config. @@ -170,7 +175,7 @@ class CpuCount(EnvironmentVariable, type=int): varname = "MODIN_CPUS" @classmethod - def _get_default(cls): + def _get_default(cls) -> int: """ Get default value of the config. @@ -208,7 +213,7 @@ class NPartitions(EnvironmentVariable, type=int): varname = "MODIN_NPARTITIONS" @classmethod - def _put(cls, value): + def _put(cls, value: int) -> None: """ Put specific value if NPartitions wasn't set by a user yet. @@ -226,7 +231,7 @@ def _put(cls, value): cls.put(value) @classmethod - def _get_default(cls): + def _get_default(cls) -> int: """ Get default value of the config. @@ -318,17 +323,17 @@ class ProgressBar(EnvironmentVariable, type=bool): default = False @classmethod - def enable(cls): + def enable(cls) -> None: """Enable ``ProgressBar`` feature.""" cls.put(True) @classmethod - def disable(cls): + def disable(cls) -> None: """Disable ``ProgressBar`` feature.""" cls.put(False) @classmethod - def put(cls, value): + def put(cls, value: bool) -> None: """ Set ``ProgressBar`` value only if synchronous benchmarking is disabled. @@ -349,7 +354,7 @@ class BenchmarkMode(EnvironmentVariable, type=bool): default = False @classmethod - def put(cls, value): + def put(cls, value: bool) -> None: """ Set ``BenchmarkMode`` value only if progress bar feature is disabled. @@ -371,17 +376,17 @@ class LogMode(EnvironmentVariable, type=ExactStr): default = "disable" @classmethod - def enable(cls): + def enable(cls) -> None: """Enable all logging levels.""" cls.put("enable") @classmethod - def disable(cls): + def disable(cls) -> None: """Disable logging feature.""" cls.put("disable") @classmethod - def enable_api_only(cls): + def enable_api_only(cls) -> None: """Enable API level logging.""" cls.put("enable_api_only") @@ -393,7 +398,7 @@ class LogMemoryInterval(EnvironmentVariable, type=int): default = 5 @classmethod - def put(cls, value): + def put(cls, value: int) -> None: """ Set ``LogMemoryInterval`` with extra checks. @@ -427,7 +432,7 @@ class LogFileSize(EnvironmentVariable, type=int): default = 10 @classmethod - def put(cls, value): + def put(cls, value: int) -> None: """ Set ``LogFileSize`` with extra checks. @@ -484,7 +489,7 @@ class OmnisciLaunchParameters(EnvironmentVariable, type=dict): } @classmethod - def get(self): + def get(self) -> dict: """ Get the resulted command-line options. @@ -515,7 +520,7 @@ class MinPartitionSize(EnvironmentVariable, type=int): default = 32 @classmethod - def put(cls, value): + def put(cls, value: int) -> None: """ Set ``MinPartitionSize`` with extra checks. @@ -529,7 +534,7 @@ def put(cls, value): super().put(value) @classmethod - def get(cls): + def get(cls) -> int: """ Get ``MinPartitionSize`` with extra checks. @@ -564,7 +569,7 @@ class ReadSqlEngine(EnvironmentVariable, type=str): choices = ("Pandas", "Connectorx") -def _check_vars(): +def _check_vars() -> None: """ Check validity of environment variables. diff --git a/modin/config/pubsub.py b/modin/config/pubsub.py index 3927ff56e0f..0262ae1a7f7 100644 --- a/modin/config/pubsub.py +++ b/modin/config/pubsub.py @@ -13,11 +13,12 @@ """Module houses ``Parameter`` class - base class for all configs.""" -import collections -import typing +from collections import defaultdict +from enum import IntEnum +from typing import Any, Callable, DefaultDict, NamedTuple, Optional, Sequence -class TypeDescriptor(typing.NamedTuple): +class TypeDescriptor(NamedTuple): """ Class for config data manipulating of exact type. @@ -35,9 +36,9 @@ class TypeDescriptor(typing.NamedTuple): Class description string. """ - decode: typing.Callable[[str], object] - normalize: typing.Callable[[object], object] - verify: typing.Callable[[object], bool] + decode: Callable[[str], object] + normalize: Callable[[object], object] + verify: Callable[[object], bool] help: str @@ -48,7 +49,7 @@ class ExactStr(str): _TYPE_PARAMS = { str: TypeDescriptor( decode=lambda value: value.strip().title(), - normalize=lambda value: value.strip().title(), + normalize=lambda value: str(value).strip().title(), verify=lambda value: True, help="a case-insensitive string", ), @@ -70,7 +71,7 @@ class ExactStr(str): ), int: TypeDescriptor( decode=lambda value: int(value.strip()), - normalize=int, + normalize=int, # type: ignore verify=lambda value: isinstance(value, int) or (isinstance(value, str) and value.strip().isdigit()), help="an integer value", @@ -85,7 +86,7 @@ class ExactStr(str): if isinstance(value, dict) else { key: int(val) if val.isdigit() else val - for key_value in value.split(",") + for key_value in str(value).split(",") for key, val in [[v.strip() for v in key_value.split("=", maxsplit=1)]] }, verify=lambda value: isinstance(value, dict) @@ -105,7 +106,7 @@ class ExactStr(str): _UNSET = object() -class ValueSource: +class ValueSource(IntEnum): # noqa: PR01 """Class that describes the method of getting the value for a parameter.""" # got from default, i.e. neither user nor configuration source had the value @@ -122,24 +123,27 @@ class Parameter(object): Attributes ---------- - choices : sequence of str + choices : Optional[Sequence[str]] Array with possible options of ``Parameter`` values. type : str String that denotes ``Parameter`` type. - default : Any + default : Optional[Any] ``Parameter`` default value. is_abstract : bool, default: True Whether or not ``Parameter`` is abstract. - _value_source : int + _value_source : Optional[ValueSource] Source of the ``Parameter`` value, should be set by ``ValueSource``. """ - choices: typing.Sequence[str] = None + choices: Optional[Sequence[str]] = None type = str - default = None + default: Optional[Any] = None is_abstract = True - _value_source = None + _value_source: Optional[ValueSource] = None + _value: Any = _UNSET + _subs: list = [] + _once: DefaultDict[Any, list] = defaultdict(list) @classmethod def _get_raw_from_config(cls) -> str: @@ -178,7 +182,7 @@ def get_help(cls) -> str: """ raise NotImplementedError() - def __init_subclass__(cls, type, abstract=False, **kw): + def __init_subclass__(cls, type: Any, abstract: bool = False, **kw: dict): """ Initialize subclass. @@ -196,11 +200,11 @@ def __init_subclass__(cls, type, abstract=False, **kw): cls.is_abstract = abstract cls._value = _UNSET cls._subs = [] - cls._once = collections.defaultdict(list) + cls._once = defaultdict(list) super().__init_subclass__(**kw) @classmethod - def subscribe(cls, callback): + def subscribe(cls, callback: Callable) -> None: """ Add `callback` to the `_subs` list and then execute it. @@ -213,7 +217,7 @@ def subscribe(cls, callback): callback(cls) @classmethod - def _get_default(cls): + def _get_default(cls) -> Any: """ Get default value of the config. @@ -224,21 +228,24 @@ def _get_default(cls): return cls.default @classmethod - def get_value_source(cls): + def get_value_source(cls) -> ValueSource: """ Get value source of the config. Returns ------- - int + ValueSource """ if cls._value_source is None: # dummy call to .get() to initialize the value cls.get() + assert ( + cls._value_source is not None + ), "_value_source must be initialized by now in get()" return cls._value_source @classmethod - def get(cls) -> typing.Any: + def get(cls) -> Any: """ Get config value. @@ -262,7 +269,7 @@ def get(cls) -> typing.Any: return cls._value @classmethod - def put(cls, value): + def put(cls, value: Any) -> None: """ Set config value. @@ -275,7 +282,7 @@ def put(cls, value): cls._value_source = ValueSource.SET_BY_USER @classmethod - def once(cls, onvalue, callback): + def once(cls, onvalue: Any, callback: Callable) -> None: """ Execute `callback` if config value matches `onvalue` value. @@ -296,7 +303,7 @@ def once(cls, onvalue, callback): cls._once[onvalue].append(callback) @classmethod - def _put_nocallback(cls, value): + def _put_nocallback(cls, value: Any) -> Any: """ Set config value without executing callbacks. @@ -317,7 +324,7 @@ def _put_nocallback(cls, value): return oldvalue @classmethod - def _check_callbacks(cls, oldvalue): + def _check_callbacks(cls, oldvalue: Any) -> None: """ Execute all needed callbacks if config value was changed. diff --git a/mypy.ini b/mypy.ini index a21f8e90f32..90ab08cbaab 100644 --- a/mypy.ini +++ b/mypy.ini @@ -18,4 +18,4 @@ warn_unreachable=True # We will add more files over time to increase coverage ; files = **/modin/*.py -files = **/modin/logging/*.py +files = **/modin/config/*.py, **/modin/logging/*.py