Merge pull request #747 from nolar/proper-memo-typing-2

Accept arbitrary types for memos & results, clarify other kwarg types

examples/12-embedded/example.py

@@ -2,19 +2,18 @@
 import contextlib
 import threading
 import time
-from typing import Union
 
 import kopf
 import pykube
 
 
 @kopf.on.create('kopfexamples')
-def create_fn(memo: Union[kopf.Memo, object], **kwargs):
+def create_fn(memo: kopf.Memo, **kwargs):
     print(memo.create_tpl.format(**kwargs))  # type: ignore
 
 
 @kopf.on.delete('kopfexamples')
-def delete_fn(memo: Union[kopf.Memo, object], **kwargs):
+def delete_fn(memo: kopf.Memo, **kwargs):
     print(memo.delete_tpl.format(**kwargs))  # type: ignore
 
 

kopf/__init__.py

@@ -88,6 +88,7 @@
     build_owner_reference,
 )
 from kopf.structs.callbacks import (
+    Logger,
     not_,
     all_,
     any_,
@@ -231,6 +232,7 @@
     'ObjectReference',
     'OwnerReference',
     'Memo', 'Index', 'Store',
+    'Logger',
     'ObjectLogger',
     'LocalObjectLogger',
     'FieldSpec',

kopf/reactor/running.py

@@ -37,7 +37,7 @@ def run(
         stop_flag: Optional[primitives.Flag] = None,
         ready_flag: Optional[primitives.Flag] = None,
         vault: Optional[credentials.Vault] = None,
-        memo: Optional[ephemera.AnyMemo] = None,
+        memo: Optional[object] = None,
         _command: Optional[Coroutine[None, None, None]] = None,
 ) -> None:
     """
@@ -91,7 +91,7 @@ async def operator(
         stop_flag: Optional[primitives.Flag] = None,
         ready_flag: Optional[primitives.Flag] = None,
         vault: Optional[credentials.Vault] = None,
-        memo: Optional[ephemera.AnyMemo] = None,
+        memo: Optional[object] = None,
         _command: Optional[Coroutine[None, None, None]] = None,
 ) -> None:
     """
@@ -146,7 +146,7 @@ async def spawn_tasks(
         stop_flag: Optional[primitives.Flag] = None,
         ready_flag: Optional[primitives.Flag] = None,
         vault: Optional[credentials.Vault] = None,
-        memo: Optional[ephemera.AnyMemo] = None,
+        memo: Optional[object] = None,
         _command: Optional[Coroutine[None, None, None]] = None,
 ) -> Collection[aiotasks.Task]:
     """
@@ -180,6 +180,7 @@ async def spawn_tasks(
     identity = identity if identity is not None else peering.detect_own_id(manual=False)
     vault = vault if vault is not None else credentials.Vault()
     memo = memo if memo is not None else ephemera.Memo()
+    memo = ephemera.AnyMemo(memo)  # type-casted
     event_queue: posting.K8sEventQueue = asyncio.Queue()
     signal_flag: aiotasks.Future = asyncio.Future()
     started_flag: asyncio.Event = asyncio.Event()

kopf/structs/callbacks.py

@@ -3,6 +3,10 @@
 
 Since these signatures contain a lot of copy-pasted kwargs and are
 not so important for the codebase, they are moved to this separate module.
+
+As a rule of thumb, for every kwarg named ``whatever``, there should be
+a corresponding type or class ``kopf.Whatever`` with all the typing tricks
+(``Union[...]``, ``Optional[...]``, partial ``Any`` values, etc) included.
 """
 import datetime
 import logging
@@ -12,50 +16,54 @@
 from kopf.structs import bodies, configuration, diffs, ephemera, \
                          patches, primitives, references, reviews
 
+# As publicly exposed: we only promise that it is based on one of the built-in loggable classes.
+# Mind that these classes have multi-versioned stubs, so we avoid redefining the protocol ourselves.
+Logger = Union[logging.Logger, logging.LoggerAdapter]
+
 # A specialised type to highlight the purpose or origin of the data of type Any,
 # to not be mixed with other arbitrary Any values, where it is indeed "any".
 Result = NewType('Result', object)
 
-# An internal typing hack to show that it can be a sync fn with the result,
-# or an async fn which returns a coroutine which returns the result.
-# Used in sync-and-async protocols only, and never exposed to other modules.
-_SyncOrAsyncResult = Union[Optional[Result], Coroutine[None, None, Optional[Result]]]
+# An internal typing hack shows that the handler can be sync fn with the result,
+# or an async fn which returns a coroutine which, in turn, returns the result.
+# Used in some protocols only and is never exposed to other modules.
+_R = TypeVar('_R')
+_SyncOrAsync = Union[_R, Coroutine[None, None, _R]]
 
 # A generic sync-or-async callable with no args/kwargs checks (unlike in protocols).
 # Used for the BaseHandler and generic invocation methods (which do not care about protocols).
-BaseFn = Callable[..., _SyncOrAsyncResult]
-
-LoggerType = Union[logging.Logger, logging.LoggerAdapter]
+BaseFn = Callable[..., _SyncOrAsync[Optional[object]]]
 
 if not TYPE_CHECKING:  # pragma: nocover
     # Define unspecified protocols for the runtime annotations -- to avoid "quoting".
-    ActivityFn = Callable[..., _SyncOrAsyncResult]
-    ResourceIndexingFn = Callable[..., _SyncOrAsyncResult]
-    ResourceWatchingFn = Callable[..., _SyncOrAsyncResult]
-    ResourceChangingFn = Callable[..., _SyncOrAsyncResult]
-    ResourceWebhookFn = Callable[..., None]
-    ResourceDaemonFn = Callable[..., _SyncOrAsyncResult]
-    ResourceTimerFn = Callable[..., _SyncOrAsyncResult]
-    WhenFilterFn = Callable[..., bool]
-    MetaFilterFn = Callable[..., bool]
+    ActivityFn = Callable[..., _SyncOrAsync[Optional[object]]]
+    ResourceIndexingFn = Callable[..., _SyncOrAsync[Optional[object]]]
+    ResourceWatchingFn = Callable[..., _SyncOrAsync[Optional[object]]]
+    ResourceChangingFn = Callable[..., _SyncOrAsync[Optional[object]]]
+    ResourceWebhookFn = Callable[..., _SyncOrAsync[None]]
+    ResourceDaemonFn = Callable[..., _SyncOrAsync[Optional[object]]]
+    ResourceTimerFn = Callable[..., _SyncOrAsync[Optional[object]]]
+    WhenFilterFn = Callable[..., bool]  # strictly sync, no async!
+    MetaFilterFn = Callable[..., bool]  # strictly sync, no async!
 else:
     from mypy_extensions import Arg, DefaultNamedArg, KwArg, NamedArg, VarArg
 
-    # TODO: Try using ParamSpec to support index type checking in callbacks 
-    # when PEP 612 is released (https://www.python.org/dev/peps/pep-0612/)
+    # TODO:1: Split to specialised LoginFn, ProbeFn, StartupFn, etc. -- with different result types.
+    # TODO:2: Try using ParamSpec to support index type checking in callbacks
+    #         when PEP 612 is released (https://www.python.org/dev/peps/pep-0612/)
     ActivityFn = Callable[
         [
             NamedArg(configuration.OperatorSettings, "settings"),
             NamedArg(ephemera.Index, "*"),
             NamedArg(int, "retry"),
             NamedArg(datetime.datetime, "started"),
             NamedArg(datetime.timedelta, "runtime"),
-            NamedArg(LoggerType, "logger"),
-            NamedArg(ephemera.AnyMemo, "memo"),
+            NamedArg(Logger, "logger"),
+            NamedArg(Any, "memo"),
             DefaultNamedArg(Any, "param"),
             KwArg(Any),
         ],
-        _SyncOrAsyncResult
+        _SyncOrAsync[Optional[object]]
     ]
 
     ResourceIndexingFn = Callable[
@@ -71,12 +79,12 @@
             NamedArg(Optional[str], "name"),
             NamedArg(Optional[str], "namespace"),
             NamedArg(patches.Patch, "patch"),
-            NamedArg(LoggerType, "logger"),
-            NamedArg(ephemera.AnyMemo, "memo"),
+            NamedArg(Logger, "logger"),
+            NamedArg(Any, "memo"),
             DefaultNamedArg(Any, "param"),
             KwArg(Any),
         ],
-        _SyncOrAsyncResult
+        _SyncOrAsync[Optional[object]]
     ]
 
     ResourceWatchingFn = Callable[
@@ -94,12 +102,12 @@
             NamedArg(Optional[str], "name"),
             NamedArg(Optional[str], "namespace"),
             NamedArg(patches.Patch, "patch"),
-            NamedArg(LoggerType, "logger"),
-            NamedArg(ephemera.AnyMemo, "memo"),
+            NamedArg(Logger, "logger"),
+            NamedArg(Any, "memo"),
             DefaultNamedArg(Any, "param"),
             KwArg(Any),
         ],
-        _SyncOrAsyncResult
+        _SyncOrAsync[Optional[object]]
     ]
 
     ResourceChangingFn = Callable[
@@ -122,12 +130,12 @@
             NamedArg(diffs.Diff, "diff"),
             NamedArg(Optional[Union[bodies.BodyEssence, Any]], "old"),
             NamedArg(Optional[Union[bodies.BodyEssence, Any]], "new"),
-            NamedArg(LoggerType, "logger"),
-            NamedArg(ephemera.AnyMemo, "memo"),
+            NamedArg(Logger, "logger"),
+            NamedArg(Any, "memo"),
             DefaultNamedArg(Any, "param"),
             KwArg(Any),
         ],
-        _SyncOrAsyncResult
+        _SyncOrAsync[Optional[object]]
     ]
 
     ResourceWebhookFn = Callable[
@@ -148,12 +156,12 @@
             NamedArg(Optional[str], "name"),
             NamedArg(Optional[str], "namespace"),
             NamedArg(patches.Patch, "patch"),
-            NamedArg(LoggerType, "logger"),
-            NamedArg(ephemera.AnyMemo, "memo"),
+            NamedArg(Logger, "logger"),
+            NamedArg(Any, "memo"),
             DefaultNamedArg(Any, "param"),
             KwArg(Any),
         ],
-        None
+        _SyncOrAsync[None]
     ]
 
     ResourceDaemonFn = Callable[
@@ -173,12 +181,12 @@
             NamedArg(Optional[str], "name"),
             NamedArg(Optional[str], "namespace"),
             NamedArg(patches.Patch, "patch"),
-            NamedArg(LoggerType, "logger"),
-            NamedArg(ephemera.AnyMemo, "memo"),
+            NamedArg(Logger, "logger"),
+            NamedArg(Any, "memo"),
             DefaultNamedArg(Any, "param"),
             KwArg(Any),
         ],
-        _SyncOrAsyncResult
+        _SyncOrAsync[Optional[object]]
     ]
 
     ResourceTimerFn = Callable[
@@ -195,12 +203,12 @@
             NamedArg(Optional[str], "name"),
             NamedArg(Optional[str], "namespace"),
             NamedArg(patches.Patch, "patch"),
-            NamedArg(LoggerType, "logger"),
-            NamedArg(ephemera.AnyMemo, "memo"),
+            NamedArg(Logger, "logger"),
+            NamedArg(Any, "memo"),
             DefaultNamedArg(Any, "param"),
             KwArg(Any),
         ],
-        _SyncOrAsyncResult
+        _SyncOrAsync[Optional[object]]
     ]
 
     WhenFilterFn = Callable[
@@ -221,12 +229,12 @@
             NamedArg(diffs.Diff, "diff"),
             NamedArg(Optional[Union[bodies.BodyEssence, Any]], "old"),
             NamedArg(Optional[Union[bodies.BodyEssence, Any]], "new"),
-            NamedArg(LoggerType, "logger"),
-            NamedArg(ephemera.AnyMemo, "memo"),
+            NamedArg(Logger, "logger"),
+            NamedArg(Any, "memo"),
             DefaultNamedArg(Any, "param"),
             KwArg(Any),
         ],
-        bool
+        bool  # strictly sync, no async!
     ]
 
     MetaFilterFn = Callable[
@@ -244,12 +252,12 @@
             NamedArg(Optional[str], "name"),
             NamedArg(Optional[str], "namespace"),
             NamedArg(patches.Patch, "patch"),
-            NamedArg(LoggerType, "logger"),
-            NamedArg(ephemera.AnyMemo, "memo"),
+            NamedArg(Logger, "logger"),
+            NamedArg(Any, "memo"),
             DefaultNamedArg(Any, "param"),
             KwArg(Any),
         ],
-        bool
+        bool  # strictly sync, no async!
     ]
 
 ResourceSpawningFn = Union[ResourceDaemonFn, ResourceTimerFn]

kopf/structs/containers.py

@@ -39,7 +39,8 @@ class ResourceMemory:
     """ A system memo about a single resource/object. Usually stored in `Memories`. """
 
     # For arbitrary user data to be stored in memory, passed as `memo` to all the handlers.
-    memo: ephemera.AnyMemo = dataclasses.field(default_factory=ephemera.Memo)
+    memo: ephemera.AnyMemo = dataclasses.field(default_factory=
+                                               lambda: ephemera.AnyMemo(ephemera.Memo))
 
     # For resuming handlers tracking and deciding on should they be called or not.
     noticed_by_listing: bool = False

kopf/structs/ephemera.py

@@ -1,9 +1,9 @@
-from typing import Any, Collection, Dict, Generic, Mapping, TypeVar, Union
+from typing import Any, Collection, Dict, Generic, Mapping, NewType, TypeVar
 
-# Used for type-checking of embedded operators, where it can have any type.
-# It is usually of type `Memo` -- but the framework must not rely on that.
-# `Memo`, despite inheritance from `object`, is added to enable IDE completions.
-AnyMemo = Union["Memo", object]
+# For users, memos are exposed as `Any`, though usually used with `kopf.Memo`.
+# However, the framework cannot rely on any methods/properties of it, so it is
+# declared as a nearly empty `object` for stricter internal type-checking.
+AnyMemo = NewType('AnyMemo', object)
 
 
 class Memo(Dict[Any, Any]):