diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index ec403077..94183bb0 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -50,7 +50,7 @@ jobs: - name: Unit tests (local) if: matrix.backend == 'local' - run: pytest -m "not mongo" + run: pytest -m "not mongo" --cov=cachier --cov-report=term --cov-report=xml:cov.xml - name: Setup docker (missing on MacOS) if: runner.os == 'macOS' && matrix.backend == 'db' @@ -77,7 +77,7 @@ jobs: docker ps -a - name: Unit tests (DB) if: matrix.backend == 'db' - run: pytest -m "mongo" + run: pytest -m "mongo" --cov=cachier --cov-report=term --cov-report=xml:cov.xml - name: Speed eval run: python tests/speed_eval.py diff --git a/pyproject.toml b/pyproject.toml index 8347a40c..f653dec8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,9 +145,6 @@ norecursedirs = [ ] addopts = [ "--color=yes", - "--cov=cachier", - "--cov-report=term", - "--cov-report=xml:cov.xml", "-r a", "-v", "-s", diff --git a/src/cachier/config.py b/src/cachier/config.py index a85373bd..54d45d14 100644 --- a/src/cachier/config.py +++ b/src/cachier/config.py @@ -2,9 +2,10 @@ import hashlib import os import pickle +import threading from collections.abc import Mapping from dataclasses import dataclass, replace -from typing import Optional, Union +from typing import Any, Optional, Union from ._types import Backend, HashFunc, Mongetter @@ -38,6 +39,17 @@ class Params: _global_params = Params() +@dataclass +class CacheEntry: + """Data class for cache entries.""" + + value: Any + time: datetime + stale: bool + being_calculated: bool + condition: Optional[threading.Condition] = None + + def _update_with_defaults( param, name: str, func_kwargs: Optional[dict] = None ): diff --git a/src/cachier/core.py b/src/cachier/core.py index 8a5791c0..b86c16e9 100644 --- a/src/cachier/core.py +++ b/src/cachier/core.py @@ -258,17 +258,17 @@ def func_wrapper(*args, **kwds): _print("No entry found. No current calc. Calling like a boss.") return _calc_entry(core, key, func, args, kwds) _print("Entry found.") - if _allow_none or entry.get("value", None) is not None: + if _allow_none or entry.value is not None: _print("Cached result found.") now = datetime.datetime.now() - if now - entry["time"] <= _stale_after: + if now - entry.time <= _stale_after: _print("And it is fresh!") - return entry["value"] + return entry.value _print("But it is stale... :(") - if entry["being_calculated"]: + if entry.being_calculated: if _next_time: _print("Returning stale.") - return entry["value"] # return stale val + return entry.value # return stale val _print("Already calc. Waiting on change.") try: return core.wait_on_entry_calc(key) @@ -283,10 +283,10 @@ def func_wrapper(*args, **kwds): ) finally: core.mark_entry_not_calculated(key) - return entry["value"] + return entry.value _print("Calling decorated function and waiting") return _calc_entry(core, key, func, args, kwds) - if entry["being_calculated"]: + if entry.being_calculated: _print("No value but being calculated. Waiting.") try: return core.wait_on_entry_calc(key) diff --git a/src/cachier/cores/base.py b/src/cachier/cores/base.py index e705d45b..c824ebdc 100644 --- a/src/cachier/cores/base.py +++ b/src/cachier/cores/base.py @@ -9,10 +9,10 @@ import abc # for the _BaseCore abstract base class import inspect import threading -from typing import Callable +from typing import Callable, Optional, Tuple from .._types import HashFunc -from ..config import _update_with_defaults +from ..config import CacheEntry, _update_with_defaults class RecalculationNeeded(Exception): @@ -51,7 +51,7 @@ def get_key(self, args, kwds): """Return a unique key based on the arguments provided.""" return self.hash_func(args, kwds) - def get_entry(self, args, kwds): + def get_entry(self, args, kwds) -> Tuple[str, Optional[CacheEntry]]: """Get entry based on given arguments. Return the result mapped to the given arguments in this core's cache, @@ -76,7 +76,7 @@ def check_calc_timeout(self, time_spent): raise RecalculationNeeded() @abc.abstractmethod - def get_entry_by_key(self, key): + def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: """Get entry based on given key. Return the result mapped to the given key in this core's cache, if such @@ -85,25 +85,25 @@ def get_entry_by_key(self, key): """ @abc.abstractmethod - def set_entry(self, key, func_res): + def set_entry(self, key: str, func_res): """Map the given result to the given key in this core's cache.""" @abc.abstractmethod - def mark_entry_being_calculated(self, key): + def mark_entry_being_calculated(self, key: str) -> None: """Mark the entry mapped by the given key as being calculated.""" @abc.abstractmethod - def mark_entry_not_calculated(self, key): + def mark_entry_not_calculated(self, key: str) -> None: """Mark the entry mapped by the given key as not being calculated.""" @abc.abstractmethod - def wait_on_entry_calc(self, key): + def wait_on_entry_calc(self, key: str) -> None: """Wait on the entry with keys being calculated and returns result.""" @abc.abstractmethod - def clear_cache(self): + def clear_cache(self) -> None: """Clear the cache of this core.""" @abc.abstractmethod - def clear_being_calculated(self): + def clear_being_calculated(self) -> None: """Mark all entries in this cache as not being calculated.""" diff --git a/src/cachier/cores/memory.py b/src/cachier/cores/memory.py index f7fe83dc..2a84666d 100644 --- a/src/cachier/cores/memory.py +++ b/src/cachier/cores/memory.py @@ -2,8 +2,10 @@ import threading from datetime import datetime +from typing import Any, Optional, Tuple from .._types import HashFunc +from ..config import CacheEntry from .base import _BaseCore, _get_func_str @@ -14,76 +16,78 @@ def __init__(self, hash_func: HashFunc, wait_for_calc_timeout: int): super().__init__(hash_func, wait_for_calc_timeout) self.cache = {} - def _hash_func_key(self, key): + def _hash_func_key(self, key: str) -> str: return f"{_get_func_str(self.func)}:{key}" - def get_entry_by_key(self, key, reload=False): + def get_entry_by_key( + self, key: str, reload=False + ) -> Tuple[str, Optional[CacheEntry]]: with self.lock: return key, self.cache.get(self._hash_func_key(key), None) - def set_entry(self, key, func_res): + def set_entry(self, key: str, func_res: Any) -> None: with self.lock: try: # we need to retain the existing condition so that # mark_entry_not_calculated can notify all possibly-waiting # threads about it - cond = self.cache[self._hash_func_key(key)]["condition"] + cond = self.cache[self._hash_func_key(key)].condition except KeyError: # pragma: no cover cond = None - self.cache[self._hash_func_key(key)] = { - "value": func_res, - "time": datetime.now(), - "stale": False, - "being_calculated": False, - "condition": cond, - } + self.cache[self._hash_func_key(key)] = CacheEntry( + value=func_res, + time=datetime.now(), + stale=False, + being_calculated=False, + condition=cond, + ) - def mark_entry_being_calculated(self, key): + def mark_entry_being_calculated(self, key: str) -> None: with self.lock: condition = threading.Condition() # condition.acquire() try: - self.cache[self._hash_func_key(key)]["being_calculated"] = True - self.cache[self._hash_func_key(key)]["condition"] = condition + self.cache[self._hash_func_key(key)].being_calculated = True + self.cache[self._hash_func_key(key)].condition = condition except KeyError: - self.cache[self._hash_func_key(key)] = { - "value": None, - "time": datetime.now(), - "stale": False, - "being_calculated": True, - "condition": condition, - } + self.cache[self._hash_func_key(key)] = CacheEntry( + value=None, + time=datetime.now(), + stale=False, + being_calculated=True, + condition=condition, + ) - def mark_entry_not_calculated(self, key): + def mark_entry_not_calculated(self, key: str) -> None: with self.lock: try: entry = self.cache[self._hash_func_key(key)] except KeyError: # pragma: no cover return # that's ok, we don't need an entry in that case - entry["being_calculated"] = False - cond = entry["condition"] + entry.being_calculated = False + cond = entry.condition if cond: cond.acquire() cond.notify_all() cond.release() - entry["condition"] = None + entry.condition = None - def wait_on_entry_calc(self, key): + def wait_on_entry_calc(self, key: str) -> Any: with self.lock: # pragma: no cover entry = self.cache[self._hash_func_key(key)] - if not entry["being_calculated"]: - return entry["value"] - entry["condition"].acquire() - entry["condition"].wait() - entry["condition"].release() - return self.cache[self._hash_func_key(key)]["value"] + if not entry.being_calculated: + return entry.value + entry.condition.acquire() + entry.condition.wait() + entry.condition.release() + return self.cache[self._hash_func_key(key)].value - def clear_cache(self): + def clear_cache(self) -> None: with self.lock: self.cache.clear() - def clear_being_calculated(self): + def clear_being_calculated(self) -> None: with self.lock: for entry in self.cache.values(): - entry["being_calculated"] = False - entry["condition"] = None + entry.being_calculated = False + entry.condition = None diff --git a/src/cachier/cores/mongo.py b/src/cachier/cores/mongo.py index b199845d..d0a7041e 100644 --- a/src/cachier/cores/mongo.py +++ b/src/cachier/cores/mongo.py @@ -13,8 +13,10 @@ import warnings # to warn if pymongo is missing from contextlib import suppress from datetime import datetime +from typing import Any, Optional, Tuple from .._types import HashFunc, Mongetter +from ..config import CacheEntry with suppress(ImportError): from bson.binary import Binary # to save binary data to mongodb @@ -65,29 +67,29 @@ def __init__( def _func_str(self) -> str: return _get_func_str(self.func) - def get_entry_by_key(self, key): + def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: res = self.mongo_collection.find_one( {"func": self._func_str, "key": key} ) if not res: return key, None try: - entry = { - "value": pickle.loads(res["value"]), # noqa: S301 - "time": res.get("time", None), - "stale": res.get("stale", False), - "being_calculated": res.get("being_calculated", False), - } + entry = CacheEntry( + value=pickle.loads(res["value"]), # noqa: S301 + time=res.get("time", None), + stale=res.get("stale", False), + being_calculated=res.get("being_calculated", False), + ) except KeyError: - entry = { - "value": None, - "time": res.get("time", None), - "stale": res.get("stale", False), - "being_calculated": res.get("being_calculated", False), - } + entry = CacheEntry( + value=None, + time=res.get("time", None), + stale=res.get("stale", False), + being_calculated=res.get("being_calculated", False), + ) return key, entry - def set_entry(self, key, func_res): + def set_entry(self, key: str, func_res: Any) -> None: thebytes = pickle.dumps(func_res) self.mongo_collection.update_one( filter={"func": self._func_str, "key": key}, @@ -104,14 +106,14 @@ def set_entry(self, key, func_res): upsert=True, ) - def mark_entry_being_calculated(self, key): + def mark_entry_being_calculated(self, key: str) -> None: self.mongo_collection.update_one( filter={"func": self._func_str, "key": key}, update={"$set": {"being_calculated": True}}, upsert=True, ) - def mark_entry_not_calculated(self, key): + def mark_entry_not_calculated(self, key: str) -> None: with suppress(OperationFailure): # don't care in this case self.mongo_collection.update_one( filter={ @@ -122,7 +124,7 @@ def mark_entry_not_calculated(self, key): upsert=False, # should not insert in this case ) - def wait_on_entry_calc(self, key): + def wait_on_entry_calc(self, key: str) -> Any: time_spent = 0 while True: time.sleep(MONGO_SLEEP_DURATION_IN_SEC) @@ -130,14 +132,14 @@ def wait_on_entry_calc(self, key): key, entry = self.get_entry_by_key(key) if entry is None: raise RecalculationNeeded() - if not entry["being_calculated"]: - return entry["value"] + if not entry.being_calculated: + return entry.value self.check_calc_timeout(time_spent) - def clear_cache(self): + def clear_cache(self) -> None: self.mongo_collection.delete_many(filter={"func": self._func_str}) - def clear_being_calculated(self): + def clear_being_calculated(self) -> None: self.mongo_collection.update_many( filter={ "func": self._func_str, diff --git a/src/cachier/cores/pickle.py b/src/cachier/cores/pickle.py index cfea2b94..196ea6fe 100644 --- a/src/cachier/cores/pickle.py +++ b/src/cachier/cores/pickle.py @@ -8,15 +8,15 @@ # Copyright (c) 2016, Shay Palachy import os import pickle # for local caching -from contextlib import suppress from datetime import datetime +from typing import Any, Dict, Optional, Tuple import portalocker # to lock on pickle cache IO from watchdog.events import PatternMatchingEventHandler from watchdog.observers import Observer from .._types import HashFunc -from ..config import _update_with_defaults +from ..config import CacheEntry, _update_with_defaults # Alternative: https://github.com/WoLpH/portalocker from .base import _BaseCore @@ -41,19 +41,19 @@ def __init__(self, filename, core, key): self.observer = None self.value = None - def inject_observer(self, observer): + def inject_observer(self, observer) -> None: """Inject the observer running this handler.""" self.observer = observer - def _check_calculation(self): + def _check_calculation(self) -> None: # print('checking calc') entry = self.core.get_entry_by_key(self.key, True)[1] # print(self.key) # print(entry) try: - if not entry["being_calculated"]: + if not entry.being_calculated: # print('stopping observer!') - self.value = entry["value"] + self.value = entry.value self.observer.stop() # else: # print('NOT stopping observer... :(') @@ -61,11 +61,11 @@ def _check_calculation(self): self.value = None self.observer.stop() - def on_created(self, event): + def on_created(self, event) -> None: """A Watchdog Event Handler method.""" # noqa: D401 self._check_calculation() # pragma: no cover - def on_modified(self, event): + def on_modified(self, event) -> None: """A Watchdog Event Handler method.""" # noqa: D401 self._check_calculation() @@ -99,23 +99,23 @@ def cache_fpath(self) -> str: os.path.join(os.path.realpath(self.cache_dir), self.cache_fname) ) - def _reload_cache(self): + def _reload_cache(self) -> None: with self.lock: try: - with portalocker.Lock( - self.cache_fpath, mode="rb" - ) as cache_file: - self.cache = pickle.load(cache_file) # noqa: S301 + with portalocker.Lock(self.cache_fpath, mode="rb") as cf: + self.cache = pickle.load(cf) # noqa: S301 except (FileNotFoundError, EOFError): self.cache = {} - def _get_cache(self): + def _get_cache(self) -> Dict[str, CacheEntry]: with self.lock: if not self.cache: self._reload_cache() return self.cache - def _get_cache_by_key(self, key=None, hash_str=None): + def _get_cache_by_key( + self, key=None, hash_str=None + ) -> Optional[Dict[str, CacheEntry]]: fpath = self.cache_fpath fpath += f"_{hash_str or key}" try: @@ -124,22 +124,24 @@ def _get_cache_by_key(self, key=None, hash_str=None): except (FileNotFoundError, EOFError): return None - def _clear_all_cache_files(self): + def _clear_all_cache_files(self) -> None: path, name = os.path.split(self.cache_fpath) for subpath in os.listdir(path): if subpath.startswith(f"{name}_"): os.remove(os.path.join(path, subpath)) - def _clear_being_calculated_all_cache_files(self): + def _clear_being_calculated_all_cache_files(self) -> None: path, name = os.path.split(self.cache_fpath) for subpath in os.listdir(path): if subpath.startswith(name): entry = self._get_cache_by_key(hash_str=subpath.split("_")[-1]) if entry is not None: - entry["being_calculated"] = False + entry.being_calculated = False self._save_cache(entry, hash_str=subpath.split("_")[-1]) - def _save_cache(self, cache, key=None, hash_str=None): + def _save_cache( + self, cache, key: str = None, hash_str: str = None + ) -> None: fpath = self.cache_fpath if key is not None: fpath += f"_{key}" @@ -152,7 +154,9 @@ def _save_cache(self, cache, key=None, hash_str=None): if key is None: self._reload_cache() - def get_entry_by_key(self, key, reload=False): + def get_entry_by_key( + self, key: str, reload: bool = False + ) -> Tuple[str, CacheEntry]: with self.lock: if self.separate_files: return key, self._get_cache_by_key(key) @@ -160,13 +164,13 @@ def get_entry_by_key(self, key, reload=False): self._reload_cache() return key, self._get_cache().get(key, None) - def set_entry(self, key, func_res): - key_data = { - "value": func_res, - "time": datetime.now(), - "stale": False, - "being_calculated": False, - } + def set_entry(self, key: str, func_res: Any) -> None: + key_data = CacheEntry( + value=func_res, + time=datetime.now(), + stale=False, + being_calculated=False, + ) if self.separate_files: self._save_cache(key_data, key) return # pragma: no cover @@ -176,23 +180,23 @@ def set_entry(self, key, func_res): cache[key] = key_data self._save_cache(cache) - def mark_entry_being_calculated_separate_files(self, key): + def mark_entry_being_calculated_separate_files(self, key: str) -> None: self._save_cache( - { - "value": None, - "time": datetime.now(), - "stale": False, - "being_calculated": True, - }, + CacheEntry( + value=None, + time=datetime.now(), + stale=False, + being_calculated=True, + ), key=key, ) - def mark_entry_not_calculated_separate_files(self, key): + def mark_entry_not_calculated_separate_files(self, key: str) -> None: _, entry = self.get_entry_by_key(key) - entry["being_calculated"] = False + entry.being_calculated = False self._save_cache(entry, key=key) - def mark_entry_being_calculated(self, key): + def mark_entry_being_calculated(self, key: str) -> None: if self.separate_files: self.mark_entry_being_calculated_separate_files(key) return # pragma: no cover @@ -200,27 +204,27 @@ def mark_entry_being_calculated(self, key): with self.lock: cache = self._get_cache() try: - cache[key]["being_calculated"] = True + cache[key].being_calculated = True except KeyError: - cache[key] = { - "value": None, - "time": datetime.now(), - "stale": False, - "being_calculated": True, - } + cache[key] = CacheEntry( + value=None, + time=datetime.now(), + stale=False, + being_calculated=True, + ) self._save_cache(cache) - def mark_entry_not_calculated(self, key): + def mark_entry_not_calculated(self, key: str) -> None: if self.separate_files: self.mark_entry_not_calculated_separate_files(key) with self.lock: cache = self._get_cache() # that's ok, we don't need an entry in that case - with suppress(KeyError): - cache[key]["being_calculated"] = False + if isinstance(cache, dict) and key in cache: + cache[key].being_calculated = False self._save_cache(cache) - def wait_on_entry_calc(self, key): + def wait_on_entry_calc(self, key: str) -> Any: if self.separate_files: entry = self._get_cache_by_key(key) filename = f"{self.cache_fname}_{key}" @@ -229,8 +233,8 @@ def wait_on_entry_calc(self, key): self._reload_cache() entry = self._get_cache()[key] filename = self.cache_fname - if not entry["being_calculated"]: - return entry["value"] + if not entry.being_calculated: + return entry.value event_handler = _PickleCore.CacheChangeHandler( filename=filename, core=self, key=key ) @@ -245,13 +249,13 @@ def wait_on_entry_calc(self, key): self.check_calc_timeout(time_spent) return event_handler.value - def clear_cache(self): + def clear_cache(self) -> None: if self.separate_files: self._clear_all_cache_files() else: self._save_cache({}) - def clear_being_calculated(self): + def clear_being_calculated(self) -> None: if self.separate_files: self._clear_being_calculated_all_cache_files() return # pragma: no cover @@ -259,5 +263,5 @@ def clear_being_calculated(self): with self.lock: cache = self._get_cache() for key in cache: - cache[key]["being_calculated"] = False + cache[key].being_calculated = False self._save_cache(cache) diff --git a/tests/test_mongo_core.py b/tests/test_mongo_core.py index 26870571..21a0622f 100644 --- a/tests/test_mongo_core.py +++ b/tests/test_mongo_core.py @@ -19,6 +19,7 @@ from pymongo_inmemory import MongoClient as InMemoryMongoClient from cachier import cachier +from cachier.config import CacheEntry from cachier.cores.base import RecalculationNeeded from cachier.cores.mongo import _MongoCore @@ -273,7 +274,9 @@ def _stalled_func(): @pytest.mark.mongo def test_stalled_mong_db_core(monkeypatch): def mock_get_entry(self, args, kwargs): - return "key", {"being_calculated": True} + return "key", CacheEntry( + being_calculated=True, value=None, time=None, stale=None + ) def mock_get_entry_by_key(self, key): return "key", None @@ -294,12 +297,12 @@ def _stalled_func(): assert res == 1 def mock_get_entry_2(self, args, kwargs): - entry = { - "being_calculated": True, - "value": 1, - "time": datetime.datetime.now() - datetime.timedelta(seconds=10), - } - return "key", entry + return "key", CacheEntry( + value=1, + time=datetime.datetime.now() - datetime.timedelta(seconds=10), + being_calculated=True, + stale=None, + ) monkeypatch.setattr( "cachier.cores.mongo._MongoCore.get_entry", mock_get_entry_2