feat: redis-separator (#278)

* feat: redis-separator

#192
#277
This adds the ability to customize the redis separator and changes the
default separator from _%&_ to :.

Additionally, it adds a way for a user to add a prefix to the key space
we use, to further differentiate their keys.

Finally, it fixes a bug with testing where redislite would not shutdown
properly and updates the fixtures to work with the newest version of
pytest_asyncio.

BREAKING CHANGE: This will result in "data loss" for existing models
stored in redis due to the change in default separator. To maintain
backwards compatbility with 0.7.0 and below, you will need to modify
your existing models to set _redis_separator = "_%&_" as a field on
them.

* ci: update tests to test new features

* ci: update noxfile and requirements for testing

* fix: fix for py3.7 and 3.8

* fix: main branch poetry.lock and constraints

* fix: fix import order

.github/workflows/constraints.txt

@@ -1,6 +1,8 @@
+
 pip==22.2.2
 nox==2022.8.7
 nox-poetry==1.0.1
 poetry==1.2.1
 virtualenv==20.16.5
 poetry-dynamic-versioning==0.19.0
+toml==0.10.2

.github/workflows/tests.yml

@@ -55,15 +55,10 @@ jobs:
           with open(os.environ["GITHUB_ENV"], mode="a") as io:
               print(f"VIRTUALENV_PIP={pip.__version__}", file=io)
 
-      - name: Install Poetry
+      - name: Install Test Requirements
         run: |
-          pipx install --pip-args=--constraint=.github/workflows/constraints.txt poetry
+          pip install -r .github/workflows/constraints.txt
           poetry --version
-
-      - name: Install Nox
-        run: |
-          pipx install --pip-args=--constraint=.github/workflows/constraints.txt nox
-          pipx inject --pip-args=--constraint=.github/workflows/constraints.txt nox nox-poetry
           nox --version
 
       - name: Compute pre-commit cache key

README.md

@@ -113,6 +113,15 @@ loop = asyncio.get_event_loop()
 loop.run_until_complete(work_with_orm())
 ```
 
+#### Custom Fields in Model
+
+| Field Name          | Required | Default      | Description                                                          |
+| ------------------- | -------- | ------------ | -------------------------------------------------------------------- |
+| \_primary_key_field | Yes      | None         | The field of your model that is the primary key                      |
+| \_redis_prefix      | No       | None         | If set, will be added to the beginning of the keys we store in redis |
+| \_redis_separator   | No       | :            | Defaults to :, used to separate prefix, table_name, and primary_key  |
+| \_table_name        | NO       | cls.**name** | Defaults to the model's name, can set a custom name in redis         |
+
 ## Contributing
 
 Contributions are very welcome.

noxfile.py

@@ -7,6 +7,7 @@
 from textwrap import dedent
 
 import nox
+import toml
 
 try:
     from nox_poetry import Session
@@ -35,24 +36,8 @@
     "docs-build",
 )
 mypy_type_packages = ()
-test_requirements = (
-    "coverage[toml]",
-    "pytest",
-    "pygments",
-    "fastapi>=0.6.3",
-    "fastapi-crudrouter>=0.8.4",
-    "httpx",
-    "pytest-asyncio",
-    "pytest-cov",
-    "pytest-env",
-    "pytest-lazy-fixture",
-    "pytest-mock",
-    "pytest-mockservers",
-    "pytest-xdist",
-    "redislite",
-    "pytest-asyncio",
-    "pytest-lazy-fixture",
-)
+pyproject = toml.load("pyproject.toml")
+test_requirements = pyproject["tool"]["poetry"]["dev-dependencies"].keys()
 
 
 def activate_virtualenv_in_precommit_hooks(session: Session) -> None:
@@ -128,20 +113,7 @@ def activate_virtualenv_in_precommit_hooks(session: Session) -> None:
 def precommit(session: Session) -> None:
     """Lint using pre-commit."""
     args = session.posargs or ["run", "--all-files", "--show-diff-on-failure"]
-    session.install(
-        "black",
-        "darglint",
-        "flake8",
-        "flake8-bandit",
-        "flake8-bugbear",
-        "flake8-docstrings",
-        "flake8-rst-docstrings",
-        "pep8-naming",
-        "pre-commit",
-        "pre-commit-hooks",
-        "pyupgrade",
-        "reorder-python-imports",
-    )
+    session.install(*test_requirements)
     session.install(".")
     session.run("pre-commit", *args)
     if args and args[0] == "install":
@@ -170,6 +142,7 @@ def mypy(session: Session) -> None:
 @session(python=python_versions[0])
 def bandit(session: Session) -> None:
     """Run bandit security tests"""
+    session.install("bandit")
     args = session.posargs or ["-r", "./pydantic_aioredis"]
     session.run("bandit", *args)
 

pydantic_aioredis/model.py

@@ -1,4 +1,5 @@
 """Module containing the model classes"""
+from functools import lru_cache
 from typing import Any
 from typing import Dict
 from typing import List
@@ -13,25 +14,56 @@
 class Model(_AbstractModel):
     """
     The section in the store that saves rows of the same kind
+
+    Model has some custom fields you can set in your models that alter the behavior of how this is stored in redis
+
+    _primary_key_field -- The field of your model that is the primary key
+    _redis_prefix -- If set, will be added to the beginning of the keys we store in redis
+    _redis_separator -- Defaults to :, used to separate prefix, table_name, and primary_key
+    _table_name -- Defaults to the model's name, can set a custom name in redis
+
+
+    If your model was named ThisModel, the primary key was "key", and prefix and separator were left at default (not set), the
+    keys stored in redis would be
+    thismodel:key
     """
 
     @classmethod
+    @lru_cache(1)
+    def _get_prefix(cls) -> str:
+        prefix_str = getattr(cls, "_redis_prefix", "").lower()
+        return f"{prefix_str}{cls._get_separator()}" if prefix_str != "" else ""
+
+    @classmethod
+    @lru_cache(1)
+    def _get_separator(cls):
+        return getattr(cls, "_redis_separator", ":").lower()
+
+    @classmethod
+    @lru_cache(1)
+    def _get_tablename(cls):
+        return cls.__name__.lower() if cls._table_name is None else cls._table_name
+
+    @classmethod
+    @lru_cache(1)
     def __get_primary_key(cls, primary_key_value: Any):
         """
-        Returns the primary key value concatenated to the table name for uniqueness
+        Uses _table_name, _table_refix, and _redis_separator from the model to build our primary key.
+
+        _table_name defaults to the name of the model class if it is not set
+        _redis_separator defualts to : if it is not set
+        _prefix defaults to nothing if it is not set
+
+        The key is contructed as {_prefix}{_redis_separator}{_table_name}{_redis_separator}{primary_key_value}
+        So a model named ThisModel with a primary key of id, by default would result in a key of thismodel:id
         """
-        table_name = (
-            cls.__name__.lower() if cls._table_name is None else cls._table_name
-        )
-        return f"{table_name}_%&_{primary_key_value}"
+
+        return f"{cls._get_prefix()}{cls._get_tablename()}{cls._get_separator()}{primary_key_value}"
 
     @classmethod
     def get_table_index_key(cls):
         """Returns the key in which the primary keys of the given table have been saved"""
-        table_name = (
-            cls.__name__.lower() if cls._table_name is None else cls._table_name
-        )
-        return f"{table_name}__index"
+        return f"{cls._get_prefix()}{cls._get_tablename()}{cls._get_separator()}__index"
 
     @classmethod
     async def _ids_to_primary_keys(

pyproject.toml

@@ -46,7 +46,6 @@ sphinx-autobuild = ">=2021.3.14"
 pre-commit = ">=2.12.1"
 flake8 = ">=3.9.1"
 black = ">=21.10b0"
-flake8-bandit = "^3.0.0"
 flake8-bugbear = ">=21.9.2"
 flake8-docstrings = "^1.6.0"
 flake8-rst-docstrings = "^0.2.7"
@@ -74,6 +73,9 @@ tox = "^3.26.0"
 pylint = "^2.13.9"
 setuptools-git-versioning = "^1.10.1"
 
+[tool.poetry.group.dev.dependencies]
+bandit = "^1.7.4"
+
 [tool.coverage.paths]
 source = ["pydantic_aioredis", "*/site-packages"]
 tests = ["tests", "*/tests"]

test/conftest.py

@@ -1,13 +1,17 @@
 import pytest
+import pytest_asyncio
 import redislite
 from pydantic_aioredis.config import RedisConfig
 from pydantic_aioredis.model import Model
 from pydantic_aioredis.store import Store
 
 
-@pytest.fixture()
+@pytest_asyncio.fixture()
 def redis_server(unused_tcp_port):
     """Sets up a fake redis server we can use for tests"""
-    instance = redislite.Redis(serverconfig={"port": unused_tcp_port})
-    yield unused_tcp_port
-    instance.close()
+    try:
+        instance = redislite.Redis(serverconfig={"port": unused_tcp_port})
+        yield unused_tcp_port
+    finally:
+        instance.close()
+        instance.shutdown()

test/ext/FastAPI/test_ext_fastapi.py

@@ -1,6 +1,7 @@
 from typing import List
 
 import pytest
+import pytest_asyncio
 from fastapi import FastAPI
 from httpx import AsyncClient
 from pydantic_aioredis.config import RedisConfig
@@ -13,7 +14,7 @@ class Model(FastAPIModel):
     name: str
 
 
-@pytest.fixture()
+@pytest_asyncio.fixture()
 async def test_app(redis_server):
     store = Store(
         name="sample",

test/ext/FastAPI/test_ext_fastapi_crudrouter.py

@@ -1,6 +1,7 @@
 from typing import List
 
 import pytest
+import pytest_asyncio
 from fastapi import FastAPI
 from httpx import AsyncClient
 from pydantic_aioredis import Model as PAModel
@@ -15,7 +16,7 @@ class Model(PAModel):
     value: int
 
 
-@pytest.fixture()
+@pytest_asyncio.fixture()
 async def test_app(redis_server):
     store = Store(
         name="sample",
@@ -31,7 +32,7 @@ async def test_app(redis_server):
     yield store, app, Model
 
 
-@pytest.fixture()
+@pytest_asyncio.fixture()
 def test_models():
     return [Model(name=f"test{i}", value=i) for i in range(1, 10)]