From 1d09552fe50875e1180fbd7db684f7a996e07dcf Mon Sep 17 00:00:00 2001 From: Guilherme Souza <101073+guilhermef@users.noreply.github.com> Date: Thu, 27 Oct 2022 02:31:20 +0200 Subject: [PATCH 1/3] Add Thumbor redis storage --- remotecv/storages/__init__.py | 0 remotecv/storages/redis_storage.py | 81 ++++++++++++++++++++++++++++++ setup.py | 2 + tests/test_redis_storage.py | 29 +++++++++++ 4 files changed, 112 insertions(+) create mode 100644 remotecv/storages/__init__.py create mode 100644 remotecv/storages/redis_storage.py create mode 100644 tests/test_redis_storage.py diff --git a/remotecv/storages/__init__.py b/remotecv/storages/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/remotecv/storages/redis_storage.py b/remotecv/storages/redis_storage.py new file mode 100644 index 0000000..b83cc70 --- /dev/null +++ b/remotecv/storages/redis_storage.py @@ -0,0 +1,81 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# thumbor imaging service +# https://github.com/thumbor/thumbor/wiki + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license +# Copyright (c) 2011 globo.com thumbor@googlegroups.com + + +from aioredis import Redis +from aioredis.sentinel import Sentinel +from redis import RedisError +from json import loads +from remotecv.utils import logger + + +SINGLE_NODE = "single_node" +SENTINEL = "sentinel" + +class Storage: + def __init__(self, context): + self.context = context + self.redis_client = self.redis_client() + + async def get_detector_data(self, path): + data = await self.redis_client.get(f"thumbor-detector-{path}") + if data: + return loads(data) + return None + + def redis_client(self): + redis_mode = str(self.context.config.REDIS_QUEUE_MODE).lower() + + if redis_mode == SINGLE_NODE: + return self.__redis_single_node_client() + if redis_mode == SENTINEL: + return self.__redis_sentinel_client() + + raise RedisError( + f"REDIS_QUEUE_MODE must be {SINGLE_NODE} or {SENTINEL}" + ) + + def __redis_single_node_client(self): + return Redis( + host=self.context.config.REDIS_QUEUE_SERVER_HOST, + port=self.context.config.REDIS_QUEUE_SERVER_PORT, + db=self.context.config.REDIS_QUEUE_SERVER_DB, + password=self.context.config.REDIS_QUEUE_SERVER_PASSWORD, + ) + + def __redis_sentinel_client(self): + instances_split = ( + self.context.config.REDIS_QUEUE_SENTINEL_INSTANCES.split(",") + ) + instances = [ + tuple(instance.split(":")) for instance in instances_split + ] + + if self.context.config.REDIS_QUEUE_SENTINEL_PASSWORD: + sentinel_instance = Sentinel( + instances, + socket_timeout=self.context.config.REDIS_QUEUE_SENTINEL_SOCKET_TIMEOUT, + sentinel_kwargs={ + "password": self.context.config.REDIS_QUEUE_SENTINEL_PASSWORD + }, + ) + else: + sentinel_instance = Sentinel( + instances, + socket_timeout=self.context.config.REDIS_QUEUE_SENTINEL_SOCKET_TIMEOUT, + ) + + return sentinel_instance.master_for( + self.context.config.REDIS_QUEUE_SENTINEL_MASTER_INSTANCE, + socket_timeout=self.context.config.REDIS_QUEUE_SENTINEL_SOCKET_TIMEOUT, + password=self.context.config.REDIS_QUEUE_SENTINEL_MASTER_PASSWORD, + db=self.context.config.REDIS_QUEUE_SENTINEL_MASTER_DB, + ) + diff --git a/setup.py b/setup.py index 03eab5c..73efa42 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,7 @@ "pytest-cov==3.*,>=3.0.0", "pytest-asyncio==0.*,>=0.18.0", "coverage==6.*,>=6.3.2", + "thumbor==7.*", ] RUNTIME_REQUIREMENTS = [ @@ -20,6 +21,7 @@ "Pillow>=9.0.0", "pyres==1.*,>=1.5.0", "sentry-sdk==0.*,>=0.14.2", + "aioredis==2.*" ] setup( diff --git a/tests/test_redis_storage.py b/tests/test_redis_storage.py new file mode 100644 index 0000000..a78278b --- /dev/null +++ b/tests/test_redis_storage.py @@ -0,0 +1,29 @@ + +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import remotecv.storages.redis_storage + +from thumbor.testing import TestCase +from tornado.testing import gen_test + +# from unittest.mock import patch, AsyncMock + +import asyncio + +class RedisStorageTestCase(TestCase): + + @gen_test + async def test_should_be_none_when_not_available(self): + storage = remotecv.storages.redis_storage.Storage(self.context) + result = await storage.get_detector_data("random_path") + self.assertIsNone(result) + + # @gen_test + # @patch('remotecv.storages.redis_storage.Storage.redis_client', new=AsyncMock) + # async def test_should_be_points_when_available(self, redis_mock): + # redis_mock.get.return_value = "[{x: 1}]" + # storage = remotecv.storages.redis_storage.Storage(self.context) + # result = await storage.get_detector_data("random_path") + # self.assertIsNone(result) + From 75db002985d41a50201db8a58668fa1504a3d302 Mon Sep 17 00:00:00 2001 From: Guilherme Souza <101073+guilhermef@users.noreply.github.com> Date: Sun, 30 Oct 2022 15:10:39 +0100 Subject: [PATCH 2/3] Fix Redis storage tests --- .gitignore | 1 + Makefile | 2 +- remotecv/storages/redis_storage.py | 11 +-- setup.py | 2 +- .../redis-sentinel/sentinel-secure.conf | 2 +- tests/test_redis_storage.py | 70 +++++++++++++++---- 6 files changed, 67 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 21d8905..13adcaf 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ build coverage.lcov .env/ .venv/ +.vscode/ diff --git a/Makefile b/Makefile index 7619806..323afad 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ run: test: run-redis unit stop-redis unit: - @pytest --cov=remotecv --cov-report term-missing --asyncio-mode=strict -r tests/ + @pytest --cov=remotecv --cov-report term-missing --asyncio-mode=strict -s -r tests/ coverage: @coverage report -m --fail-under=52 diff --git a/remotecv/storages/redis_storage.py b/remotecv/storages/redis_storage.py index b83cc70..3b17191 100644 --- a/remotecv/storages/redis_storage.py +++ b/remotecv/storages/redis_storage.py @@ -9,20 +9,21 @@ # Copyright (c) 2011 globo.com thumbor@googlegroups.com +from json import loads + from aioredis import Redis from aioredis.sentinel import Sentinel from redis import RedisError -from json import loads -from remotecv.utils import logger SINGLE_NODE = "single_node" SENTINEL = "sentinel" + class Storage: def __init__(self, context): self.context = context - self.redis_client = self.redis_client() + self.redis_client = self.get_redis_client() async def get_detector_data(self, path): data = await self.redis_client.get(f"thumbor-detector-{path}") @@ -30,7 +31,7 @@ async def get_detector_data(self, path): return loads(data) return None - def redis_client(self): + def get_redis_client(self): redis_mode = str(self.context.config.REDIS_QUEUE_MODE).lower() if redis_mode == SINGLE_NODE: @@ -51,6 +52,7 @@ def __redis_single_node_client(self): ) def __redis_sentinel_client(self): + instances_split = ( self.context.config.REDIS_QUEUE_SENTINEL_INSTANCES.split(",") ) @@ -78,4 +80,3 @@ def __redis_sentinel_client(self): password=self.context.config.REDIS_QUEUE_SENTINEL_MASTER_PASSWORD, db=self.context.config.REDIS_QUEUE_SENTINEL_MASTER_DB, ) - diff --git a/setup.py b/setup.py index 73efa42..7943e67 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ "Pillow>=9.0.0", "pyres==1.*,>=1.5.0", "sentry-sdk==0.*,>=0.14.2", - "aioredis==2.*" + "aioredis==2.*", ] setup( diff --git a/tests/fixtures/redis-sentinel/sentinel-secure.conf b/tests/fixtures/redis-sentinel/sentinel-secure.conf index da5facc..7bd9493 100644 --- a/tests/fixtures/redis-sentinel/sentinel-secure.conf +++ b/tests/fixtures/redis-sentinel/sentinel-secure.conf @@ -4,7 +4,7 @@ dir /tmp requirepass SENTINEL_PASSWORD sentinel resolve-hostnames yes -sentinel announce_hostnames yes +sentinel announce-hostnames yes sentinel monitor MASTER_INSTANCE localhost MASTER_PORT SENTINEL_QUORUM sentinel down-after-milliseconds MASTER_INSTANCE SENTINEL_DOWN_AFTER sentinel parallel-syncs MASTER_INSTANCE 1 diff --git a/tests/test_redis_storage.py b/tests/test_redis_storage.py index a78278b..66b4bac 100644 --- a/tests/test_redis_storage.py +++ b/tests/test_redis_storage.py @@ -1,29 +1,73 @@ - #!/usr/bin/python # -*- coding: utf-8 -*- -import remotecv.storages.redis_storage + +import uuid + +from redis import RedisError from thumbor.testing import TestCase from tornado.testing import gen_test +from preggy import expect -# from unittest.mock import patch, AsyncMock +import remotecv.storages.redis_storage -import asyncio class RedisStorageTestCase(TestCase): - @gen_test async def test_should_be_none_when_not_available(self): storage = remotecv.storages.redis_storage.Storage(self.context) - result = await storage.get_detector_data("random_path") + result = await storage.get_detector_data(uuid.uuid4()) + expect(result).to_be_null() self.assertIsNone(result) - # @gen_test - # @patch('remotecv.storages.redis_storage.Storage.redis_client', new=AsyncMock) - # async def test_should_be_points_when_available(self, redis_mock): - # redis_mock.get.return_value = "[{x: 1}]" - # storage = remotecv.storages.redis_storage.Storage(self.context) - # result = await storage.get_detector_data("random_path") - # self.assertIsNone(result) + @gen_test + async def test_should_be_points_when_available(self): + key = uuid.uuid4() + storage = remotecv.storages.redis_storage.Storage(self.context) + await storage.redis_client.set(f"thumbor-detector-{key}", '[{"x": 1}]') + result = await storage.get_detector_data(key) + expect(result).to_equal([{"x": 1}]) + @gen_test + async def test_should_be_error_when_invalid_redis_mode(self): + self.context.config.REDIS_QUEUE_MODE = "invalid" + with self.assertRaises(RedisError): + remotecv.storages.redis_storage.Storage(self.context) + + @gen_test + async def test_should_be_points_when_available_in_sentinal(self): + self.context.config.REDIS_QUEUE_MODE = "sentinel" + self.context.config.REDIS_QUEUE_SENTINEL_INSTANCES = "localhost:26379" + self.context.config.REDIS_QUEUE_SENTINEL_PASSWORD = None + self.context.config.REDIS_QUEUE_SENTINEL_MASTER_INSTANCE = ( + "redismaster" + ) + self.context.config.REDIS_QUEUE_SENTINEL_MASTER_PASSWORD = None + self.context.config.REDIS_QUEUE_SENTINEL_MASTER_DB = 0 + + key = uuid.uuid4() + storage = remotecv.storages.redis_storage.Storage(self.context) + await storage.redis_client.set(f"thumbor-detector-{key}", '[{"x": 1}]') + result = await storage.get_detector_data(key) + expect(result).to_equal([{"x": 1}]) + + @gen_test + async def test_should_be_points_when_available_in_sentinal_without_auth( + self, + ): + self.context.config.REDIS_QUEUE_MODE = "sentinel" + self.context.config.REDIS_QUEUE_SENTINEL_INSTANCES = "localhost:26380" + self.context.config.REDIS_QUEUE_SENTINEL_PASSWORD = "superpassword" + self.context.config.REDIS_QUEUE_SENTINEL_MASTER_INSTANCE = ( + "redismaster" + ) + self.context.config.REDIS_QUEUE_SENTINEL_MASTER_PASSWORD = None + self.context.config.REDIS_QUEUE_SENTINEL_MASTER_DB = 0 + + storage = remotecv.storages.redis_storage.Storage(self.context) + await storage.redis_client.set( + "thumbor-detector-random_path", '[{"x": 1}]' + ) + result = await storage.get_detector_data("random_path") + expect(result).to_equal([{"x": 1}]) From b255ba5411c5291dd871700d88b993faf66e8f35 Mon Sep 17 00:00:00 2001 From: Guilherme Souza <101073+guilhermef@users.noreply.github.com> Date: Wed, 2 Nov 2022 18:56:27 +0100 Subject: [PATCH 3/3] Address @scorphus comments --- .gitignore | 1 - remotecv/storages/redis_storage.py | 6 +++--- setup.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 13adcaf..21d8905 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,3 @@ build coverage.lcov .env/ .venv/ -.vscode/ diff --git a/remotecv/storages/redis_storage.py b/remotecv/storages/redis_storage.py index 3b17191..8a8dca6 100644 --- a/remotecv/storages/redis_storage.py +++ b/remotecv/storages/redis_storage.py @@ -11,8 +11,8 @@ from json import loads -from aioredis import Redis -from aioredis.sentinel import Sentinel +from redis.asyncio import Redis +from redis.asyncio.sentinel import Sentinel from redis import RedisError @@ -57,7 +57,7 @@ def __redis_sentinel_client(self): self.context.config.REDIS_QUEUE_SENTINEL_INSTANCES.split(",") ) instances = [ - tuple(instance.split(":")) for instance in instances_split + tuple(instance.split(":", 1)) for instance in instances_split ] if self.context.config.REDIS_QUEUE_SENTINEL_PASSWORD: diff --git a/setup.py b/setup.py index 7943e67..159b4f5 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ "Pillow>=9.0.0", "pyres==1.*,>=1.5.0", "sentry-sdk==0.*,>=0.14.2", - "aioredis==2.*", + "redis==4.*,>=4.2.0", ] setup(