From 4fb4cd04e0261b83e1461cbb72c3dfba041c28da Mon Sep 17 00:00:00 2001 From: Will Date: Thu, 17 Jun 2021 12:48:54 -0500 Subject: [PATCH] feat(api-updates): update mappings (#25) --- .github/workflows/ci.yml | 10 +- docker-compose.yml | 6 +- postgres-init/init.sh | 2 +- src/mds/agg_mds/datastore/__init__.py | 45 ++++++ .../agg_mds/{ => datastore}/redis_cache.py | 5 +- src/mds/agg_mds/query.py | 37 ++--- src/mds/config.py | 2 +- src/mds/main.py | 11 +- src/mds/populate.py | 28 ++-- tests/conftest.py | 114 ++++++++-------- tests/test_agg_mds_datastore.py | 85 ++++++++++++ tests/test_agg_mds_query.py | 65 ++------- tests/test_agg_mds_redis_cache.py | 129 ++++++++++++++++-- tests/test_populate.py | 6 +- 14 files changed, 369 insertions(+), 176 deletions(-) create mode 100644 src/mds/agg_mds/datastore/__init__.py rename src/mds/agg_mds/{ => datastore}/redis_cache.py (96%) create mode 100644 tests/test_agg_mds_datastore.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e8c12b8..a5b8a7bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,9 +10,9 @@ jobs: postgres: image: postgres:9.6 env: - POSTGRES_USER: mds - POSTGRES_PASSWORD: mds - POSTGRES_DB: test_mds + POSTGRES_USER: metadata_user + POSTGRES_PASSWORD: metadata_pass + POSTGRES_DB: test_metadata ports: - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 @@ -40,8 +40,8 @@ jobs: env: DB_HOST: localhost DB_PORT: ${{ job.services.postgres.ports[5432] }} - DB_USER: mds - DB_PASSWORD: mds + DB_USER: metadata_user + DB_PASSWORD: metadata_pass USE_AGG_MDS: true run: | $HOME/.poetry/bin/poetry run pytest --cov=src --cov=migrations/versions --cov-fail-under=94 --cov-report xml diff --git a/docker-compose.yml b/docker-compose.yml index e6c84542..c90ad0f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - redis_migration environment: - DB_HOST=db - - DB_USER=mds + - DB_USER=metadata_user - USE_AGG_MDS=true - REDIS_DB_HOST=redis command: /env/bin/uvicorn --host 0.0.0.0 --port 80 mds.asgi:app --reload @@ -24,7 +24,7 @@ services: - db environment: - DB_HOST=db - - DB_USER=mds + - DB_USER=metadata_user command: /env/bin/alembic upgrade head redis_migration: build: . @@ -38,7 +38,7 @@ services: db: image: postgres environment: - - POSTGRES_USER=mds + - POSTGRES_USER=metadata_user - POSTGRES_HOST_AUTH_METHOD=trust volumes: - ./postgres-data:/var/lib/postgresql/data diff --git a/postgres-init/init.sh b/postgres-init/init.sh index 9ddc8c6a..b921e6c2 100755 --- a/postgres-init/init.sh +++ b/postgres-init/init.sh @@ -1,3 +1,3 @@ #!/usr/bin/env sh -createdb -U mds test_mds +createdb -U metadata_user test_metadata diff --git a/src/mds/agg_mds/datastore/__init__.py b/src/mds/agg_mds/datastore/__init__.py new file mode 100644 index 00000000..96df87ec --- /dev/null +++ b/src/mds/agg_mds/datastore/__init__.py @@ -0,0 +1,45 @@ +from .redis_cache import redis_cache as redis_client + + +async def init(hostname, port): + await redis_client.init_cache(hostname, port) + + +async def drop_all(): + await redis_client.json_sets("commons", []) + + +async def close(): + await redis_client.close() + + +async def get_status(): + return await redis_client.get_status() + + +async def update_metadata(*args): + await redis_client.update_metadata(*args) + + +async def get_commons_metadata(*args): + return await redis_client.get_commons_metadata(*args) + + +async def get_all_named_commons_metadata(*args): + return await redis_client.get_all_named_commons_metadata(*args) + + +async def get_commons_metadata_guid(*args): + return await redis_client.get_commons_metadata_guid(*args) + + +async def get_commons_attribute(*args): + return await redis_client.get_commons_attribute(*args) + + +async def get_commons(): + return await redis_client.get_commons() + + +async def get_all_metadata(*args): + return await redis_client.get_all_metadata(*args) diff --git a/src/mds/agg_mds/redis_cache.py b/src/mds/agg_mds/datastore/redis_cache.py similarity index 96% rename from src/mds/agg_mds/redis_cache.py rename to src/mds/agg_mds/datastore/redis_cache.py index eb3027d7..d6a28f70 100644 --- a/src/mds/agg_mds/redis_cache.py +++ b/src/mds/agg_mds/datastore/redis_cache.py @@ -10,7 +10,6 @@ def __init__(self): self.redis_cache: Optional[Redis] = None async def init_cache(self, hostname: str = "0.0.0.0", port: int = 6379): - print(create_redis_pool) self.redis_cache = await create_redis_pool( f"redis://{hostname}:{port}/0?encoding=utf-8" ) @@ -48,8 +47,7 @@ async def close(self): async def update_metadata( self, name: str, - data: dict, - mapping: dict, + data: List[Dict], guid_arr: List[str], tags: Dict[str, List[str]], info: Dict[str, str], @@ -57,7 +55,6 @@ async def update_metadata( ): await self.json_sets(f"{name}", {}) await self.json_sets(name, data, ".metadata") - await self.json_sets(name, mapping, ".field_mapping") await self.json_sets(name, guid_arr, ".guids") await self.json_sets(name, tags, ".tags") await self.json_sets(name, info, ".info") diff --git a/src/mds/agg_mds/query.py b/src/mds/agg_mds/query.py index bfc42e98..e776593d 100644 --- a/src/mds/agg_mds/query.py +++ b/src/mds/agg_mds/query.py @@ -1,6 +1,6 @@ from fastapi import HTTPException, Query, APIRouter, Request from starlette.status import HTTP_404_NOT_FOUND -from mds.agg_mds.redis_cache import redis_cache +from mds.agg_mds import datastore from mds import config mod = APIRouter() @@ -12,7 +12,7 @@ async def get_commons(): Returns a list of all registered commons :return: """ - return await redis_cache.get_commons() + return await datastore.get_commons() @mod.get("/aggregate/metadata") @@ -35,16 +35,15 @@ async def metadata( ... } """ - - return await redis_cache.get_all_metadata(limit, offset) + return await datastore.get_all_metadata(limit, offset) @mod.get("/aggregate/metadata/{name}") -async def metdata_name(name: str): +async def metadata_name(name: str): """ Returns the all the metadata from the named commons. """ - res = await redis_cache.get_all_named_commons_metadata(name) + res = await datastore.get_all_named_commons_metadata(name) if res: return res else: @@ -55,11 +54,11 @@ async def metdata_name(name: str): @mod.get("/aggregate/metadata/{name}/tags") -async def metdata_tags(name: str): +async def metadata_tags(name: str): """ Returns the tags associated with the named commons. """ - res = await redis_cache.get_commons_attribute(name, "tags") + res = await datastore.get_commons_attribute(name, "tags") if res: return res else: @@ -70,23 +69,11 @@ async def metdata_tags(name: str): @mod.get("/aggregate/metadata/{name}/info") -async def metdata_info(name: str): +async def metadata_info(name: str): """ Returns information from the named commons. """ - res = await redis_cache.get_commons_attribute(name, "info") - if res: - return res - else: - raise HTTPException( - HTTP_404_NOT_FOUND, - {"message": f"no common exists with the given: {name}", "code": 404}, - ) - - -@mod.get("/aggregate/metadata/{name}/columns_to_fields") -async def metadata_columns_to_fields(name: str): - res = await redis_cache.get_commons_attribute(name, "field_mapping") + res = await datastore.get_commons_attribute(name, "info") if res: return res else: @@ -97,8 +84,8 @@ async def metadata_columns_to_fields(name: str): @mod.get("/aggregate/metadata/{name}/aggregations") -async def metadtata_aggregations(name: str): - res = await redis_cache.get_commons_attribute(name, "aggregations") +async def metadata_aggregations(name: str): + res = await datastore.get_commons_attribute(name, "aggregations") if res: return res else: @@ -111,7 +98,7 @@ async def metadtata_aggregations(name: str): @mod.get("/aggregate/metadata/{name}/guid/{guid}:path") async def metadata_name_guid(name: str, guid: str): """Get the metadata of the GUID in the named commons.""" - res = await redis_cache.get_commons_metadata_guid(name, guid) + res = await datastore.get_commons_metadata_guid(name, guid) if res: return res else: diff --git a/src/mds/config.py b/src/mds/config.py index e6f74807..18a43373 100644 --- a/src/mds/config.py +++ b/src/mds/config.py @@ -31,7 +31,7 @@ def __init__(self, value): REDIS_PORT = config("REDIS_DB_PORT", cast=int, default=6379) if TESTING: - DB_DATABASE = "test_" + (DB_DATABASE or "mds") + DB_DATABASE = "test_" + (DB_DATABASE or "metadata") TEST_KEEP_DB = config("TEST_KEEP_DB", cast=bool, default=False) DB_DSN = config( diff --git a/src/mds/main.py b/src/mds/main.py index 6f177226..5e3b685a 100644 --- a/src/mds/main.py +++ b/src/mds/main.py @@ -5,7 +5,7 @@ from fastapi import FastAPI, APIRouter import httpx -from mds.agg_mds.redis_cache import redis_cache +from mds.agg_mds import datastore as aggregate_datastore try: from importlib.metadata import entry_points @@ -32,17 +32,16 @@ def get_app(): @app.on_event("shutdown") async def shutdown_event(): if config.USE_AGG_MDS: - logger.info("Closing redis cache.") - await redis_cache.close() + logger.info("Closing aggregate datastore.") + await aggregate_datastore.close() logger.info("Closing async client.") - await redis_cache.close() await app.async_client.aclose() @app.on_event("startup") async def startup_event(): if config.USE_AGG_MDS: - logger.info("Starting redis cache.") - await redis_cache.init_cache( + logger.info("Creating aggregate datastore.") + await aggregate_datastore.init( hostname=config.REDIS_HOST, port=config.REDIS_PORT ) diff --git a/src/mds/populate.py b/src/mds/populate.py index b381ec74..841854f6 100644 --- a/src/mds/populate.py +++ b/src/mds/populate.py @@ -2,9 +2,9 @@ from argparse import Namespace from typing import Any, Dict, List from collections import Counter +from mds.agg_mds import datastore from mds.agg_mds.mds import pull_mds from mds.agg_mds.commons import MDSInstance, Commons, parse_config_from_file -from mds.agg_mds.redis_cache import redis_cache from mds import config from pathlib import Path import argparse @@ -47,8 +47,8 @@ async def main(commons_config: Commons, hostname: str, port: int) -> None: print("aggregate MDS disabled") exit(1) - await redis_cache.init_cache(hostname, port) - await redis_cache.json_sets("commons", []) + await datastore.init(hostname, port) + await datastore.drop_all() for name, common in commons_config.commons.items(): results = pull_mds(common.mds_url) @@ -103,17 +103,29 @@ async def main(commons_config: Commons, hostname: str, port: int) -> None: for k, v in tags.items(): tags[k] = list(tags[k]) + def normalize(entry: Dict[Any, Any]): + # The entry is an object with one top-level key. Its own id. + id = list(entry.keys())[0] + for column, field in common.columns_to_fields.items(): + if field == column: + continue + if column in entry[id]["gen3_discovery"]: + entry[id]["gen3_discovery"][field] = entry[id]["gen3_discovery"][ + column + ] + return entry + + data = [normalize(x) for x in mds_arr] + # build index of keys. which is used to compute the index into the .metadata array # Admittedly a hack but will be faster than using json path, until the release of RedisJson v1.2 keys = list(results.keys()) info = {"commons_url": common.commons_url} - await redis_cache.update_metadata( - name, mds_arr, common.columns_to_fields, keys, tags, info, aggregations - ) + await datastore.update_metadata(name, mds_arr, keys, tags, info, aggregations) - res = await redis_cache.get_status() + res = await datastore.get_status() print(res) - await redis_cache.close() + await datastore.close() async def filter_entries( diff --git a/tests/conftest.py b/tests/conftest.py index ea6c6f83..890322b6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,13 +10,11 @@ from starlette.testclient import TestClient from unittest.mock import MagicMock, patch -import fakeredis.aioredis import asyncio -from aioredis import Redis, create_redis_pool environ["TESTING"] = "TRUE" from mds import config -from mds.agg_mds.redis_cache import RedisCache +from mds.agg_mds import datastore # NOTE: AsyncMock is included in unittest.mock but ONLY in Python 3.8+ @@ -38,79 +36,85 @@ def setup_test_database(): main(["--raiseerr", "downgrade", "base"]) -class MockRedisJSONStore: - def __init__(self): - self.store = {} - - def _deep_set(self, keys, value, state): - last = keys.pop() - for key in keys: - if key not in state: - state[key] = {} - state = state[key] - state[last] = value - - def _deep_get(self, keys, state): - key = keys.pop(0) - is_last = len(keys) == 0 - next_state = state.get(key, None) - return ( - next_state - if is_last or next_state is None - else self._deep_get(keys, next_state) - ) +@pytest.fixture() +def mock_aggregate_datastore(): + store = { + "commons": {}, + "info": {}, + "aggregations": {}, + "tags": {}, + } - def set(self, key, path, value): - keys = [x for x in f"{key}.{path}".split(".") if x] - self._deep_set(keys, value, self.store) + async def mock_init(hostname, port): + pass - def get(self, key, path=""): - keys = [x for x in f"{key}.{path}".split(".") if x] - return self._deep_get(keys, self.store) + async def mock_drop_all(): + pass + async def mock_get_status(): + pass -@pytest.fixture() -def mock_redis_cache(): - store = MockRedisJSONStore() + async def mock_close(): + pass + + async def mock_get_all_metadata(limit, offset): + return store["commons"] - async def mock_init_cache(self, hostname, port): - self.redis_cache = await fakeredis.aioredis.create_redis_pool() + async def mock_get_all_named_commons_metadata(name): + return store["commons"].get(name, {}) - async def mock_json_sets(self, key, value, path="."): - store.set(key, path, value) + async def mock_update_metadata(name, data, guid_arr, tags, info, aggregations): + store["commons"][name] = data + store["info"][name] = info + store["aggregations"][name] = aggregations + store["tags"][name] = tags - async def mock_json_get(self, key, path="."): - return store.get(key, path) + async def mock_get_commons(): + keys = list(store["commons"].keys()) + return None if len(keys) == 0 else {"commons": keys} - async def mock_json_arr_appends(self, key, value, path="."): - arr = store.get(key, path) or [] - arr.append(value) - store.set(key, path, arr) + async def mock_get_commons_attribute(name, type): + return store[type].get(name, {}) - async def mock_json_arr_index(self, key, guid): - arr = store.get(key) or [] - if guid in arr: - return arr.index(guid) - return -1 + async def mock_get_commons_metadata_guid(name, guid): + studies = store["commons"].get(name, []) + return next((x for x in studies if list(x.keys())[0] == guid), None) patches = [] - patches.append(patch.object(RedisCache, "init_cache", mock_init_cache)) - patches.append(patch.object(RedisCache, "json_sets", mock_json_sets)) - patches.append(patch.object(RedisCache, "json_get", mock_json_get)) - patches.append(patch.object(RedisCache, "json_arr_appends", mock_json_arr_appends)) - patches.append(patch.object(RedisCache, "json_arr_index", mock_json_arr_index)) + patches.append(patch.object(datastore, "init", mock_init)) + patches.append(patch.object(datastore, "drop_all", mock_drop_all)) + patches.append(patch.object(datastore, "get_status", mock_get_status)) + patches.append(patch.object(datastore, "close", mock_close)) + patches.append(patch.object(datastore, "get_all_metadata", mock_get_all_metadata)) + patches.append( + patch.object( + datastore, + "get_all_named_commons_metadata", + mock_get_all_named_commons_metadata, + ) + ) + patches.append(patch.object(datastore, "update_metadata", mock_update_metadata)) + patches.append(patch.object(datastore, "get_commons", mock_get_commons)) + patches.append( + patch.object(datastore, "get_commons_attribute", mock_get_commons_attribute) + ) + patches.append( + patch.object( + datastore, "get_commons_metadata_guid", mock_get_commons_metadata_guid + ) + ) for patched_function in patches: patched_function.start() - yield RedisCache + yield datastore for patched_function in patches: patched_function.stop() @pytest.fixture() -def client(mock_redis_cache): +def client(mock_aggregate_datastore): from mds import config from mds.main import get_app diff --git a/tests/test_agg_mds_datastore.py b/tests/test_agg_mds_datastore.py new file mode 100644 index 00000000..885cd725 --- /dev/null +++ b/tests/test_agg_mds_datastore.py @@ -0,0 +1,85 @@ +import pytest +import nest_asyncio +from unittest.mock import patch +from conftest import AsyncMock +from mds.agg_mds import datastore + +# https://github.com/encode/starlette/issues/440 +nest_asyncio.apply() + + +@pytest.mark.asyncio +async def test_init(): + with patch("mds.agg_mds.datastore.redis_client", AsyncMock()) as mock_redis_cache: + await datastore.init("host", 9999) + mock_redis_cache.init_cache.assert_called_with("host", 9999) + + +@pytest.mark.asyncio +async def test_drop_all(): + with patch("mds.agg_mds.datastore.redis_client", AsyncMock()) as mock_redis_cache: + await datastore.drop_all() + mock_redis_cache.json_sets.assert_called_with("commons", []) + + +@pytest.mark.asyncio +async def test_close(): + with patch("mds.agg_mds.datastore.redis_client", AsyncMock()) as mock_redis_cache: + await datastore.close() + mock_redis_cache.close.assert_called_with() + + +@pytest.mark.asyncio +async def test_get_status(): + with patch("mds.agg_mds.datastore.redis_client", AsyncMock()) as mock_redis_cache: + await datastore.get_status() + mock_redis_cache.get_status.assert_called_with() + + +@pytest.mark.asyncio +async def test_update_metadata(): + with patch("mds.agg_mds.datastore.redis_client", AsyncMock()) as mock_redis_cache: + await datastore.update_metadata() + mock_redis_cache.update_metadata.assert_called_with() + + +@pytest.mark.asyncio +async def test_get_commons_metadata(): + with patch("mds.agg_mds.datastore.redis_client", AsyncMock()) as mock_redis_cache: + await datastore.get_commons_metadata() + mock_redis_cache.get_commons_metadata.assert_called_with() + + +@pytest.mark.asyncio +async def test_get_all_named_commons_metadata(): + with patch("mds.agg_mds.datastore.redis_client", AsyncMock()) as mock_redis_cache: + await datastore.get_all_named_commons_metadata() + mock_redis_cache.get_all_named_commons_metadata.assert_called_with() + + +@pytest.mark.asyncio +async def test_get_commons_metadata_guid(): + with patch("mds.agg_mds.datastore.redis_client", AsyncMock()) as mock_redis_cache: + await datastore.get_commons_metadata_guid() + mock_redis_cache.get_commons_metadata_guid.assert_called_with() + + +@pytest.mark.asyncio +async def test_get_commons_attribute(): + with patch("mds.agg_mds.datastore.redis_client", AsyncMock()) as mock_redis_cache: + await datastore.get_commons_attribute() + mock_redis_cache.get_commons_attribute.assert_called_with() + + +@pytest.mark.asyncio +async def test_get_commons(): + with patch("mds.agg_mds.datastore.redis_client", AsyncMock()) as mock_redis_cache: + await datastore.get_commons() + mock_redis_cache.get_commons.assert_called_with() + + +@pytest.mark.asyncio +async def test_get_all_metadata(): + with patch("mds.agg_mds.datastore.redis_client", AsyncMock()) as mock_redis_cache: + await datastore.get_all_metadata() + mock_redis_cache.get_all_metadata.assert_called_with() diff --git a/tests/test_agg_mds_query.py b/tests/test_agg_mds_query.py index 0e8a29e1..92350e3a 100644 --- a/tests/test_agg_mds_query.py +++ b/tests/test_agg_mds_query.py @@ -1,7 +1,7 @@ from typing import Dict import pytest import nest_asyncio -from mds.agg_mds.redis_cache import redis_cache +from mds.agg_mds import datastore # https://github.com/encode/starlette/issues/440 @@ -14,19 +14,17 @@ async def test_aggregate_commons(client): assert resp.status_code == 200 assert resp.json() == None - await redis_cache.update_metadata( + await datastore.update_metadata( "commons1", [], - {}, [], None, None, None, ) - await redis_cache.update_metadata( + await datastore.update_metadata( "commons2", [], - {}, [], None, None, @@ -43,27 +41,25 @@ async def test_aggregate_metadata(client): assert resp.status_code == 200 assert resp.json() == {} - await redis_cache.update_metadata( + await datastore.update_metadata( "commons1", [ { "study1": {}, } ], - {}, ["study1"], None, None, None, ) - await redis_cache.update_metadata( + await datastore.update_metadata( "commons2", [ { "study2": {}, } ], - {}, ["study2"], None, None, @@ -86,7 +82,7 @@ async def test_aggregate_metadata(client): @pytest.mark.asyncio -async def test_aggregate_metdata_name(client): +async def test_aggregate_metadata_name(client): resp = client.get("/aggregate/metadata/commons1") assert resp.status_code == 404 assert resp.json() == { @@ -96,14 +92,13 @@ async def test_aggregate_metdata_name(client): } } - await redis_cache.update_metadata( + await datastore.update_metadata( "commons1", [ { "study1": {}, } ], - {}, ["study1"], None, None, @@ -115,7 +110,7 @@ async def test_aggregate_metdata_name(client): @pytest.mark.asyncio -async def test_aggregate_metdata_tags(client): +async def test_aggregate_metadata_tags(client): resp = client.get("/aggregate/metadata/commons1/tags") assert resp.status_code == 404 assert resp.json() == { @@ -125,14 +120,13 @@ async def test_aggregate_metdata_tags(client): } } - await redis_cache.update_metadata( + await datastore.update_metadata( "commons1", [ { "study1": {}, } ], - {}, ["study1"], ["mytag1"], None, @@ -144,7 +138,7 @@ async def test_aggregate_metdata_tags(client): @pytest.mark.asyncio -async def test_aggregate_metdata_info(client): +async def test_aggregate_metadata_info(client): resp = client.get("/aggregate/metadata/commons1/info") assert resp.status_code == 404 assert resp.json() == { @@ -154,14 +148,13 @@ async def test_aggregate_metdata_info(client): } } - await redis_cache.update_metadata( + await datastore.update_metadata( "commons1", [ { "study1": {}, } ], - {}, ["guid1"], None, {"commons_url": "http://commons"}, @@ -173,36 +166,7 @@ async def test_aggregate_metdata_info(client): @pytest.mark.asyncio -async def test_aggregate_metdata_field_to_columns(client): - resp = client.get("/aggregate/metadata/commons1/columns_to_fields") - assert resp.status_code == 404 - assert resp.json() == { - "detail": { - "code": 404, - "message": "no common exists with the given: commons1", - } - } - - await redis_cache.update_metadata( - "commons1", - [ - { - "study1": {}, - } - ], - {"fields": {"some_key": "other_key"}}, - ["study1"], - None, - {"commons_url": "http://commons"}, - None, - ) - resp = client.get("/aggregate/metadata/commons1/columns_to_fields") - assert resp.status_code == 200 - assert resp.json() == {"fields": {"some_key": "other_key"}} - - -@pytest.mark.asyncio -async def test_metadtata_aggregations(client): +async def test_metadata_aggregations(client): resp = client.get("/aggregate/metadata/commons1/aggregations") assert resp.status_code == 404 assert resp.json() == { @@ -214,7 +178,7 @@ async def test_metadtata_aggregations(client): @pytest.mark.asyncio -async def test_aggregate_metdata_name_guid(client): +async def test_aggregate_metadata_name_guid(client): resp = client.get("/aggregate/metadata/commons1/guid/study2:path") assert resp.status_code == 404 assert resp.json() == { @@ -224,7 +188,7 @@ async def test_aggregate_metdata_name_guid(client): } } - await redis_cache.update_metadata( + await datastore.update_metadata( "commons1", [ { @@ -234,7 +198,6 @@ async def test_aggregate_metdata_name_guid(client): "study2": {}, }, ], - {"fields": {"some_key": "other_key"}}, ["study1", "study2"], None, {"commons_url": "http://commons"}, diff --git a/tests/test_agg_mds_redis_cache.py b/tests/test_agg_mds_redis_cache.py index 7b17a82c..3826145a 100644 --- a/tests/test_agg_mds_redis_cache.py +++ b/tests/test_agg_mds_redis_cache.py @@ -1,12 +1,14 @@ import json -from unittest.mock import patch +from unittest.mock import patch, call, MagicMock from conftest import AsyncMock import pytest -from mds.agg_mds.redis_cache import RedisCache -import mds.agg_mds.redis_cache +import mds +from mds.agg_mds.datastore.redis_cache import RedisCache +import mds.agg_mds.datastore.redis_cache import nest_asyncio import fakeredis.aioredis import aioredis +from datetime import datetime @pytest.mark.asyncio @@ -16,7 +18,9 @@ async def test_init_cache(): async def mock_pool(address): return f"mock:result:{address}" - with patch.object(mds.agg_mds.redis_cache, "create_redis_pool", mock_pool): + with patch.object( + mds.agg_mds.datastore.redis_cache, "create_redis_pool", mock_pool + ): await cache.init_cache() assert cache.redis_cache == "mock:result:redis://0.0.0.0:6379/0?encoding=utf-8" @@ -25,6 +29,7 @@ async def mock_pool(address): async def test_keys(): cache = RedisCache() cache.redis_cache = await fakeredis.aioredis.create_redis_pool() + await cache.redis_cache.set("commons1", "some data") keys = await cache.keys("commons1") assert keys == [b"commons1"] @@ -47,28 +52,128 @@ async def test_json_get(): @pytest.mark.asyncio -async def test_get_commons_metadata(): +async def test_get_status(): + cache = RedisCache() + + mock_data = ["status2", "status1", ["commons1", "commons2"]] + + async def mock_json_get(arg1, arg2=None): + return mock_data.pop() + + patch.object(cache, "json_get", mock_json_get).start() + + assert await cache.get_status() == { + "commons1": "status1", + "commons2": "status2", + } + + +@pytest.mark.asyncio +async def test_close(): cache = RedisCache() cache.redis_cache = await fakeredis.aioredis.create_redis_pool() + patch.object(cache.redis_cache, "close", MagicMock()).start() + patch.object(cache.redis_cache, "wait_closed", AsyncMock()).start() + + await cache.close() + + cache.redis_cache.close.assert_called_with() + cache.redis_cache.wait_closed.assert_called_with() + + +@pytest.mark.asyncio +async def test_update_metadata(): + cache = RedisCache() + cache.json_sets = AsyncMock() + cache.json_arr_appends = AsyncMock() + + now = datetime.now() + with patch("mds.agg_mds.datastore.redis_cache.datetime") as mock_date: + mock_date.now = MagicMock(return_value=now) + await cache.update_metadata("commons1", [], [], {}, {}, {}) + + cache.json_sets.assert_has_calls( + [ + call("commons1", {}), + call("commons1", [], ".metadata"), + call("commons1", [], ".guids"), + call("commons1", {}, ".tags"), + call("commons1", {}, ".info"), + call("commons1", {}, ".aggregations"), + call( + "commons1.status", + { + "last_update": now.strftime("%Y%m%d%H%M%S"), + "error": 0, + "count": "none", + }, + ), + ] + ) + cache.json_arr_appends.assert_called_with("commons", "commons1") + + +@pytest.mark.asyncio +async def test_get_commons_metadata(): + cache = RedisCache() + with patch.object(cache, "json_get", AsyncMock(return_value=None)): - keys = await cache.get_commons_metadata("commons1", 3, 2) - assert keys == None + assert await cache.get_commons_metadata("commons1", 3, 2) == None with patch.object( cache, "json_get", AsyncMock(return_value=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) ): - keys = await cache.get_commons_metadata("commons1", 3, 2) - assert keys == [3, 4, 5] + assert await cache.get_commons_metadata("commons1", 3, 2) == [3, 4, 5] @pytest.mark.asyncio async def test_get_commons_metadata_guid(): cache = RedisCache() - cache.redis_cache = await fakeredis.aioredis.create_redis_pool() patch.object(cache, "json_get", AsyncMock(return_value=[])).start() patch.object(cache, "json_arr_index", AsyncMock(return_value=None)).start() - keys = await cache.get_commons_metadata_guid("commons1", "guid1") - assert keys == None + assert await cache.get_commons_metadata_guid("commons1", "guid1") == None + + patch.object( + cache, "json_get", AsyncMock(return_value=["commons0", "commons1", "commons2"]) + ).start() + patch.object(cache, "json_arr_index", AsyncMock(return_value=1)).start() + + assert await cache.get_commons_metadata_guid("commons1", "guid1") == "commons1" + + +@pytest.mark.asyncio +async def test_get_commons_attribute(): + cache = RedisCache() + cache.json_get = AsyncMock() + + await cache.get_commons_attribute("something", "other") + + cache.json_get.assert_called_with("something", "other") + + +@pytest.mark.asyncio +async def test_get_all_metadata(): + cache = RedisCache() + + patch.object(cache, "json_get", AsyncMock(return_value=None)).start() + + assert await cache.get_all_metadata(2, 4) == {} + + mock_data = [ + [None, "recordX2", "recordY2", "recordZ2", None], + [None, None, None, "recordX1", "recordY1", "recordZ1", None], + ["commons1", "commons2"], + ] + + async def mock_json_get(arg1, arg2=None): + return mock_data.pop() + + patch.object(cache, "json_get", mock_json_get).start() + + assert await cache.get_all_metadata(2, 4) == { + "commons1": ["recordY1", "recordZ1"], + "commons2": [None], + } diff --git a/tests/test_populate.py b/tests/test_populate.py index f54a8245..377c9231 100644 --- a/tests/test_populate.py +++ b/tests/test_populate.py @@ -1,13 +1,9 @@ import pytest from argparse import Namespace from mds.populate import parse_args, main, filter_entries -import mds.agg_mds.mds -from mds.agg_mds import mds from mds.agg_mds.commons import MDSInstance, Commons -from mds.agg_mds.redis_cache import redis_cache, RedisCache import respx from unittest.mock import patch -import fakeredis @pytest.mark.asyncio @@ -31,7 +27,7 @@ async def test_parse_args(): @pytest.mark.asyncio -async def test_main(mock_redis_cache): +async def test_main(mock_aggregate_datastore): def mock_pull_mds(url): return { "thing": {