Merge branch 'main' of https://github.com/materialsproject/maggma int…

…o ci_updates
materialsproject · Jun 8, 2023 · 6b4f68e · 6b4f68e
2 parents dc3b808 + bcd0f06
commit 6b4f68e
Show file tree

Hide file tree

Showing 36 changed files with 2,982 additions and 242 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -1,2 +1,2 @@
 [run]
-omit = *test*
+omit = *test*, rabbitmq*
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
@@ -55,6 +55,11 @@ jobs:
         ports:
           - 27017:27017
 
+      azurite:
+        image: mcr.microsoft.com/azure-storage/azurite
+        ports:
+          - 10000:10000
+
     strategy:
       max-parallel: 6
       matrix:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,12 +1,34 @@
+default_stages: [commit]
+
+default_install_hook_types: [pre-commit, commit-msg]
+
 repos:
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.0.261
+    hooks:
+      - id: ruff
+        args: [--fix, --ignore, "D,E501"]
+
+  - repo: https://github.com/psf/black
+    rev: 23.3.0
+    hooks:
+      - id: black-jupyter
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
+    rev: v4.4.0
     hooks:
-    -   id: check-yaml
-    -   id: end-of-file-fixer
-    -   id: trailing-whitespace
-    -   id: flake8
-  - repo: https://github.com/ambv/black
-    rev: 22.10.0
+      - id: check-case-conflict
+      - id: check-symlinks
+      - id: check-yaml
+      - id: destroyed-symlinks
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: trailing-whitespace
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.4
     hooks:
-    - id: black
+      - id: codespell
+        stages: [commit, commit-msg]
+        exclude_types: [json, bib, svg]
+        args: [--ignore-words-list, "mater,fwe,te"]
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -1,5 +1,191 @@
 # Changelog
 
+## [v0.51.6](https://github.com/materialsproject/maggma/tree/v0.51.6) (2023-06-08)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.51.5...v0.51.6)
+
+**Merged pull requests:**
+
+- Use tqdm.auto [\#795](https://github.com/materialsproject/maggma/pull/795) ([utf](https://github.com/utf))
+
+## [v0.51.5](https://github.com/materialsproject/maggma/tree/v0.51.5) (2023-06-06)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.51.4...v0.51.5)
+
+**Merged pull requests:**
+
+- Disable worker timeouts by default [\#793](https://github.com/materialsproject/maggma/pull/793) ([munrojm](https://github.com/munrojm))
+
+## [v0.51.4](https://github.com/materialsproject/maggma/tree/v0.51.4) (2023-06-02)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.51.3...v0.51.4)
+
+**Merged pull requests:**
+
+- modify JSONStore file creation [\#792](https://github.com/materialsproject/maggma/pull/792) ([gpetretto](https://github.com/gpetretto))
+
+## [v0.51.3](https://github.com/materialsproject/maggma/tree/v0.51.3) (2023-05-29)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.51.2...v0.51.3)
+
+## [v0.51.2](https://github.com/materialsproject/maggma/tree/v0.51.2) (2023-05-29)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.51.1...v0.51.2)
+
+**Merged pull requests:**
+
+- Add orjson options in JSONStore [\#791](https://github.com/materialsproject/maggma/pull/791) ([gpetretto](https://github.com/gpetretto))
+- Implementation of an AzureBlobStore for Azure blobs [\#790](https://github.com/materialsproject/maggma/pull/790) ([gpetretto](https://github.com/gpetretto))
+
+## [v0.51.1](https://github.com/materialsproject/maggma/tree/v0.51.1) (2023-05-22)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.51.0...v0.51.1)
+
+**Merged pull requests:**
+
+- Add ruamel-yaml as a dependency [\#789](https://github.com/materialsproject/maggma/pull/789) ([sivonxay](https://github.com/sivonxay))
+
+## [v0.51.0](https://github.com/materialsproject/maggma/tree/v0.51.0) (2023-05-22)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.50.4...v0.51.0)
+
+**Merged pull requests:**
+
+- Create a MultiStore object and a Store-like object to access it [\#787](https://github.com/materialsproject/maggma/pull/787) ([sivonxay](https://github.com/sivonxay))
+
+## [v0.50.4](https://github.com/materialsproject/maggma/tree/v0.50.4) (2023-04-28)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.50.3...v0.50.4)
+
+**Merged pull requests:**
+
+- Update MongoStore `count` method [\#785](https://github.com/materialsproject/maggma/pull/785) ([munrojm](https://github.com/munrojm))
+
+## [v0.50.3](https://github.com/materialsproject/maggma/tree/v0.50.3) (2023-02-17)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.50.2...v0.50.3)
+
+**Merged pull requests:**
+
+- Remove extra heartbeats from workers [\#779](https://github.com/materialsproject/maggma/pull/779) ([munrojm](https://github.com/munrojm))
+
+## [v0.50.2](https://github.com/materialsproject/maggma/tree/v0.50.2) (2023-02-17)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.50.1...v0.50.2)
+
+**Merged pull requests:**
+
+- Pydantic CLI settings [\#778](https://github.com/materialsproject/maggma/pull/778) ([munrojm](https://github.com/munrojm))
+
+## [v0.50.1](https://github.com/materialsproject/maggma/tree/v0.50.1) (2023-02-16)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.50.0...v0.50.1)
+
+**Merged pull requests:**
+
+- Remove stray print in worker debug [\#777](https://github.com/materialsproject/maggma/pull/777) ([munrojm](https://github.com/munrojm))
+
+## [v0.50.0](https://github.com/materialsproject/maggma/tree/v0.50.0) (2023-02-16)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.49.18...v0.50.0)
+
+**Merged pull requests:**
+
+- Overhaul distributed framework and add RabbitMQ support [\#776](https://github.com/materialsproject/maggma/pull/776) ([munrojm](https://github.com/munrojm))
+
+## [v0.49.18](https://github.com/materialsproject/maggma/tree/v0.49.18) (2023-02-13)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.49.17...v0.49.18)
+
+**Merged pull requests:**
+
+- Add more heartbeat pings from worker [\#775](https://github.com/materialsproject/maggma/pull/775) ([munrojm](https://github.com/munrojm))
+
+## [v0.49.17](https://github.com/materialsproject/maggma/tree/v0.49.17) (2023-01-30)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.49.16...v0.49.17)
+
+**Merged pull requests:**
+
+- Remove default sorting from API [\#770](https://github.com/materialsproject/maggma/pull/770) ([munrojm](https://github.com/munrojm))
+
+## [v0.49.16](https://github.com/materialsproject/maggma/tree/v0.49.16) (2023-01-23)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.49.15...v0.49.16)
+
+**Merged pull requests:**
+
+- Query pipeline out of memory fix [\#767](https://github.com/materialsproject/maggma/pull/767) ([munrojm](https://github.com/munrojm))
+
+## [v0.49.15](https://github.com/materialsproject/maggma/tree/v0.49.15) (2023-01-23)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.49.14...v0.49.15)
+
+**Merged pull requests:**
+
+- Fix server-side API sorting [\#766](https://github.com/materialsproject/maggma/pull/766) ([munrojm](https://github.com/munrojm))
+
+## [v0.49.14](https://github.com/materialsproject/maggma/tree/v0.49.14) (2023-01-18)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.49.13...v0.49.14)
+
+**Merged pull requests:**
+
+- Fix S3 store queries in API [\#761](https://github.com/materialsproject/maggma/pull/761) ([munrojm](https://github.com/munrojm))
+
+## [v0.49.13](https://github.com/materialsproject/maggma/tree/v0.49.13) (2023-01-10)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.49.12...v0.49.13)
+
+**Merged pull requests:**
+
+- Aggregation pipelines in resource classes [\#759](https://github.com/materialsproject/maggma/pull/759) ([munrojm](https://github.com/munrojm))
+
+## [v0.49.12](https://github.com/materialsproject/maggma/tree/v0.49.12) (2023-01-09)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.49.11...v0.49.12)
+
+**Merged pull requests:**
+
+- Add default sort parameter to `MongoStore` [\#758](https://github.com/materialsproject/maggma/pull/758) ([munrojm](https://github.com/munrojm))
+
+## [v0.49.11](https://github.com/materialsproject/maggma/tree/v0.49.11) (2022-12-15)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.49.10...v0.49.11)
+
+**Merged pull requests:**
+
+- Async to sync for fastapi funcs [\#750](https://github.com/materialsproject/maggma/pull/750) ([munrojm](https://github.com/munrojm))
+
+## [v0.49.10](https://github.com/materialsproject/maggma/tree/v0.49.10) (2022-12-07)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.49.9...v0.49.10)
+
+**Merged pull requests:**
+
+- Enable disk use in mongo find [\#749](https://github.com/materialsproject/maggma/pull/749) ([munrojm](https://github.com/munrojm))
+
+## [v0.49.9](https://github.com/materialsproject/maggma/tree/v0.49.9) (2022-11-01)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.49.8...v0.49.9)
+
+**Merged pull requests:**
+
+- Parse datetime with dateutil [\#741](https://github.com/materialsproject/maggma/pull/741) ([munrojm](https://github.com/munrojm))
+
+## [v0.49.8](https://github.com/materialsproject/maggma/tree/v0.49.8) (2022-10-25)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.49.7...v0.49.8)
+
+## [v0.49.7](https://github.com/materialsproject/maggma/tree/v0.49.7) (2022-10-25)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.49.6...v0.49.7)
+
+**Merged pull requests:**
+
+- FileStore: fix metadata overwriting path [\#736](https://github.com/materialsproject/maggma/pull/736) ([rkingsbury](https://github.com/rkingsbury))
+- JSONStore: fix last\_updated serialization problem [\#735](https://github.com/materialsproject/maggma/pull/735) ([rkingsbury](https://github.com/rkingsbury))
+
 ## [v0.49.6](https://github.com/materialsproject/maggma/tree/v0.49.6) (2022-10-21)
 
 [Full Changelog](https://github.com/materialsproject/maggma/compare/v0.49.5...v0.49.6)
@@ -298,19 +484,19 @@
 
 ## [v0.36.0](https://github.com/materialsproject/maggma/tree/v0.36.0) (2021-12-06)
 
-[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.35.0...v0.36.0)
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.34.0...v0.36.0)
 
 **Merged pull requests:**
 
 - Added on-disk MongoDB compatible MontyStore [\#514](https://github.com/materialsproject/maggma/pull/514) ([utf](https://github.com/utf))
 
-## [v0.35.0](https://github.com/materialsproject/maggma/tree/v0.35.0) (2021-12-01)
+## [v0.34.0](https://github.com/materialsproject/maggma/tree/v0.34.0) (2021-12-01)
 
-[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.34.0...v0.35.0)
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.35.0...v0.34.0)
 
-## [v0.34.0](https://github.com/materialsproject/maggma/tree/v0.34.0) (2021-12-01)
+## [v0.35.0](https://github.com/materialsproject/maggma/tree/v0.35.0) (2021-12-01)
 
-[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.33.2...v0.34.0)
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.33.2...v0.35.0)
 
 **Merged pull requests:**
 

diff --git a/docs/getting_started/stores.md b/docs/getting_started/stores.md
@@ -21,6 +21,7 @@ Current working and tested `Store` include:
 - `AliasingStore`: aliases keys from the underlying store to new names
 - `SandboxStore: provides permission control to documents via a `_sbxn` sandbox key
 - `JointStore`: joins several MongoDB collections together, merging documents with the same `key`, so they look like one collection
+- `AzureBlobStore`: provides an interface to Azure Blobs for the storage of large amount of data
 
 ## The `Store` interface
 

diff --git a/requirements-docs.txt b/requirements-docs.txt
@@ -3,3 +3,4 @@ mkdocs-material==8.3.9
 mkdocs-minify-plugin==0.5.0
 mkdocstrings==0.18.1
 jinja2<3.2.0
+ruamel-yaml<18.0
diff --git a/requirements-optional.txt b/requirements-optional.txt
@@ -4,3 +4,5 @@ IPython==7.31.1;python_version>"3.6"
 nbformat==5.4.0
 regex==2022.9.13
 montydb==2.4.0
+azure-storage-blob==12.16.0
+azure-identity==1.12.0
diff --git a/requirements-testing.txt b/requirements-testing.txt
@@ -11,4 +11,5 @@ mypy==0.971
 mypy-extensions==0.4.3
 responses<0.22.0
 types-PyYAML==6.0.11
-types-setuptools==65.4.0.0
+types-setuptools==65.4.0.0
+types-python-dateutil==2.8.19.2
diff --git a/requirements.txt b/requirements.txt
@@ -15,5 +15,8 @@ uvicorn==0.18.3
 sshtunnel==0.4.0
 msgpack==1.0.3
 msgpack-python==0.5.6
-orjson==3.8.0
+orjson==3.8.14
 boto3==1.24.42
+python-dateutil==2.8.2
+pydantic
+ruamel.yaml<0.18
diff --git a/setup.py b/setup.py
@@ -27,6 +27,7 @@
     include_package_data=True,
     install_requires=[
         "setuptools",
+        "pydantic",
         "pymongo>=4.2.0",
         "monty>=1.0.2",
         "mongomock>=3.10.0",
@@ -44,11 +45,13 @@
         "msgpack>=0.5.6",
         "orjson>=3.6.0",
         "boto3>=1.20.41",
+        "python-dateutil>=2.8.2"
     ],
     extras_require={
         "vault": ["hvac>=0.9.5"],
         "montydb": ["montydb>=2.3.12"],
         "notebook_runner": ["IPython>=7.16", "nbformat>=5.0", "regex>=2020.6"],
+        "azure": ["azure-storage-blob>=12.16.0", "azure-identity>=1.12.0"]
     },
     classifiers=[
         "Programming Language :: Python :: 3",

diff --git a/src/maggma/api/resource/aggregation.py b/src/maggma/api/resource/aggregation.py
@@ -66,7 +66,7 @@ def build_dynamic_model_search(self):
 
         model_name = self.model.__name__
 
-        async def search(**queries: Dict[str, STORE_PARAMS]) -> Dict:
+        def search(**queries: Dict[str, STORE_PARAMS]) -> Dict:
             request: Request = queries.pop("request")  # type: ignore
             temp_response: Response = queries.pop("temp_response")  # type: ignore