Merge pull request #245 from boozallen/161-upversion-pydantic-to-2.8.x

#161 upversion pydantic to 2.8.x
boozallen · Jul 29, 2024 · 93b3864 · 93b3864
2 parents 9af2b56 + e522070
commit 93b3864
Show file tree

Hide file tree

Showing 19 changed files with 34 additions and 72 deletions.
diff --git a/DRAFT_RELEASE_NOTES.md b/DRAFT_RELEASE_NOTES.md
@@ -37,6 +37,7 @@ intervention. The helm charts referencing these containers have already been upd
   - `aissemble-airflow`
   - `aissemble-kafka`
   - `aissemble-mlflow`
+- Upgraded Pydantic v1.10.x to 2.8.x to incorporate performance improvements and incorporate the availability of new features for future. See [here](https://docs.pydantic.dev/latest/migration/) for the guide on how to migrate pydantic V1 into V2.
 
 # Known Issues
 - There is currently a bug with the Sagemaker training Docker image generated by the `aissemble-sagemaker-training-docker` `Fermenter` profile. The installation of the `logistic-training` module's `requirements.txt` fails, due to an unresolvable set of dependencies.

diff --git a/extensions/aissemble-extensions-model-training-api-sagemaker/pyproject.toml b/extensions/aissemble-extensions-model-training-api-sagemaker/pyproject.toml
@@ -12,7 +12,7 @@ packages = [
 python = ">=3.11.4, <3.12"
 fastapi = ">=0.95.0"
 uvicorn = {version = "^0.18.0", extras = ["standard"]}
-pydantic = ">=1.8.0,<2.0.0"
+pydantic = ">=2.8.0"
 kubernetes = ">=26.1.0"
 urllib3 = "^1.26.18"
 krausening = ">=19"
@@ -27,11 +27,6 @@ setuptools = "^69.0.3"
 kappa-maki = ">=1.0.1"
 pylint = "^3.1.0"
 
-[tool.pylint.'MESSAGES CONTROL']
-# Adding the pydantic package to pylint's whitelist as it raises linting errors. 
-# See issue: https://github.com/pydantic/pydantic/issues/1961
-extension-pkg-whitelist = "pydantic"
-
 [build-system]
 requires = ["poetry-core>=1.7.0"]
 build-backend = "poetry.core.masonry.api"
diff --git a/extensions/extensions-transform/aissemble-extensions-transform-spark-python/pyproject.toml b/extensions/extensions-transform/aissemble-extensions-transform-spark-python/pyproject.toml
@@ -11,7 +11,7 @@ packages = [
 [tool.poetry.dependencies]
 python = ">=3.11.4, <4"
 krausening = ">=19"
-pydantic = ">=1.8.0,<2.0.0"
+pydantic = ">=2.8.0"
 pyspark = "3.4.0"
 cryptography = ">=42.0.4"
 urllib3 = "^1.26.18"

diff --git a/foundation/aissemble-foundation-core-python/pyproject.toml b/foundation/aissemble-foundation-core-python/pyproject.toml
@@ -16,7 +16,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.11.4, <4"
-pydantic = ">=1.8.0,<2.0.0"
+pydantic = ">=2.8.0"
 krausening = ">=19"
 cryptography = ">=42.0.4"
 pyjwt = ">=2.3.0"
@@ -36,11 +36,6 @@ nose = ">=1.3.7"
 kappa-maki = ">=1.0.1"
 pylint = "^3.1.0"
 
-[tool.pylint.'MESSAGES CONTROL']
-# Adding the pydantic package to pylint's whitelist as it raises linting errors. 
-# See issue: https://github.com/pydantic/pydantic/issues/1961
-extension-pkg-whitelist = "pydantic"
-
 [build-system]
 requires = ["poetry-core>=1.7.0"]
 build-backend = "poetry.core.masonry.api"
diff --git a/foundation/aissemble-foundation-core-python/src/policy_manager/policy/json/policy_input.py b/foundation/aissemble-foundation-core-python/src/policy_manager/policy/json/policy_input.py
@@ -28,13 +28,13 @@ class PolicyRuleInput(BaseModel):
     """
     The configuration used for the rule.
     """
-    configurations: Optional[Dict[str, Any]]
+    configurations: Optional[Dict[str, Any]] = None
 
     """
     Any configurations for the target set of data that is needed by this
     rule.
     """
-    targetConfigurations: Optional[Dict[str, Any]]
+    targetConfigurations: Optional[Dict[str, Any]] = None
 
 
 class PolicyInput(BaseModel):
@@ -54,17 +54,17 @@ class PolicyInput(BaseModel):
     """
     The description of the policy.
     """
-    description: Optional[str]
+    description: Optional[str] = None
 
     """
     The targets this policy will be invoked on.
     """
-    targets: Optional[List[Target]]
+    targets: Optional[List[Target]] = None
 
     """
     The optional configuration for whether alerts should be sent or not.
     """
-    shouldSendAlert: Optional[AlertOptions]
+    shouldSendAlert: Optional[AlertOptions] = None
 
     """
     The rules for this policy.
@@ -76,14 +76,14 @@ class PolicyInput(BaseModel):
     a List of Target objects of a single Target attribute 'target'.
     This attribute is replaced by `targets`.
     """
-    target: Optional[Target]
+    target: Optional[Target] = None
 
     def getAnyTargets(self) -> List[Target]:
         """
         Used to check both target attributes to contain backwards compatibility with policies still using deprecated 'target'
         """
         if self.target is not None:
-            PolicyInput.__logger.warn(
+            self.__logger.warn(
                 "Detected use of deprecated Json Property 'target'. Existing "
                 + "values should be moved to the new Json Property 'targets'."
             )

diff --git a/foundation/aissemble-foundation-core-python/src/policy_manager/policy/policy.py b/foundation/aissemble-foundation-core-python/src/policy_manager/policy/policy.py
@@ -7,7 +7,7 @@
 # This software package is licensed under the Booz Allen Public License. All Rights Reserved.
 # #L%
 ###
-from pydantic import BaseModel
+from pydantic import ConfigDict, BaseModel
 from enum import Enum
 from typing import Any, List, Dict, Optional
 from krausening.logging import LogManager
@@ -29,8 +29,8 @@ class Target(BaseModel):
     Target contains the target information for the policy.
     """
 
-    retrieve_url: Optional[str]
-    type: Optional[str]
+    retrieve_url: Optional[str] = None
+    type: Optional[str] = None
 
 
 class ConfiguredTarget(Target):
@@ -53,7 +53,7 @@ class ConfiguredRule(BaseModel):
     __logger = LogManager.get_instance().get_logger("ConfiguredRule")
     __deprecated_set_methods = {"target": "set_deprecated_targetConfigurations"}
     className: str
-    configurations: Optional[Dict[str, Any]]
+    configurations: Optional[Dict[str, Any]] = None
     configuredTargets: Optional[List[ConfiguredTarget]] = []
 
     @property
@@ -103,7 +103,7 @@ class Policy(BaseModel):
     __deprecated_set_methods = {"target": "set_deprecated_target"}
     alertOptions: AlertOptions = AlertOptions.ON_DETECTION
     identifier: str
-    description: Optional[str]
+    description: Optional[str] = None
     targets: Optional[List[Target]] = []
     rules: List[ConfiguredRule] = []
 
@@ -132,9 +132,7 @@ def set_deprecated_target(self, new_value: Target):
         )
         self.targets = [new_value]
 
-    # Pydantic model config to allow policy subclasses to contain additional fields of any type
-    class Config:
-        arbitrary_types_allowed = True
+    model_config = ConfigDict(arbitrary_types_allowed=True)
 
     # Links Policy 'target' attribute to 'set_deprecated_target()' method to support
     # people still assigning values to the old attribute.

diff --git a/...emble-foundation-core-python/src/policy_manager/policy/result/policy_invocation_result.py b/...emble-foundation-core-python/src/policy_manager/policy/result/policy_invocation_result.py
@@ -17,5 +17,5 @@ class PolicyInvocationResult(BaseModel):
     """
 
     policyName: str
-    policyDescription: Optional[str]
+    policyDescription: Optional[str] = None
     timestamp: str
diff --git a/foundation/aissemble-foundation-model-training-api/pyproject.toml b/foundation/aissemble-foundation-model-training-api/pyproject.toml
@@ -13,7 +13,7 @@ aissemble-foundation-messaging-python = {path = "../foundation-messaging/foundat
 python = ">=3.11.4, <3.12"
 fastapi = ">=0.95.0"
 uvicorn = {version = "^0.18.0", extras = ["standard"]}
-pydantic = ">=1.8.0,<2.0.0"
+pydantic = ">=2.8.0"
 kubernetes = ">=26.1.0"
 urllib3 = "^1.26.18"
 krausening = ">=19"

diff --git a/foundation/aissemble-foundation-versioning-service/pyproject.toml b/foundation/aissemble-foundation-versioning-service/pyproject.toml
@@ -17,7 +17,7 @@ python = ">=3.11.4, <3.12"
 mlflow = "^2.3.1"
 fastapi = ">=0.95.0"
 uvicorn = {version = "^0.18.0", extras = ["standard"]}
-pydantic = ">=1.8.0,<2.0.0"
+pydantic = ">=2.8.0"
 starlette = ">=0.25.0"
 urllib3 = "^1.26.18"
 
@@ -32,11 +32,6 @@ setuptools = "^69.0.3"
 kappa-maki = ">=1.0.1"
 pylint = "^3.1.0"
 
-[tool.pylint.'MESSAGES CONTROL']
-# Adding the pydantic package to pylint's whitelist as it raises linting errors. 
-# See issue: https://github.com/pydantic/pydantic/issues/1961
-extension-pkg-whitelist = "pydantic"
-
 [build-system]
 requires = ["poetry-core>=1.7.0"]
 build-backend = "poetry.core.masonry.api"
diff --git a/foundation/aissemble-foundation-versioning-service/src/model_versioning/versioning_api.py b/foundation/aissemble-foundation-versioning-service/src/model_versioning/versioning_api.py
@@ -25,7 +25,7 @@
 class VersioningResponse(BaseModel):
     success: bool
     message: str
-    version: Optional[str]
+    version: Optional[str] = None
 
 
 class JWTBearer(HTTPBearer):

diff --git a/...ndation-drift-detection/aissemble-foundation-drift-detection-client-python/pyproject.toml b/...ndation-drift-detection/aissemble-foundation-drift-detection-client-python/pyproject.toml
@@ -11,7 +11,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.11.4, <4"
-pydantic = ">=1.8.0,<2.0.0"
+pydantic = ">=2.8.0"
 requests = ">=2.32.2"
 krausening = ">=19"
 cryptography = ">=42.0.4"
@@ -23,11 +23,6 @@ behave = ">=1.2.6"
 kappa-maki = ">=1.0.1"
 pylint = "^3.1.0"
 
-[tool.pylint.'MESSAGES CONTROL']
-# Adding the pydantic package to pylint's whitelist as it raises linting errors. 
-# See issue: https://github.com/pydantic/pydantic/issues/1961
-extension-pkg-whitelist = "pydantic"
-
 [build-system]
 requires = ["poetry-core>=1.7.0"]
 build-backend = "poetry.core.masonry.api"
diff --git a/...dation-drift-detection-client-python/src/drift/detection/rest/client/drift_rest_client.py b/...dation-drift-detection-client-python/src/drift/detection/rest/client/drift_rest_client.py
@@ -26,7 +26,7 @@ class DriftVariable(DriftData):
     Represents a single drift variable
     """
 
-    type = "single"
+    type: str = "single"
     value: float
 
 
@@ -35,7 +35,7 @@ class DriftVariables(DriftData):
     Represents multiple drift variables
     """
 
-    type = "multiple"
+    type: str = "multiple"
     variables: List[DriftVariable] = []
 
 
@@ -44,8 +44,8 @@ class DriftDataInput(BaseModel):
     Represents the data needed.
     """
 
-    input: Optional[DriftData]
-    control: Optional[DriftData]
+    input: Optional[DriftData] = None
+    control: Optional[DriftData] = None
 
 
 class DriftDetectionResult(BaseModel):
@@ -67,11 +67,9 @@ def invoke_drift(self, policy_identifier, input, control):
         drift_data_input = DriftDataInput(input=input, control=control)
         query_params = {"policyIdentifier": policy_identifier}
         data_headers = {"Content-type": "application/json"}
-        # uncomment below for debugging purposes
-        # print('JSON being sent to drift invocation service: ' + drift_data_input.json())
         return requests.post(
             hostname + "/invoke-drift",
             params=query_params,
-            data=drift_data_input.json(),
+            data=drift_data_input.model_dump_json(serialize_as_any=True),
             headers=data_headers,
         )
diff --git a/...oundation-encryption-policy-python/src/aissemble_encrypt_policy/data_encryption_policy.py b/...oundation-encryption-policy-python/src/aissemble_encrypt_policy/data_encryption_policy.py
@@ -17,6 +17,6 @@ class DataEncryptionPolicy(Policy):
     configurations that are needed for data encryption.
     """
 
-    encryptPhase: Optional[str]
+    encryptPhase: Optional[str] = None
     encryptFields: List[str] = []
-    encryptAlgorithm: Optional[str]
+    encryptAlgorithm: Optional[str] = None
diff --git a/...-policy-python/src/aissemble_encrypt_policy/encrypt_input/data_encryption_policy_input.py b/...-policy-python/src/aissemble_encrypt_policy/encrypt_input/data_encryption_policy_input.py
@@ -17,6 +17,6 @@ class DataEncryptionPolicyInput(PolicyInput):
     configurations that are needed for data encryption.
     """
 
-    encryptPhase: Optional[str]
+    encryptPhase: Optional[str] = None
     encryptFields: List[str] = []
-    encryptAlgorithm: Optional[str]
+    encryptAlgorithm: Optional[str] = None
diff --git a/...ation/foundation-mda/src/main/resources/templates/inference/inference.api.grpc.base.py.vm b/...ation/foundation-mda/src/main/resources/templates/inference/inference.api.grpc.base.py.vm
@@ -1,5 +1,5 @@
 from google.protobuf import json_format
-from pydantic.error_wrappers import ValidationError
+from pydantic import ValidationError
 from grpc import StatusCode
 import logging
 

diff --git a/foundation/foundation-mda/src/main/resources/templates/inference/pyproject.toml.vm b/foundation/foundation-mda/src/main/resources/templates/inference/pyproject.toml.vm
@@ -35,11 +35,6 @@ grpcio-tools = "^1.50.0"
 grpcio-testing = "^1.50.0"
 pylint = "^3.1.0"
 
-[tool.pylint.'MESSAGES CONTROL']
-# Adding the pydantic package to pylint's whitelist as it raises linting errors. 
-# See issue: https://github.com/pydantic/pydantic/issues/1961
-extension-pkg-whitelist = "pydantic"
-
 [tool.pylint.MASTER]
 # Ignores protobuf generated files that fail linting
 ignore-patterns = '.*pb2[\S]*.py'

diff --git a/foundation/foundation-transform/aissemble-foundation-transform-core-python/pyproject.toml b/foundation/foundation-transform/aissemble-foundation-transform-core-python/pyproject.toml
@@ -11,7 +11,7 @@ packages = [
 [tool.poetry.dependencies]
 python = ">=3.11.4, <4"
 aissemble-foundation-core-python = {path = "../../aissemble-foundation-core-python", develop = true}
-pydantic = ">=1.8.0,<2.0.0"
+pydantic = ">=2.8.0"
 krausening = ">=19"
 cryptography = ">=42.0.4"
 
@@ -22,11 +22,6 @@ nose = ">=1.3.7"
 kappa-maki = ">=1.0.1"
 pylint = "^3.1.0"
 
-[tool.pylint.'MESSAGES CONTROL']
-# Adding the pydantic package to pylint's whitelist as it raises linting errors. 
-# See issue: https://github.com/pydantic/pydantic/issues/1961
-extension-pkg-whitelist = "pydantic"
-
 [build-system]
 requires = ["poetry-core>=1.7.0"]
 build-backend = "poetry.core.masonry.api"
diff --git a/...le-foundation-transform-core-python/src/data_transform_core/mediator/mediation_objects.py b/...le-foundation-transform-core-python/src/data_transform_core/mediator/mediation_objects.py
@@ -49,7 +49,7 @@ class MediationConfiguration(MediationContext):
     """
 
     className: str
-    properties: Optional[List[MediationProperty]]
+    properties: Optional[List[MediationProperty]] = None
 
     def getShortClassName(self) -> str:
         """

diff --git a/...da-models/test-machine-learning-model/aissemble-machine-learning-inference/pyproject.toml b/...da-models/test-machine-learning-model/aissemble-machine-learning-inference/pyproject.toml
@@ -36,11 +36,6 @@ grpcio-testing = "^1.50.0"
 kappa-maki = ">=1.0.1"
 pylint = "^3.1.0"
 
-[tool.pylint.'MESSAGES CONTROL']
-# Adding the pydantic package to pylint's whitelist as it raises linting errors. 
-# See issue: https://github.com/pydantic/pydantic/issues/1961
-extension-pkg-whitelist = "pydantic"
-
 [tool.pylint.MASTER]
 ignore-patterns = '.*pb2[\S]*.py'