diff --git a/DRAFT_RELEASE_NOTES.md b/DRAFT_RELEASE_NOTES.md index 628a7b2e8..204c29f3f 100644 --- a/DRAFT_RELEASE_NOTES.md +++ b/DRAFT_RELEASE_NOTES.md @@ -37,6 +37,7 @@ intervention. The helm charts referencing these containers have already been upd - `aissemble-airflow` - `aissemble-kafka` - `aissemble-mlflow` +- Upgraded Pydantic v1.10.x to 2.8.x to incorporate performance improvements and incorporate the availability of new features for future. See [here](https://docs.pydantic.dev/latest/migration/) for the guide on how to migrate pydantic V1 into V2. # Known Issues - There is currently a bug with the Sagemaker training Docker image generated by the `aissemble-sagemaker-training-docker` `Fermenter` profile. The installation of the `logistic-training` module's `requirements.txt` fails, due to an unresolvable set of dependencies. diff --git a/extensions/aissemble-extensions-model-training-api-sagemaker/pyproject.toml b/extensions/aissemble-extensions-model-training-api-sagemaker/pyproject.toml index 2792ad11d..4c90db2f6 100644 --- a/extensions/aissemble-extensions-model-training-api-sagemaker/pyproject.toml +++ b/extensions/aissemble-extensions-model-training-api-sagemaker/pyproject.toml @@ -12,7 +12,7 @@ packages = [ python = ">=3.11.4, <3.12" fastapi = ">=0.95.0" uvicorn = {version = "^0.18.0", extras = ["standard"]} -pydantic = ">=1.8.0,<2.0.0" +pydantic = ">=2.8.0" kubernetes = ">=26.1.0" urllib3 = "^1.26.18" krausening = ">=19" @@ -27,11 +27,6 @@ setuptools = "^69.0.3" kappa-maki = ">=1.0.1" pylint = "^3.1.0" -[tool.pylint.'MESSAGES CONTROL'] -# Adding the pydantic package to pylint's whitelist as it raises linting errors. -# See issue: https://github.com/pydantic/pydantic/issues/1961 -extension-pkg-whitelist = "pydantic" - [build-system] requires = ["poetry-core>=1.7.0"] build-backend = "poetry.core.masonry.api" diff --git a/extensions/extensions-transform/aissemble-extensions-transform-spark-python/pyproject.toml b/extensions/extensions-transform/aissemble-extensions-transform-spark-python/pyproject.toml index 77ef8407c..5badc5dc9 100644 --- a/extensions/extensions-transform/aissemble-extensions-transform-spark-python/pyproject.toml +++ b/extensions/extensions-transform/aissemble-extensions-transform-spark-python/pyproject.toml @@ -11,7 +11,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.11.4, <4" krausening = ">=19" -pydantic = ">=1.8.0,<2.0.0" +pydantic = ">=2.8.0" pyspark = "3.4.0" cryptography = ">=42.0.4" urllib3 = "^1.26.18" diff --git a/foundation/aissemble-foundation-core-python/pyproject.toml b/foundation/aissemble-foundation-core-python/pyproject.toml index f55d92853..1c1d7eda9 100644 --- a/foundation/aissemble-foundation-core-python/pyproject.toml +++ b/foundation/aissemble-foundation-core-python/pyproject.toml @@ -16,7 +16,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.11.4, <4" -pydantic = ">=1.8.0,<2.0.0" +pydantic = ">=2.8.0" krausening = ">=19" cryptography = ">=42.0.4" pyjwt = ">=2.3.0" @@ -36,11 +36,6 @@ nose = ">=1.3.7" kappa-maki = ">=1.0.1" pylint = "^3.1.0" -[tool.pylint.'MESSAGES CONTROL'] -# Adding the pydantic package to pylint's whitelist as it raises linting errors. -# See issue: https://github.com/pydantic/pydantic/issues/1961 -extension-pkg-whitelist = "pydantic" - [build-system] requires = ["poetry-core>=1.7.0"] build-backend = "poetry.core.masonry.api" diff --git a/foundation/aissemble-foundation-core-python/src/policy_manager/policy/json/policy_input.py b/foundation/aissemble-foundation-core-python/src/policy_manager/policy/json/policy_input.py index 28e387a37..577f6fb04 100644 --- a/foundation/aissemble-foundation-core-python/src/policy_manager/policy/json/policy_input.py +++ b/foundation/aissemble-foundation-core-python/src/policy_manager/policy/json/policy_input.py @@ -28,13 +28,13 @@ class PolicyRuleInput(BaseModel): """ The configuration used for the rule. """ - configurations: Optional[Dict[str, Any]] + configurations: Optional[Dict[str, Any]] = None """ Any configurations for the target set of data that is needed by this rule. """ - targetConfigurations: Optional[Dict[str, Any]] + targetConfigurations: Optional[Dict[str, Any]] = None class PolicyInput(BaseModel): @@ -54,17 +54,17 @@ class PolicyInput(BaseModel): """ The description of the policy. """ - description: Optional[str] + description: Optional[str] = None """ The targets this policy will be invoked on. """ - targets: Optional[List[Target]] + targets: Optional[List[Target]] = None """ The optional configuration for whether alerts should be sent or not. """ - shouldSendAlert: Optional[AlertOptions] + shouldSendAlert: Optional[AlertOptions] = None """ The rules for this policy. @@ -76,14 +76,14 @@ class PolicyInput(BaseModel): a List of Target objects of a single Target attribute 'target'. This attribute is replaced by `targets`. """ - target: Optional[Target] + target: Optional[Target] = None def getAnyTargets(self) -> List[Target]: """ Used to check both target attributes to contain backwards compatibility with policies still using deprecated 'target' """ if self.target is not None: - PolicyInput.__logger.warn( + self.__logger.warn( "Detected use of deprecated Json Property 'target'. Existing " + "values should be moved to the new Json Property 'targets'." ) diff --git a/foundation/aissemble-foundation-core-python/src/policy_manager/policy/policy.py b/foundation/aissemble-foundation-core-python/src/policy_manager/policy/policy.py index cee9a2d59..ccd17e8f6 100644 --- a/foundation/aissemble-foundation-core-python/src/policy_manager/policy/policy.py +++ b/foundation/aissemble-foundation-core-python/src/policy_manager/policy/policy.py @@ -7,7 +7,7 @@ # This software package is licensed under the Booz Allen Public License. All Rights Reserved. # #L% ### -from pydantic import BaseModel +from pydantic import ConfigDict, BaseModel from enum import Enum from typing import Any, List, Dict, Optional from krausening.logging import LogManager @@ -29,8 +29,8 @@ class Target(BaseModel): Target contains the target information for the policy. """ - retrieve_url: Optional[str] - type: Optional[str] + retrieve_url: Optional[str] = None + type: Optional[str] = None class ConfiguredTarget(Target): @@ -53,7 +53,7 @@ class ConfiguredRule(BaseModel): __logger = LogManager.get_instance().get_logger("ConfiguredRule") __deprecated_set_methods = {"target": "set_deprecated_targetConfigurations"} className: str - configurations: Optional[Dict[str, Any]] + configurations: Optional[Dict[str, Any]] = None configuredTargets: Optional[List[ConfiguredTarget]] = [] @property @@ -103,7 +103,7 @@ class Policy(BaseModel): __deprecated_set_methods = {"target": "set_deprecated_target"} alertOptions: AlertOptions = AlertOptions.ON_DETECTION identifier: str - description: Optional[str] + description: Optional[str] = None targets: Optional[List[Target]] = [] rules: List[ConfiguredRule] = [] @@ -132,9 +132,7 @@ def set_deprecated_target(self, new_value: Target): ) self.targets = [new_value] - # Pydantic model config to allow policy subclasses to contain additional fields of any type - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) # Links Policy 'target' attribute to 'set_deprecated_target()' method to support # people still assigning values to the old attribute. diff --git a/foundation/aissemble-foundation-core-python/src/policy_manager/policy/result/policy_invocation_result.py b/foundation/aissemble-foundation-core-python/src/policy_manager/policy/result/policy_invocation_result.py index fb2744ce0..46097b2b8 100644 --- a/foundation/aissemble-foundation-core-python/src/policy_manager/policy/result/policy_invocation_result.py +++ b/foundation/aissemble-foundation-core-python/src/policy_manager/policy/result/policy_invocation_result.py @@ -17,5 +17,5 @@ class PolicyInvocationResult(BaseModel): """ policyName: str - policyDescription: Optional[str] + policyDescription: Optional[str] = None timestamp: str diff --git a/foundation/aissemble-foundation-model-training-api/pyproject.toml b/foundation/aissemble-foundation-model-training-api/pyproject.toml index 9110e62c8..1037bc355 100644 --- a/foundation/aissemble-foundation-model-training-api/pyproject.toml +++ b/foundation/aissemble-foundation-model-training-api/pyproject.toml @@ -13,7 +13,7 @@ aissemble-foundation-messaging-python = {path = "../foundation-messaging/foundat python = ">=3.11.4, <3.12" fastapi = ">=0.95.0" uvicorn = {version = "^0.18.0", extras = ["standard"]} -pydantic = ">=1.8.0,<2.0.0" +pydantic = ">=2.8.0" kubernetes = ">=26.1.0" urllib3 = "^1.26.18" krausening = ">=19" diff --git a/foundation/aissemble-foundation-versioning-service/pyproject.toml b/foundation/aissemble-foundation-versioning-service/pyproject.toml index d939f8fe6..34bda9308 100644 --- a/foundation/aissemble-foundation-versioning-service/pyproject.toml +++ b/foundation/aissemble-foundation-versioning-service/pyproject.toml @@ -17,7 +17,7 @@ python = ">=3.11.4, <3.12" mlflow = "^2.3.1" fastapi = ">=0.95.0" uvicorn = {version = "^0.18.0", extras = ["standard"]} -pydantic = ">=1.8.0,<2.0.0" +pydantic = ">=2.8.0" starlette = ">=0.25.0" urllib3 = "^1.26.18" @@ -32,11 +32,6 @@ setuptools = "^69.0.3" kappa-maki = ">=1.0.1" pylint = "^3.1.0" -[tool.pylint.'MESSAGES CONTROL'] -# Adding the pydantic package to pylint's whitelist as it raises linting errors. -# See issue: https://github.com/pydantic/pydantic/issues/1961 -extension-pkg-whitelist = "pydantic" - [build-system] requires = ["poetry-core>=1.7.0"] build-backend = "poetry.core.masonry.api" diff --git a/foundation/aissemble-foundation-versioning-service/src/model_versioning/versioning_api.py b/foundation/aissemble-foundation-versioning-service/src/model_versioning/versioning_api.py index f786974fc..65425ee04 100644 --- a/foundation/aissemble-foundation-versioning-service/src/model_versioning/versioning_api.py +++ b/foundation/aissemble-foundation-versioning-service/src/model_versioning/versioning_api.py @@ -25,7 +25,7 @@ class VersioningResponse(BaseModel): success: bool message: str - version: Optional[str] + version: Optional[str] = None class JWTBearer(HTTPBearer): diff --git a/foundation/foundation-drift-detection/aissemble-foundation-drift-detection-client-python/pyproject.toml b/foundation/foundation-drift-detection/aissemble-foundation-drift-detection-client-python/pyproject.toml index d5fb40643..90743c2f5 100644 --- a/foundation/foundation-drift-detection/aissemble-foundation-drift-detection-client-python/pyproject.toml +++ b/foundation/foundation-drift-detection/aissemble-foundation-drift-detection-client-python/pyproject.toml @@ -11,7 +11,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.11.4, <4" -pydantic = ">=1.8.0,<2.0.0" +pydantic = ">=2.8.0" requests = ">=2.32.2" krausening = ">=19" cryptography = ">=42.0.4" @@ -23,11 +23,6 @@ behave = ">=1.2.6" kappa-maki = ">=1.0.1" pylint = "^3.1.0" -[tool.pylint.'MESSAGES CONTROL'] -# Adding the pydantic package to pylint's whitelist as it raises linting errors. -# See issue: https://github.com/pydantic/pydantic/issues/1961 -extension-pkg-whitelist = "pydantic" - [build-system] requires = ["poetry-core>=1.7.0"] build-backend = "poetry.core.masonry.api" diff --git a/foundation/foundation-drift-detection/aissemble-foundation-drift-detection-client-python/src/drift/detection/rest/client/drift_rest_client.py b/foundation/foundation-drift-detection/aissemble-foundation-drift-detection-client-python/src/drift/detection/rest/client/drift_rest_client.py index ff23054b8..2ce7a7a8a 100644 --- a/foundation/foundation-drift-detection/aissemble-foundation-drift-detection-client-python/src/drift/detection/rest/client/drift_rest_client.py +++ b/foundation/foundation-drift-detection/aissemble-foundation-drift-detection-client-python/src/drift/detection/rest/client/drift_rest_client.py @@ -26,7 +26,7 @@ class DriftVariable(DriftData): Represents a single drift variable """ - type = "single" + type: str = "single" value: float @@ -35,7 +35,7 @@ class DriftVariables(DriftData): Represents multiple drift variables """ - type = "multiple" + type: str = "multiple" variables: List[DriftVariable] = [] @@ -44,8 +44,8 @@ class DriftDataInput(BaseModel): Represents the data needed. """ - input: Optional[DriftData] - control: Optional[DriftData] + input: Optional[DriftData] = None + control: Optional[DriftData] = None class DriftDetectionResult(BaseModel): @@ -67,11 +67,9 @@ def invoke_drift(self, policy_identifier, input, control): drift_data_input = DriftDataInput(input=input, control=control) query_params = {"policyIdentifier": policy_identifier} data_headers = {"Content-type": "application/json"} - # uncomment below for debugging purposes - # print('JSON being sent to drift invocation service: ' + drift_data_input.json()) return requests.post( hostname + "/invoke-drift", params=query_params, - data=drift_data_input.json(), + data=drift_data_input.model_dump_json(serialize_as_any=True), headers=data_headers, ) diff --git a/foundation/foundation-encryption/aissemble-foundation-encryption-policy-python/src/aissemble_encrypt_policy/data_encryption_policy.py b/foundation/foundation-encryption/aissemble-foundation-encryption-policy-python/src/aissemble_encrypt_policy/data_encryption_policy.py index 29732d352..4dc54026a 100644 --- a/foundation/foundation-encryption/aissemble-foundation-encryption-policy-python/src/aissemble_encrypt_policy/data_encryption_policy.py +++ b/foundation/foundation-encryption/aissemble-foundation-encryption-policy-python/src/aissemble_encrypt_policy/data_encryption_policy.py @@ -17,6 +17,6 @@ class DataEncryptionPolicy(Policy): configurations that are needed for data encryption. """ - encryptPhase: Optional[str] + encryptPhase: Optional[str] = None encryptFields: List[str] = [] - encryptAlgorithm: Optional[str] + encryptAlgorithm: Optional[str] = None diff --git a/foundation/foundation-encryption/aissemble-foundation-encryption-policy-python/src/aissemble_encrypt_policy/encrypt_input/data_encryption_policy_input.py b/foundation/foundation-encryption/aissemble-foundation-encryption-policy-python/src/aissemble_encrypt_policy/encrypt_input/data_encryption_policy_input.py index ab94e360f..319959028 100644 --- a/foundation/foundation-encryption/aissemble-foundation-encryption-policy-python/src/aissemble_encrypt_policy/encrypt_input/data_encryption_policy_input.py +++ b/foundation/foundation-encryption/aissemble-foundation-encryption-policy-python/src/aissemble_encrypt_policy/encrypt_input/data_encryption_policy_input.py @@ -17,6 +17,6 @@ class DataEncryptionPolicyInput(PolicyInput): configurations that are needed for data encryption. """ - encryptPhase: Optional[str] + encryptPhase: Optional[str] = None encryptFields: List[str] = [] - encryptAlgorithm: Optional[str] + encryptAlgorithm: Optional[str] = None diff --git a/foundation/foundation-mda/src/main/resources/templates/inference/inference.api.grpc.base.py.vm b/foundation/foundation-mda/src/main/resources/templates/inference/inference.api.grpc.base.py.vm index 9d4493b36..5f43fa352 100644 --- a/foundation/foundation-mda/src/main/resources/templates/inference/inference.api.grpc.base.py.vm +++ b/foundation/foundation-mda/src/main/resources/templates/inference/inference.api.grpc.base.py.vm @@ -1,5 +1,5 @@ from google.protobuf import json_format -from pydantic.error_wrappers import ValidationError +from pydantic import ValidationError from grpc import StatusCode import logging diff --git a/foundation/foundation-mda/src/main/resources/templates/inference/pyproject.toml.vm b/foundation/foundation-mda/src/main/resources/templates/inference/pyproject.toml.vm index 6e3ba2d08..1654cdcad 100644 --- a/foundation/foundation-mda/src/main/resources/templates/inference/pyproject.toml.vm +++ b/foundation/foundation-mda/src/main/resources/templates/inference/pyproject.toml.vm @@ -35,11 +35,6 @@ grpcio-tools = "^1.50.0" grpcio-testing = "^1.50.0" pylint = "^3.1.0" -[tool.pylint.'MESSAGES CONTROL'] -# Adding the pydantic package to pylint's whitelist as it raises linting errors. -# See issue: https://github.com/pydantic/pydantic/issues/1961 -extension-pkg-whitelist = "pydantic" - [tool.pylint.MASTER] # Ignores protobuf generated files that fail linting ignore-patterns = '.*pb2[\S]*.py' diff --git a/foundation/foundation-transform/aissemble-foundation-transform-core-python/pyproject.toml b/foundation/foundation-transform/aissemble-foundation-transform-core-python/pyproject.toml index 512530085..46a2a9332 100644 --- a/foundation/foundation-transform/aissemble-foundation-transform-core-python/pyproject.toml +++ b/foundation/foundation-transform/aissemble-foundation-transform-core-python/pyproject.toml @@ -11,7 +11,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.11.4, <4" aissemble-foundation-core-python = {path = "../../aissemble-foundation-core-python", develop = true} -pydantic = ">=1.8.0,<2.0.0" +pydantic = ">=2.8.0" krausening = ">=19" cryptography = ">=42.0.4" @@ -22,11 +22,6 @@ nose = ">=1.3.7" kappa-maki = ">=1.0.1" pylint = "^3.1.0" -[tool.pylint.'MESSAGES CONTROL'] -# Adding the pydantic package to pylint's whitelist as it raises linting errors. -# See issue: https://github.com/pydantic/pydantic/issues/1961 -extension-pkg-whitelist = "pydantic" - [build-system] requires = ["poetry-core>=1.7.0"] build-backend = "poetry.core.masonry.api" diff --git a/foundation/foundation-transform/aissemble-foundation-transform-core-python/src/data_transform_core/mediator/mediation_objects.py b/foundation/foundation-transform/aissemble-foundation-transform-core-python/src/data_transform_core/mediator/mediation_objects.py index 2a4dac00b..8028c5d46 100644 --- a/foundation/foundation-transform/aissemble-foundation-transform-core-python/src/data_transform_core/mediator/mediation_objects.py +++ b/foundation/foundation-transform/aissemble-foundation-transform-core-python/src/data_transform_core/mediator/mediation_objects.py @@ -49,7 +49,7 @@ class MediationConfiguration(MediationContext): """ className: str - properties: Optional[List[MediationProperty]] + properties: Optional[List[MediationProperty]] = None def getShortClassName(self) -> str: """ diff --git a/test/test-mda-models/test-machine-learning-model/aissemble-machine-learning-inference/pyproject.toml b/test/test-mda-models/test-machine-learning-model/aissemble-machine-learning-inference/pyproject.toml index feddd2b12..48dd7fc74 100644 --- a/test/test-mda-models/test-machine-learning-model/aissemble-machine-learning-inference/pyproject.toml +++ b/test/test-mda-models/test-machine-learning-model/aissemble-machine-learning-inference/pyproject.toml @@ -36,11 +36,6 @@ grpcio-testing = "^1.50.0" kappa-maki = ">=1.0.1" pylint = "^3.1.0" -[tool.pylint.'MESSAGES CONTROL'] -# Adding the pydantic package to pylint's whitelist as it raises linting errors. -# See issue: https://github.com/pydantic/pydantic/issues/1961 -extension-pkg-whitelist = "pydantic" - [tool.pylint.MASTER] ignore-patterns = '.*pb2[\S]*.py'