From a44d783ca481ca6cbde0f16dfce6f5bdca516ada Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Fri, 29 Nov 2024 17:40:04 +0100 Subject: [PATCH 1/6] refactor(backend): Avoid double specifying provider and cred types on `credentials` inputs - Add `credentials_provider` and `credentials_types` to field schema from `CredentialsMetaInput` instead of `CredentialsField` - Remove `provider` and `supported_credential_types` params from `CredentialsField` and all its usages - Add `credentials` schema validation logic to `CredentialsMetaInput`, which is called from `BlockSchema.__pydantic_init_subclass__` --- .../blocks/ai_image_generator_block.py | 10 +-- .../backend/blocks/ai_music_generator.py | 12 ++- .../blocks/ai_shortform_video_block.py | 12 ++- .../backend/backend/blocks/discord.py | 6 +- .../backend/backend/blocks/github/_auth.py | 4 - .../backend/backend/blocks/google/_auth.py | 2 - .../backend/backend/blocks/google_maps.py | 6 +- .../backend/backend/blocks/ideogram.py | 11 +-- .../backend/backend/blocks/jina/_auth.py | 16 ---- .../backend/backend/blocks/llm.py | 2 - .../backend/backend/blocks/medium.py | 10 +-- .../backend/backend/blocks/pinecone.py | 7 +- .../backend/blocks/replicate_flux_advanced.py | 12 ++- .../backend/backend/blocks/search.py | 2 - .../backend/backend/blocks/talking_head.py | 12 ++- .../backend/blocks/text_to_speech_block.py | 2 - .../backend/backend/data/block.py | 5 ++ .../backend/backend/data/model.py | 79 +++++++++++++++---- 18 files changed, 102 insertions(+), 108 deletions(-) diff --git a/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py b/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py index ee94f016aa55..74b820fa5252 100644 --- a/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py +++ b/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py @@ -101,12 +101,10 @@ class ImageGenModel(str, Enum): class AIImageGeneratorBlock(Block): class Input(BlockSchema): - credentials: CredentialsMetaInput[Literal["replicate"], Literal["api_key"]] = ( - CredentialsField( - provider="replicate", - supported_credential_types={"api_key"}, - description="Enter your Replicate API key to access the image generation API. You can obtain an API key from https://replicate.com/account/api-tokens.", - ) + credentials: CredentialsMetaInput[ + Literal["replicate"], Literal["api_key"] + ] = CredentialsField( + description="Enter your Replicate API key to access the image generation API. You can obtain an API key from https://replicate.com/account/api-tokens.", ) prompt: str = SchemaField( description="Text prompt for image generation", diff --git a/autogpt_platform/backend/backend/blocks/ai_music_generator.py b/autogpt_platform/backend/backend/blocks/ai_music_generator.py index f70d43ce370e..610eba85dd14 100644 --- a/autogpt_platform/backend/backend/blocks/ai_music_generator.py +++ b/autogpt_platform/backend/backend/blocks/ai_music_generator.py @@ -54,13 +54,11 @@ class NormalizationStrategy(str, Enum): class AIMusicGeneratorBlock(Block): class Input(BlockSchema): - credentials: CredentialsMetaInput[Literal["replicate"], Literal["api_key"]] = ( - CredentialsField( - provider="replicate", - supported_credential_types={"api_key"}, - description="The Replicate integration can be used with " - "any API key with sufficient permissions for the blocks it is used on.", - ) + credentials: CredentialsMetaInput[ + Literal["replicate"], Literal["api_key"] + ] = CredentialsField( + description="The Replicate integration can be used with " + "any API key with sufficient permissions for the blocks it is used on.", ) prompt: str = SchemaField( description="A description of the music you want to generate", diff --git a/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py b/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py index 08023b877118..a85413fb3daf 100644 --- a/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py +++ b/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py @@ -140,13 +140,11 @@ class VisualMediaType(str, Enum): class AIShortformVideoCreatorBlock(Block): class Input(BlockSchema): - credentials: CredentialsMetaInput[Literal["revid"], Literal["api_key"]] = ( - CredentialsField( - provider="revid", - supported_credential_types={"api_key"}, - description="The revid.ai integration can be used with " - "any API key with sufficient permissions for the blocks it is used on.", - ) + credentials: CredentialsMetaInput[ + Literal["revid"], Literal["api_key"] + ] = CredentialsField( + description="The revid.ai integration can be used with " + "any API key with sufficient permissions for the blocks it is used on.", ) script: str = SchemaField( description="""1. Use short and punctuated sentences\n\n2. Use linebreaks to create a new clip\n\n3. Text outside of brackets is spoken by the AI, and [text between brackets] will be used to guide the visual generation. For example, [close-up of a cat] will show a close-up of a cat.""", diff --git a/autogpt_platform/backend/backend/blocks/discord.py b/autogpt_platform/backend/backend/blocks/discord.py index c638e402508b..530906c2bf6c 100644 --- a/autogpt_platform/backend/backend/blocks/discord.py +++ b/autogpt_platform/backend/backend/blocks/discord.py @@ -17,11 +17,7 @@ def DiscordCredentialsField() -> DiscordCredentials: - return CredentialsField( - description="Discord bot token", - provider="discord", - supported_credential_types={"api_key"}, - ) + return CredentialsField(description="Discord bot token") TEST_CREDENTIALS = APIKeyCredentials( diff --git a/autogpt_platform/backend/backend/blocks/github/_auth.py b/autogpt_platform/backend/backend/blocks/github/_auth.py index 72aa8f648015..cbdb644d630d 100644 --- a/autogpt_platform/backend/backend/blocks/github/_auth.py +++ b/autogpt_platform/backend/backend/blocks/github/_auth.py @@ -30,10 +30,6 @@ def GithubCredentialsField(scope: str) -> GithubCredentialsInput: scope: The authorization scope needed for the block to work. ([list of available scopes](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes)) """ # noqa return CredentialsField( - provider="github", - supported_credential_types=( - {"api_key", "oauth2"} if GITHUB_OAUTH_IS_CONFIGURED else {"api_key"} - ), required_scopes={scope}, description="The GitHub integration can be used with OAuth, " "or any API key with sufficient permissions for the blocks it is used on.", diff --git a/autogpt_platform/backend/backend/blocks/google/_auth.py b/autogpt_platform/backend/backend/blocks/google/_auth.py index ccae2e462244..0336052315ba 100644 --- a/autogpt_platform/backend/backend/blocks/google/_auth.py +++ b/autogpt_platform/backend/backend/blocks/google/_auth.py @@ -23,8 +23,6 @@ def GoogleCredentialsField(scopes: list[str]) -> GoogleCredentialsInput: scopes: The authorization scopes needed for the block to work. """ return CredentialsField( - provider="google", - supported_credential_types={"oauth2"}, required_scopes=set(scopes), description="The Google integration requires OAuth2 authentication.", ) diff --git a/autogpt_platform/backend/backend/blocks/google_maps.py b/autogpt_platform/backend/backend/blocks/google_maps.py index d211fe8ff3f0..0db1a57c4bd0 100644 --- a/autogpt_platform/backend/backend/blocks/google_maps.py +++ b/autogpt_platform/backend/backend/blocks/google_maps.py @@ -39,11 +39,7 @@ class GoogleMapsSearchBlock(Block): class Input(BlockSchema): credentials: CredentialsMetaInput[ Literal["google_maps"], Literal["api_key"] - ] = CredentialsField( - provider="google_maps", - supported_credential_types={"api_key"}, - description="Google Maps API Key", - ) + ] = CredentialsField(description="Google Maps API Key") query: str = SchemaField( description="Search query for local businesses", placeholder="e.g., 'restaurants in New York'", diff --git a/autogpt_platform/backend/backend/blocks/ideogram.py b/autogpt_platform/backend/backend/blocks/ideogram.py index b6a21e7acec2..4395e0a0ce3e 100644 --- a/autogpt_platform/backend/backend/blocks/ideogram.py +++ b/autogpt_platform/backend/backend/blocks/ideogram.py @@ -83,13 +83,10 @@ class UpscaleOption(str, Enum): class IdeogramModelBlock(Block): class Input(BlockSchema): - - credentials: CredentialsMetaInput[Literal["ideogram"], Literal["api_key"]] = ( - CredentialsField( - provider="ideogram", - supported_credential_types={"api_key"}, - description="The Ideogram integration can be used with any API key with sufficient permissions for the blocks it is used on.", - ) + credentials: CredentialsMetaInput[ + Literal["ideogram"], Literal["api_key"] + ] = CredentialsField( + description="The Ideogram integration can be used with any API key with sufficient permissions for the blocks it is used on.", ) prompt: str = SchemaField( description="Text prompt for image generation", diff --git a/autogpt_platform/backend/backend/blocks/jina/_auth.py b/autogpt_platform/backend/backend/blocks/jina/_auth.py index 2bffeecce754..de113246d58a 100644 --- a/autogpt_platform/backend/backend/blocks/jina/_auth.py +++ b/autogpt_platform/backend/backend/blocks/jina/_auth.py @@ -10,20 +10,6 @@ Literal["api_key"], ] -TEST_CREDENTIALS = APIKeyCredentials( - id="01234567-89ab-cdef-0123-456789abcdef", - provider="jina", - api_key=SecretStr("mock-jina-api-key"), - title="Mock Jina API key", - expires_at=None, -) -TEST_CREDENTIALS_INPUT = { - "provider": TEST_CREDENTIALS.provider, - "id": TEST_CREDENTIALS.id, - "type": TEST_CREDENTIALS.type, - "title": TEST_CREDENTIALS.type, -} - def JinaCredentialsField() -> JinaCredentialsInput: """ @@ -31,8 +17,6 @@ def JinaCredentialsField() -> JinaCredentialsInput: """ return CredentialsField( - provider="jina", - supported_credential_types={"api_key"}, description="The Jina integration can be used with an API Key.", ) diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.py index 50a9bfb3c626..fc900879c400 100644 --- a/autogpt_platform/backend/backend/blocks/llm.py +++ b/autogpt_platform/backend/backend/blocks/llm.py @@ -48,8 +48,6 @@ def AICredentialsField() -> AICredentials: return CredentialsField( description="API key for the LLM provider.", - provider=["anthropic", "groq", "openai", "ollama", "open_router"], - supported_credential_types={"api_key"}, discriminator="model", discriminator_mapping={ model.value: model.metadata.provider for model in LlmModel diff --git a/autogpt_platform/backend/backend/blocks/medium.py b/autogpt_platform/backend/backend/blocks/medium.py index da8c367c7860..a712ff8cbd34 100644 --- a/autogpt_platform/backend/backend/blocks/medium.py +++ b/autogpt_platform/backend/backend/blocks/medium.py @@ -77,12 +77,10 @@ class Input(BlockSchema): description="Whether to notify followers that the user has published", placeholder="False", ) - credentials: CredentialsMetaInput[Literal["medium"], Literal["api_key"]] = ( - CredentialsField( - provider="medium", - supported_credential_types={"api_key"}, - description="The Medium integration can be used with any API key with sufficient permissions for the blocks it is used on.", - ) + credentials: CredentialsMetaInput[ + Literal["medium"], Literal["api_key"] + ] = CredentialsField( + description="The Medium integration can be used with any API key with sufficient permissions for the blocks it is used on.", ) class Output(BlockSchema): diff --git a/autogpt_platform/backend/backend/blocks/pinecone.py b/autogpt_platform/backend/backend/blocks/pinecone.py index a62c1fa77739..28f36598b4ab 100644 --- a/autogpt_platform/backend/backend/blocks/pinecone.py +++ b/autogpt_platform/backend/backend/blocks/pinecone.py @@ -19,13 +19,8 @@ def PineconeCredentialsField() -> PineconeCredentialsInput: - """ - Creates a Pinecone credentials input on a block. - - """ + """Creates a Pinecone credentials input on a block.""" return CredentialsField( - provider="pinecone", - supported_credential_types={"api_key"}, description="The Pinecone integration can be used with an API Key.", ) diff --git a/autogpt_platform/backend/backend/blocks/replicate_flux_advanced.py b/autogpt_platform/backend/backend/blocks/replicate_flux_advanced.py index b346c87a38a0..11b36751b68a 100644 --- a/autogpt_platform/backend/backend/blocks/replicate_flux_advanced.py +++ b/autogpt_platform/backend/backend/blocks/replicate_flux_advanced.py @@ -54,13 +54,11 @@ class ImageType(str, Enum): class ReplicateFluxAdvancedModelBlock(Block): class Input(BlockSchema): - credentials: CredentialsMetaInput[Literal["replicate"], Literal["api_key"]] = ( - CredentialsField( - provider="replicate", - supported_credential_types={"api_key"}, - description="The Replicate integration can be used with " - "any API key with sufficient permissions for the blocks it is used on.", - ) + credentials: CredentialsMetaInput[ + Literal["replicate"], Literal["api_key"] + ] = CredentialsField( + description="The Replicate integration can be used with " + "any API key with sufficient permissions for the blocks it is used on.", ) prompt: str = SchemaField( description="Text prompt for image generation", diff --git a/autogpt_platform/backend/backend/blocks/search.py b/autogpt_platform/backend/backend/blocks/search.py index f89232703aa6..fa5ef0ea70fc 100644 --- a/autogpt_platform/backend/backend/blocks/search.py +++ b/autogpt_platform/backend/backend/blocks/search.py @@ -67,8 +67,6 @@ class Input(BlockSchema): credentials: CredentialsMetaInput[ Literal["openweathermap"], Literal["api_key"] ] = CredentialsField( - provider="openweathermap", - supported_credential_types={"api_key"}, description="The OpenWeatherMap integration can be used with " "any API key with sufficient permissions for the blocks it is used on.", ) diff --git a/autogpt_platform/backend/backend/blocks/talking_head.py b/autogpt_platform/backend/backend/blocks/talking_head.py index 20aadcd214fe..94516d0696df 100644 --- a/autogpt_platform/backend/backend/blocks/talking_head.py +++ b/autogpt_platform/backend/backend/blocks/talking_head.py @@ -29,13 +29,11 @@ class CreateTalkingAvatarVideoBlock(Block): class Input(BlockSchema): - credentials: CredentialsMetaInput[Literal["d_id"], Literal["api_key"]] = ( - CredentialsField( - provider="d_id", - supported_credential_types={"api_key"}, - description="The D-ID integration can be used with " - "any API key with sufficient permissions for the blocks it is used on.", - ) + credentials: CredentialsMetaInput[ + Literal["d_id"], Literal["api_key"] + ] = CredentialsField( + description="The D-ID integration can be used with " + "any API key with sufficient permissions for the blocks it is used on.", ) script_input: str = SchemaField( description="The text input for the script", diff --git a/autogpt_platform/backend/backend/blocks/text_to_speech_block.py b/autogpt_platform/backend/backend/blocks/text_to_speech_block.py index b92cc9fa4468..12dc7dd7c832 100644 --- a/autogpt_platform/backend/backend/blocks/text_to_speech_block.py +++ b/autogpt_platform/backend/backend/blocks/text_to_speech_block.py @@ -40,8 +40,6 @@ class Input(BlockSchema): credentials: CredentialsMetaInput[ Literal["unreal_speech"], Literal["api_key"] ] = CredentialsField( - provider="unreal_speech", - supported_credential_types={"api_key"}, description="The Unreal Speech integration can be used with " "any API key with sufficient permissions for the blocks it is used on.", ) diff --git a/autogpt_platform/backend/backend/data/block.py b/autogpt_platform/backend/backend/data/block.py index e1035e84fe18..071129056627 100644 --- a/autogpt_platform/backend/backend/data/block.py +++ b/autogpt_platform/backend/backend/data/block.py @@ -176,6 +176,11 @@ def __pydantic_init_subclass__(cls, **kwargs): f"Field 'credentials' on {cls.__qualname__} " f"must be of type {CredentialsMetaInput.__name__}" ) + if credentials_field := cls.model_fields.get(CREDENTIALS_FIELD_NAME): + credentials_input_type = cast( + CredentialsMetaInput, credentials_field.annotation + ) + credentials_input_type.validate_credentials_field_schema(cls) BlockSchemaInputType = TypeVar("BlockSchemaInputType", bound=BlockSchema) diff --git a/autogpt_platform/backend/backend/data/model.py b/autogpt_platform/backend/backend/data/model.py index f8b6781e0ce1..ed23af677670 100644 --- a/autogpt_platform/backend/backend/data/model.py +++ b/autogpt_platform/backend/backend/data/model.py @@ -2,6 +2,7 @@ import logging from typing import ( + TYPE_CHECKING, Annotated, Any, Callable, @@ -11,19 +12,31 @@ Optional, TypedDict, TypeVar, + get_args, ) from uuid import uuid4 -from pydantic import BaseModel, Field, GetCoreSchemaHandler, SecretStr, field_serializer +from pydantic import ( + BaseModel, + ConfigDict, + Field, + GetCoreSchemaHandler, + SecretStr, + field_serializer, +) from pydantic_core import ( CoreSchema, PydanticUndefined, PydanticUndefinedType, + ValidationError, core_schema, ) from backend.util.settings import Secrets +if TYPE_CHECKING: + from backend.data.block import BlockSchema + T = TypeVar("T") logger = logging.getLogger(__name__) @@ -233,19 +246,51 @@ class CredentialsMetaInput(BaseModel, Generic[CP, CT]): provider: CP type: CT + @staticmethod + def _add_json_schema_extra(schema, cls: CredentialsMetaInput): + schema["credentials_provider"] = get_args( + cls.model_fields["provider"].annotation + ) + schema["credentials_types"] = get_args(cls.model_fields["type"].annotation) -class CredentialsFieldSchemaExtra(BaseModel, Generic[CP, CT]): + model_config = ConfigDict( + json_schema_extra=_add_json_schema_extra, # type: ignore + ) + + @classmethod + def validate_credentials_field_schema(cls, model: type["BlockSchema"]): + """Validates the schema of a `credentials` field""" + field_schema = model.jsonschema()["properties"][CREDENTIALS_FIELD_NAME] + try: + schema_extra = _CredentialsFieldSchemaExtra[CP, CT].model_validate( + field_schema + ) + except ValidationError as e: + if "Field required [type=missing" not in str(e): + raise + + raise TypeError( + "Field 'credentials' JSON schema lacks required extra items: " + f"{field_schema}" + ) from e + + if ( + len(schema_extra.credentials_provider) > 1 + and not schema_extra.credentials_provider + ): + raise TypeError("Multi-provider CredentialsField requires discriminator!") + + +class _CredentialsFieldSchemaExtra(BaseModel, Generic[CP, CT]): # TODO: move discrimination mechanism out of CredentialsField (frontend + backend) credentials_provider: list[CP] - credentials_scopes: Optional[list[str]] + credentials_scopes: Optional[list[str]] = None credentials_types: list[CT] discriminator: Optional[str] = None discriminator_mapping: Optional[dict[str, CP]] = None def CredentialsField( - provider: CP | list[CP], - supported_credential_types: set[CT], required_scopes: set[str] = set(), *, discriminator: Optional[str] = None, @@ -253,26 +298,26 @@ def CredentialsField( title: Optional[str] = None, description: Optional[str] = None, **kwargs, -) -> CredentialsMetaInput[CP, CT]: +) -> CredentialsMetaInput: """ `CredentialsField` must and can only be used on fields named `credentials`. This is enforced by the `BlockSchema` base class. """ - if not isinstance(provider, str) and len(provider) > 1 and not discriminator: - raise TypeError("Multi-provider CredentialsField requires discriminator!") - - field_schema_extra = CredentialsFieldSchemaExtra[CP, CT]( - credentials_provider=[provider] if isinstance(provider, str) else provider, - credentials_scopes=list(required_scopes) or None, # omit if empty - credentials_types=list(supported_credential_types), - discriminator=discriminator, - discriminator_mapping=discriminator_mapping, - ) + + field_schema_extra = { + k: v + for k, v in { + "credentials_scopes": list(required_scopes) or None, + "discriminator": discriminator, + "discriminator_mapping": discriminator_mapping, + }.items() + if v is not None + } return Field( title=title, description=description, - json_schema_extra=field_schema_extra.model_dump(exclude_none=True), + json_schema_extra=field_schema_extra, # validated on BlockSchema init **kwargs, ) From ec1c6f4f76bd898b05c995205c91d6d2ac45102b Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Wed, 20 Nov 2024 00:06:03 +0000 Subject: [PATCH 2/6] refactor(backend): Use `ProviderName` enum globally - Add missing providers to `ProviderName` - Replace string literals for provider names with `ProviderName.{PROVIDER}` on blocks and in `CredentialsMetaInput`, `BlockWebhookConfig`, `*OAuthHandler` - docs: Update instructions for blocks with authentication --- .../blocks/ai_image_generator_block.py | 3 +- .../backend/blocks/ai_music_generator.py | 3 +- .../blocks/ai_shortform_video_block.py | 3 +- .../backend/backend/blocks/discord.py | 5 +++- .../backend/backend/blocks/fal/_auth.py | 5 ++-- .../backend/backend/blocks/github/_auth.py | 3 +- .../backend/backend/blocks/google/_auth.py | 5 +++- .../backend/backend/blocks/google_maps.py | 3 +- .../backend/backend/blocks/hubspot/_auth.py | 5 ++-- .../backend/backend/blocks/ideogram.py | 3 +- .../backend/backend/blocks/jina/_auth.py | 3 +- .../backend/backend/blocks/llm.py | 10 ++++++- .../backend/backend/blocks/medium.py | 3 +- .../backend/backend/blocks/pinecone.py | 3 +- .../backend/blocks/replicate_flux_advanced.py | 3 +- .../backend/backend/blocks/search.py | 3 +- .../backend/backend/blocks/talking_head.py | 3 +- .../backend/blocks/text_to_speech_block.py | 3 +- .../backend/backend/data/model.py | 3 +- .../backend/integrations/creds_manager.py | 8 +++-- .../backend/integrations/oauth/__init__.py | 9 ++++-- .../backend/integrations/oauth/base.py | 7 +++-- .../backend/integrations/oauth/github.py | 3 +- .../backend/integrations/oauth/google.py | 3 +- .../backend/integrations/oauth/notion.py | 3 +- .../backend/backend/integrations/providers.py | 20 +++++++++++++ .../backend/server/integrations/router.py | 18 ++++++++---- docs/content/platform/new_blocks.md | 29 +++++++++++++------ 28 files changed, 126 insertions(+), 46 deletions(-) diff --git a/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py b/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py index 74b820fa5252..ebd79dda9ac9 100644 --- a/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py +++ b/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py @@ -12,6 +12,7 @@ CredentialsMetaInput, SchemaField, ) +from backend.integrations.providers import ProviderName class ImageSize(str, Enum): @@ -102,7 +103,7 @@ class ImageGenModel(str, Enum): class AIImageGeneratorBlock(Block): class Input(BlockSchema): credentials: CredentialsMetaInput[ - Literal["replicate"], Literal["api_key"] + Literal[ProviderName.REPLICATE], Literal["api_key"] ] = CredentialsField( description="Enter your Replicate API key to access the image generation API. You can obtain an API key from https://replicate.com/account/api-tokens.", ) diff --git a/autogpt_platform/backend/backend/blocks/ai_music_generator.py b/autogpt_platform/backend/backend/blocks/ai_music_generator.py index 610eba85dd14..708203510877 100644 --- a/autogpt_platform/backend/backend/blocks/ai_music_generator.py +++ b/autogpt_platform/backend/backend/blocks/ai_music_generator.py @@ -13,6 +13,7 @@ CredentialsMetaInput, SchemaField, ) +from backend.integrations.providers import ProviderName logger = logging.getLogger(__name__) @@ -55,7 +56,7 @@ class NormalizationStrategy(str, Enum): class AIMusicGeneratorBlock(Block): class Input(BlockSchema): credentials: CredentialsMetaInput[ - Literal["replicate"], Literal["api_key"] + Literal[ProviderName.REPLICATE], Literal["api_key"] ] = CredentialsField( description="The Replicate integration can be used with " "any API key with sufficient permissions for the blocks it is used on.", diff --git a/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py b/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py index a85413fb3daf..df2b3a27263c 100644 --- a/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py +++ b/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py @@ -12,6 +12,7 @@ CredentialsMetaInput, SchemaField, ) +from backend.integrations.providers import ProviderName from backend.util.request import requests TEST_CREDENTIALS = APIKeyCredentials( @@ -141,7 +142,7 @@ class VisualMediaType(str, Enum): class AIShortformVideoCreatorBlock(Block): class Input(BlockSchema): credentials: CredentialsMetaInput[ - Literal["revid"], Literal["api_key"] + Literal[ProviderName.REVID], Literal["api_key"] ] = CredentialsField( description="The revid.ai integration can be used with " "any API key with sufficient permissions for the blocks it is used on.", diff --git a/autogpt_platform/backend/backend/blocks/discord.py b/autogpt_platform/backend/backend/blocks/discord.py index 530906c2bf6c..08ba8af074cd 100644 --- a/autogpt_platform/backend/backend/blocks/discord.py +++ b/autogpt_platform/backend/backend/blocks/discord.py @@ -12,8 +12,11 @@ CredentialsMetaInput, SchemaField, ) +from backend.integrations.providers import ProviderName -DiscordCredentials = CredentialsMetaInput[Literal["discord"], Literal["api_key"]] +DiscordCredentials = CredentialsMetaInput[ + Literal[ProviderName.DISCORD], Literal["api_key"] +] def DiscordCredentialsField() -> DiscordCredentials: diff --git a/autogpt_platform/backend/backend/blocks/fal/_auth.py b/autogpt_platform/backend/backend/blocks/fal/_auth.py index 3271ee095028..5d02186e5797 100644 --- a/autogpt_platform/backend/backend/blocks/fal/_auth.py +++ b/autogpt_platform/backend/backend/blocks/fal/_auth.py @@ -3,10 +3,11 @@ from pydantic import SecretStr from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput +from backend.integrations.providers import ProviderName FalCredentials = APIKeyCredentials FalCredentialsInput = CredentialsMetaInput[ - Literal["fal"], + Literal[ProviderName.FAL], Literal["api_key"], ] @@ -30,7 +31,5 @@ def FalCredentialsField() -> FalCredentialsInput: Creates a FAL credentials input on a block. """ return CredentialsField( - provider="fal", - supported_credential_types={"api_key"}, description="The FAL integration can be used with an API Key.", ) diff --git a/autogpt_platform/backend/backend/blocks/github/_auth.py b/autogpt_platform/backend/backend/blocks/github/_auth.py index cbdb644d630d..df7eed90f701 100644 --- a/autogpt_platform/backend/backend/blocks/github/_auth.py +++ b/autogpt_platform/backend/backend/blocks/github/_auth.py @@ -8,6 +8,7 @@ CredentialsMetaInput, OAuth2Credentials, ) +from backend.integrations.providers import ProviderName from backend.util.settings import Secrets secrets = Secrets() @@ -17,7 +18,7 @@ GithubCredentials = APIKeyCredentials | OAuth2Credentials GithubCredentialsInput = CredentialsMetaInput[ - Literal["github"], + Literal[ProviderName.GITHUB], Literal["api_key", "oauth2"] if GITHUB_OAUTH_IS_CONFIGURED else Literal["api_key"], ] diff --git a/autogpt_platform/backend/backend/blocks/google/_auth.py b/autogpt_platform/backend/backend/blocks/google/_auth.py index 0336052315ba..2b364dbd4091 100644 --- a/autogpt_platform/backend/backend/blocks/google/_auth.py +++ b/autogpt_platform/backend/backend/blocks/google/_auth.py @@ -3,6 +3,7 @@ from pydantic import SecretStr from backend.data.model import CredentialsField, CredentialsMetaInput, OAuth2Credentials +from backend.integrations.providers import ProviderName from backend.util.settings import Secrets # --8<-- [start:GoogleOAuthIsConfigured] @@ -12,7 +13,9 @@ ) # --8<-- [end:GoogleOAuthIsConfigured] GoogleCredentials = OAuth2Credentials -GoogleCredentialsInput = CredentialsMetaInput[Literal["google"], Literal["oauth2"]] +GoogleCredentialsInput = CredentialsMetaInput[ + Literal[ProviderName.GOOGLE], Literal["oauth2"] +] def GoogleCredentialsField(scopes: list[str]) -> GoogleCredentialsInput: diff --git a/autogpt_platform/backend/backend/blocks/google_maps.py b/autogpt_platform/backend/backend/blocks/google_maps.py index 0db1a57c4bd0..9e7f79353123 100644 --- a/autogpt_platform/backend/backend/blocks/google_maps.py +++ b/autogpt_platform/backend/backend/blocks/google_maps.py @@ -10,6 +10,7 @@ CredentialsMetaInput, SchemaField, ) +from backend.integrations.providers import ProviderName TEST_CREDENTIALS = APIKeyCredentials( id="01234567-89ab-cdef-0123-456789abcdef", @@ -38,7 +39,7 @@ class Place(BaseModel): class GoogleMapsSearchBlock(Block): class Input(BlockSchema): credentials: CredentialsMetaInput[ - Literal["google_maps"], Literal["api_key"] + Literal[ProviderName.GOOGLE_MAPS], Literal["api_key"] ] = CredentialsField(description="Google Maps API Key") query: str = SchemaField( description="Search query for local businesses", diff --git a/autogpt_platform/backend/backend/blocks/hubspot/_auth.py b/autogpt_platform/backend/backend/blocks/hubspot/_auth.py index c32af8c38c01..b32456d5d5b2 100644 --- a/autogpt_platform/backend/backend/blocks/hubspot/_auth.py +++ b/autogpt_platform/backend/backend/blocks/hubspot/_auth.py @@ -3,10 +3,11 @@ from pydantic import SecretStr from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput +from backend.integrations.providers import ProviderName HubSpotCredentials = APIKeyCredentials HubSpotCredentialsInput = CredentialsMetaInput[ - Literal["hubspot"], + Literal[ProviderName.HUBSPOT], Literal["api_key"], ] @@ -14,8 +15,6 @@ def HubSpotCredentialsField() -> HubSpotCredentialsInput: """Creates a HubSpot credentials input on a block.""" return CredentialsField( - provider="hubspot", - supported_credential_types={"api_key"}, description="The HubSpot integration requires an API Key.", ) diff --git a/autogpt_platform/backend/backend/blocks/ideogram.py b/autogpt_platform/backend/backend/blocks/ideogram.py index 4395e0a0ce3e..82eb91238b40 100644 --- a/autogpt_platform/backend/backend/blocks/ideogram.py +++ b/autogpt_platform/backend/backend/blocks/ideogram.py @@ -11,6 +11,7 @@ CredentialsMetaInput, SchemaField, ) +from backend.integrations.providers import ProviderName from backend.util.request import requests TEST_CREDENTIALS = APIKeyCredentials( @@ -84,7 +85,7 @@ class UpscaleOption(str, Enum): class IdeogramModelBlock(Block): class Input(BlockSchema): credentials: CredentialsMetaInput[ - Literal["ideogram"], Literal["api_key"] + Literal[ProviderName.IDEOGRAM], Literal["api_key"] ] = CredentialsField( description="The Ideogram integration can be used with any API key with sufficient permissions for the blocks it is used on.", ) diff --git a/autogpt_platform/backend/backend/blocks/jina/_auth.py b/autogpt_platform/backend/backend/blocks/jina/_auth.py index de113246d58a..5bf0ddd5cf4c 100644 --- a/autogpt_platform/backend/backend/blocks/jina/_auth.py +++ b/autogpt_platform/backend/backend/blocks/jina/_auth.py @@ -3,10 +3,11 @@ from pydantic import SecretStr from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput +from backend.integrations.providers import ProviderName JinaCredentials = APIKeyCredentials JinaCredentialsInput = CredentialsMetaInput[ - Literal["jina"], + Literal[ProviderName.JINA], Literal["api_key"], ] diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.py index fc900879c400..2488cd469b32 100644 --- a/autogpt_platform/backend/backend/blocks/llm.py +++ b/autogpt_platform/backend/backend/blocks/llm.py @@ -7,6 +7,8 @@ from pydantic import SecretStr +from backend.integrations.providers import ProviderName + if TYPE_CHECKING: from enum import _EnumMemberT @@ -27,7 +29,13 @@ logger = logging.getLogger(__name__) -LLMProviderName = Literal["anthropic", "groq", "openai", "ollama", "open_router"] +LLMProviderName = Literal[ + ProviderName.ANTHROPIC, + ProviderName.GROQ, + ProviderName.OLLAMA, + ProviderName.OPENAI, + ProviderName.OPEN_ROUTER, +] AICredentials = CredentialsMetaInput[LLMProviderName, Literal["api_key"]] TEST_CREDENTIALS = APIKeyCredentials( diff --git a/autogpt_platform/backend/backend/blocks/medium.py b/autogpt_platform/backend/backend/blocks/medium.py index a712ff8cbd34..6d871b4caac5 100644 --- a/autogpt_platform/backend/backend/blocks/medium.py +++ b/autogpt_platform/backend/backend/blocks/medium.py @@ -12,6 +12,7 @@ SchemaField, SecretField, ) +from backend.integrations.providers import ProviderName from backend.util.request import requests TEST_CREDENTIALS = APIKeyCredentials( @@ -78,7 +79,7 @@ class Input(BlockSchema): placeholder="False", ) credentials: CredentialsMetaInput[ - Literal["medium"], Literal["api_key"] + Literal[ProviderName.MEDIUM], Literal["api_key"] ] = CredentialsField( description="The Medium integration can be used with any API key with sufficient permissions for the blocks it is used on.", ) diff --git a/autogpt_platform/backend/backend/blocks/pinecone.py b/autogpt_platform/backend/backend/blocks/pinecone.py index 28f36598b4ab..f1b79f44311c 100644 --- a/autogpt_platform/backend/backend/blocks/pinecone.py +++ b/autogpt_platform/backend/backend/blocks/pinecone.py @@ -10,10 +10,11 @@ CredentialsMetaInput, SchemaField, ) +from backend.integrations.providers import ProviderName PineconeCredentials = APIKeyCredentials PineconeCredentialsInput = CredentialsMetaInput[ - Literal["pinecone"], + Literal[ProviderName.PINECONE], Literal["api_key"], ] diff --git a/autogpt_platform/backend/backend/blocks/replicate_flux_advanced.py b/autogpt_platform/backend/backend/blocks/replicate_flux_advanced.py index 11b36751b68a..c913ead132e6 100644 --- a/autogpt_platform/backend/backend/blocks/replicate_flux_advanced.py +++ b/autogpt_platform/backend/backend/blocks/replicate_flux_advanced.py @@ -13,6 +13,7 @@ CredentialsMetaInput, SchemaField, ) +from backend.integrations.providers import ProviderName TEST_CREDENTIALS = APIKeyCredentials( id="01234567-89ab-cdef-0123-456789abcdef", @@ -55,7 +56,7 @@ class ImageType(str, Enum): class ReplicateFluxAdvancedModelBlock(Block): class Input(BlockSchema): credentials: CredentialsMetaInput[ - Literal["replicate"], Literal["api_key"] + Literal[ProviderName.REPLICATE], Literal["api_key"] ] = CredentialsField( description="The Replicate integration can be used with " "any API key with sufficient permissions for the blocks it is used on.", diff --git a/autogpt_platform/backend/backend/blocks/search.py b/autogpt_platform/backend/backend/blocks/search.py index fa5ef0ea70fc..633ad3109113 100644 --- a/autogpt_platform/backend/backend/blocks/search.py +++ b/autogpt_platform/backend/backend/blocks/search.py @@ -11,6 +11,7 @@ CredentialsMetaInput, SchemaField, ) +from backend.integrations.providers import ProviderName class GetWikipediaSummaryBlock(Block, GetRequest): @@ -65,7 +66,7 @@ class Input(BlockSchema): description="Location to get weather information for" ) credentials: CredentialsMetaInput[ - Literal["openweathermap"], Literal["api_key"] + Literal[ProviderName.OPENWEATHERMAP], Literal["api_key"] ] = CredentialsField( description="The OpenWeatherMap integration can be used with " "any API key with sufficient permissions for the blocks it is used on.", diff --git a/autogpt_platform/backend/backend/blocks/talking_head.py b/autogpt_platform/backend/backend/blocks/talking_head.py index 94516d0696df..e1965dbc64bf 100644 --- a/autogpt_platform/backend/backend/blocks/talking_head.py +++ b/autogpt_platform/backend/backend/blocks/talking_head.py @@ -10,6 +10,7 @@ CredentialsMetaInput, SchemaField, ) +from backend.integrations.providers import ProviderName from backend.util.request import requests TEST_CREDENTIALS = APIKeyCredentials( @@ -30,7 +31,7 @@ class CreateTalkingAvatarVideoBlock(Block): class Input(BlockSchema): credentials: CredentialsMetaInput[ - Literal["d_id"], Literal["api_key"] + Literal[ProviderName.D_ID], Literal["api_key"] ] = CredentialsField( description="The D-ID integration can be used with " "any API key with sufficient permissions for the blocks it is used on.", diff --git a/autogpt_platform/backend/backend/blocks/text_to_speech_block.py b/autogpt_platform/backend/backend/blocks/text_to_speech_block.py index 12dc7dd7c832..dc9e0bf26e68 100644 --- a/autogpt_platform/backend/backend/blocks/text_to_speech_block.py +++ b/autogpt_platform/backend/backend/blocks/text_to_speech_block.py @@ -9,6 +9,7 @@ CredentialsMetaInput, SchemaField, ) +from backend.integrations.providers import ProviderName from backend.util.request import requests TEST_CREDENTIALS = APIKeyCredentials( @@ -38,7 +39,7 @@ class Input(BlockSchema): default="Scarlett", ) credentials: CredentialsMetaInput[ - Literal["unreal_speech"], Literal["api_key"] + Literal[ProviderName.UNREAL_SPEECH], Literal["api_key"] ] = CredentialsField( description="The Unreal Speech integration can be used with " "any API key with sufficient permissions for the blocks it is used on.", diff --git a/autogpt_platform/backend/backend/data/model.py b/autogpt_platform/backend/backend/data/model.py index ed23af677670..24c58233f984 100644 --- a/autogpt_platform/backend/backend/data/model.py +++ b/autogpt_platform/backend/backend/data/model.py @@ -32,6 +32,7 @@ core_schema, ) +from backend.integrations.providers import ProviderName from backend.util.settings import Secrets if TYPE_CHECKING: @@ -233,7 +234,7 @@ class UserIntegrations(BaseModel): oauth_states: list[OAuthState] = Field(default_factory=list) -CP = TypeVar("CP", bound=str) +CP = TypeVar("CP", bound=ProviderName) CT = TypeVar("CT", bound=CredentialsType) diff --git a/autogpt_platform/backend/backend/integrations/creds_manager.py b/autogpt_platform/backend/backend/integrations/creds_manager.py index 7cbf8f4af7f1..17633bd58721 100644 --- a/autogpt_platform/backend/backend/integrations/creds_manager.py +++ b/autogpt_platform/backend/backend/integrations/creds_manager.py @@ -1,6 +1,7 @@ import logging from contextlib import contextmanager from datetime import datetime +from typing import TYPE_CHECKING from autogpt_libs.utils.synchronize import RedisKeyedMutex from redis.lock import Lock as RedisLock @@ -8,10 +9,13 @@ from backend.data import redis from backend.data.model import Credentials from backend.integrations.credentials_store import IntegrationCredentialsStore -from backend.integrations.oauth import HANDLERS_BY_NAME, BaseOAuthHandler +from backend.integrations.oauth import HANDLERS_BY_NAME from backend.util.exceptions import MissingConfigError from backend.util.settings import Settings +if TYPE_CHECKING: + from backend.integrations.oauth import BaseOAuthHandler + logger = logging.getLogger(__name__) settings = Settings() @@ -148,7 +152,7 @@ def release_all_locks(self): self.store.locks.release_all_locks() -def _get_provider_oauth_handler(provider_name: str) -> BaseOAuthHandler: +def _get_provider_oauth_handler(provider_name: str) -> "BaseOAuthHandler": if provider_name not in HANDLERS_BY_NAME: raise KeyError(f"Unknown provider '{provider_name}'") diff --git a/autogpt_platform/backend/backend/integrations/oauth/__init__.py b/autogpt_platform/backend/backend/integrations/oauth/__init__.py index 834293da92dd..f5888f07a83e 100644 --- a/autogpt_platform/backend/backend/integrations/oauth/__init__.py +++ b/autogpt_platform/backend/backend/integrations/oauth/__init__.py @@ -1,10 +1,15 @@ -from .base import BaseOAuthHandler +from typing import TYPE_CHECKING + from .github import GitHubOAuthHandler from .google import GoogleOAuthHandler from .notion import NotionOAuthHandler +if TYPE_CHECKING: + from ..providers import ProviderName + from .base import BaseOAuthHandler + # --8<-- [start:HANDLERS_BY_NAMEExample] -HANDLERS_BY_NAME: dict[str, type[BaseOAuthHandler]] = { +HANDLERS_BY_NAME: dict["ProviderName", type["BaseOAuthHandler"]] = { handler.PROVIDER_NAME: handler for handler in [ GitHubOAuthHandler, diff --git a/autogpt_platform/backend/backend/integrations/oauth/base.py b/autogpt_platform/backend/backend/integrations/oauth/base.py index ad5433d734ec..54786ba1aa4c 100644 --- a/autogpt_platform/backend/backend/integrations/oauth/base.py +++ b/autogpt_platform/backend/backend/integrations/oauth/base.py @@ -4,13 +4,14 @@ from typing import ClassVar from backend.data.model import OAuth2Credentials +from backend.integrations.providers import ProviderName logger = logging.getLogger(__name__) class BaseOAuthHandler(ABC): # --8<-- [start:BaseOAuthHandler1] - PROVIDER_NAME: ClassVar[str] + PROVIDER_NAME: ClassVar[ProviderName] DEFAULT_SCOPES: ClassVar[list[str]] = [] # --8<-- [end:BaseOAuthHandler1] @@ -76,6 +77,8 @@ def handle_default_scopes(self, scopes: list[str]) -> list[str]: """Handles the default scopes for the provider""" # If scopes are empty, use the default scopes for the provider if not scopes: - logger.debug(f"Using default scopes for provider {self.PROVIDER_NAME}") + logger.debug( + f"Using default scopes for provider {self.PROVIDER_NAME.value}" + ) scopes = self.DEFAULT_SCOPES return scopes diff --git a/autogpt_platform/backend/backend/integrations/oauth/github.py b/autogpt_platform/backend/backend/integrations/oauth/github.py index ed883b63205f..d83c9b2093a0 100644 --- a/autogpt_platform/backend/backend/integrations/oauth/github.py +++ b/autogpt_platform/backend/backend/integrations/oauth/github.py @@ -3,6 +3,7 @@ from urllib.parse import urlencode from backend.data.model import OAuth2Credentials +from backend.integrations.providers import ProviderName from backend.util.request import requests from .base import BaseOAuthHandler @@ -23,7 +24,7 @@ class GitHubOAuthHandler(BaseOAuthHandler): access token *with no refresh token*. """ # noqa - PROVIDER_NAME = "github" + PROVIDER_NAME = ProviderName.GITHUB def __init__(self, client_id: str, client_secret: str, redirect_uri: str): self.client_id = client_id diff --git a/autogpt_platform/backend/backend/integrations/oauth/google.py b/autogpt_platform/backend/backend/integrations/oauth/google.py index 13175b0b469f..5a03e615a4c7 100644 --- a/autogpt_platform/backend/backend/integrations/oauth/google.py +++ b/autogpt_platform/backend/backend/integrations/oauth/google.py @@ -9,6 +9,7 @@ from pydantic import SecretStr from backend.data.model import OAuth2Credentials +from backend.integrations.providers import ProviderName from .base import BaseOAuthHandler @@ -21,7 +22,7 @@ class GoogleOAuthHandler(BaseOAuthHandler): Based on the documentation at https://developers.google.com/identity/protocols/oauth2/web-server """ # noqa - PROVIDER_NAME = "google" + PROVIDER_NAME = ProviderName.GOOGLE EMAIL_ENDPOINT = "https://www.googleapis.com/oauth2/v2/userinfo" DEFAULT_SCOPES = [ "https://www.googleapis.com/auth/userinfo.email", diff --git a/autogpt_platform/backend/backend/integrations/oauth/notion.py b/autogpt_platform/backend/backend/integrations/oauth/notion.py index 7f5458ae9a10..e71bae29560c 100644 --- a/autogpt_platform/backend/backend/integrations/oauth/notion.py +++ b/autogpt_platform/backend/backend/integrations/oauth/notion.py @@ -2,6 +2,7 @@ from urllib.parse import urlencode from backend.data.model import OAuth2Credentials +from backend.integrations.providers import ProviderName from backend.util.request import requests from .base import BaseOAuthHandler @@ -16,7 +17,7 @@ class NotionOAuthHandler(BaseOAuthHandler): - Notion doesn't use scopes """ - PROVIDER_NAME = "notion" + PROVIDER_NAME = ProviderName.NOTION def __init__(self, client_id: str, client_secret: str, redirect_uri: str): self.client_id = client_id diff --git a/autogpt_platform/backend/backend/integrations/providers.py b/autogpt_platform/backend/backend/integrations/providers.py index b38becf5ca4e..c74e50352d17 100644 --- a/autogpt_platform/backend/backend/integrations/providers.py +++ b/autogpt_platform/backend/backend/integrations/providers.py @@ -1,7 +1,27 @@ from enum import Enum +# --8<-- [start:ProviderName] class ProviderName(str, Enum): + ANTHROPIC = "anthropic" + DISCORD = "discord" + D_ID = "d_id" + FAL = "fal" GITHUB = "github" GOOGLE = "google" + GOOGLE_MAPS = "google_maps" + GROQ = "groq" + HUBSPOT = "hubspot" + IDEOGRAM = "ideogram" + JINA = "jina" + MEDIUM = "medium" NOTION = "notion" + OLLAMA = "ollama" + OPENAI = "openai" + OPENWEATHERMAP = "openweathermap" + OPEN_ROUTER = "open_router" + PINECONE = "pinecone" + REPLICATE = "replicate" + REVID = "revid" + UNREAL_SPEECH = "unreal_speech" + # --8<-- [end:ProviderName] diff --git a/autogpt_platform/backend/backend/server/integrations/router.py b/autogpt_platform/backend/backend/server/integrations/router.py index 4220ac3c0311..e91125649a41 100644 --- a/autogpt_platform/backend/backend/server/integrations/router.py +++ b/autogpt_platform/backend/backend/server/integrations/router.py @@ -1,5 +1,5 @@ import logging -from typing import Annotated, Literal +from typing import TYPE_CHECKING, Annotated, Literal from fastapi import APIRouter, Body, Depends, HTTPException, Path, Query, Request from pydantic import BaseModel, Field, SecretStr @@ -20,12 +20,16 @@ ) from backend.executor.manager import ExecutionManager from backend.integrations.creds_manager import IntegrationCredentialsManager -from backend.integrations.oauth import HANDLERS_BY_NAME, BaseOAuthHandler +from backend.integrations.oauth import HANDLERS_BY_NAME +from backend.integrations.providers import ProviderName from backend.integrations.webhooks import WEBHOOK_MANAGERS_BY_NAME from backend.util.exceptions import NeedConfirmation from backend.util.service import get_service_client from backend.util.settings import Settings +if TYPE_CHECKING: + from backend.integrations.oauth import BaseOAuthHandler + from ..utils import get_user_id logger = logging.getLogger(__name__) @@ -264,7 +268,9 @@ async def delete_credentials( @router.post("/{provider}/webhooks/{webhook_id}/ingress") async def webhook_ingress_generic( request: Request, - provider: Annotated[str, Path(title="Provider where the webhook was registered")], + provider: Annotated[ + ProviderName, Path(title="Provider where the webhook was registered") + ], webhook_id: Annotated[str, Path(title="Our ID for the webhook")], ): logger.debug(f"Received {provider} webhook ingress for ID {webhook_id}") @@ -302,7 +308,9 @@ async def webhook_ingress_generic( @router.post("/{provider}/webhooks/{webhook_id}/ping") async def webhook_ping( - provider: Annotated[str, Path(title="Provider where the webhook was registered")], + provider: Annotated[ + ProviderName, Path(title="Provider where the webhook was registered") + ], webhook_id: Annotated[str, Path(title="Our ID for the webhook")], user_id: Annotated[str, Depends(get_user_id)], # require auth ): @@ -349,7 +357,7 @@ async def remove_all_webhooks_for_credentials( logger.warning(f"Webhook #{webhook.id} failed to prune") -def _get_provider_oauth_handler(req: Request, provider_name: str) -> BaseOAuthHandler: +def _get_provider_oauth_handler(req: Request, provider_name: str) -> "BaseOAuthHandler": if provider_name not in HANDLERS_BY_NAME: raise HTTPException( status_code=404, detail=f"Unknown provider '{provider_name}'" diff --git a/docs/content/platform/new_blocks.md b/docs/content/platform/new_blocks.md index 623b1d9b5257..9fb22db5a913 100644 --- a/docs/content/platform/new_blocks.md +++ b/docs/content/platform/new_blocks.md @@ -121,6 +121,7 @@ from backend.data.model import ( from backend.data.block import Block, BlockOutput, BlockSchema from backend.data.model import CredentialsField +from backend.integrations.providers import ProviderName # API Key auth: @@ -128,9 +129,9 @@ class BlockWithAPIKeyAuth(Block): class Input(BlockSchema): # Note that the type hint below is require or you will get a type error. # The first argument is the provider name, the second is the credential type. - credentials: CredentialsMetaInput[Literal['github'], Literal['api_key']] = CredentialsField( - provider="github", - supported_credential_types={"api_key"}, + credentials: CredentialsMetaInput[ + Literal[ProviderName.GITHUB], Literal["api_key"] + ] = CredentialsField( description="The GitHub integration can be used with " "any API key with sufficient permissions for the blocks it is used on.", ) @@ -151,9 +152,9 @@ class BlockWithOAuth(Block): class Input(BlockSchema): # Note that the type hint below is require or you will get a type error. # The first argument is the provider name, the second is the credential type. - credentials: CredentialsMetaInput[Literal['github'], Literal['oauth2']] = CredentialsField( - provider="github", - supported_credential_types={"oauth2"}, + credentials: CredentialsMetaInput[ + Literal[ProviderName.GITHUB], Literal["oauth2"] + ] = CredentialsField( required_scopes={"repo"}, description="The GitHub integration can be used with OAuth.", ) @@ -174,9 +175,9 @@ class BlockWithAPIKeyAndOAuth(Block): class Input(BlockSchema): # Note that the type hint below is require or you will get a type error. # The first argument is the provider name, the second is the credential type. - credentials: CredentialsMetaInput[Literal['github'], Literal['api_key', 'oauth2']] = CredentialsField( - provider="github", - supported_credential_types={"api_key", "oauth2"}, + credentials: CredentialsMetaInput[ + Literal[ProviderName.GITHUB], Literal["api_key", "oauth2"] + ] = CredentialsField( required_scopes={"repo"}, description="The GitHub integration can be used with OAuth, " "or any API key with sufficient permissions for the blocks it is used on.", @@ -227,6 +228,16 @@ response = requests.post( ) ``` +The `ProviderName` enum is the single source of truth for which providers exist in our system. +Naturally, to add an authenticated block for a new provider, you'll have to add it here too. +
+ProviderName definition + +```python title="backend/integrations/providers.py" +--8<-- "autogpt_platform/backend/backend/integrations/providers.py:ProviderName" +``` +
+ #### Adding an OAuth2 service integration To add support for a new OAuth2-authenticated service, you'll need to add an `OAuthHandler`. From 0b9adab6a4cd3ac50e31f3dc783fd76cd4dd2238 Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Sat, 30 Nov 2024 18:48:21 +0100 Subject: [PATCH 3/6] Use `ProviderName` in webhook managers and fix use of their value in strings --- .../backend/integrations/webhooks/__init__.py | 3 +- .../backend/integrations/webhooks/base.py | 5 +- .../backend/integrations/webhooks/github.py | 3 +- .../webhooks/graph_lifecycle_hooks.py | 18 +++++- .../backend/server/integrations/router.py | 62 ++++++++++++++----- 5 files changed, 69 insertions(+), 22 deletions(-) diff --git a/autogpt_platform/backend/backend/integrations/webhooks/__init__.py b/autogpt_platform/backend/backend/integrations/webhooks/__init__.py index 14d1f7216567..8d373d02f7f8 100644 --- a/autogpt_platform/backend/backend/integrations/webhooks/__init__.py +++ b/autogpt_platform/backend/backend/integrations/webhooks/__init__.py @@ -3,10 +3,11 @@ from .github import GithubWebhooksManager if TYPE_CHECKING: + from ..providers import ProviderName from .base import BaseWebhooksManager # --8<-- [start:WEBHOOK_MANAGERS_BY_NAME] -WEBHOOK_MANAGERS_BY_NAME: dict[str, type["BaseWebhooksManager"]] = { +WEBHOOK_MANAGERS_BY_NAME: dict["ProviderName", type["BaseWebhooksManager"]] = { handler.PROVIDER_NAME: handler for handler in [ GithubWebhooksManager, diff --git a/autogpt_platform/backend/backend/integrations/webhooks/base.py b/autogpt_platform/backend/backend/integrations/webhooks/base.py index 61a07ce20353..68ed3b4cb70e 100644 --- a/autogpt_platform/backend/backend/integrations/webhooks/base.py +++ b/autogpt_platform/backend/backend/integrations/webhooks/base.py @@ -9,6 +9,7 @@ from backend.data import integrations from backend.data.model import Credentials +from backend.integrations.providers import ProviderName from backend.util.exceptions import MissingConfigError from backend.util.settings import Config @@ -20,7 +21,7 @@ class BaseWebhooksManager(ABC, Generic[WT]): # --8<-- [start:BaseWebhooksManager1] - PROVIDER_NAME: ClassVar[str] + PROVIDER_NAME: ClassVar[ProviderName] # --8<-- [end:BaseWebhooksManager1] WebhookType: WT @@ -141,7 +142,7 @@ async def _create_webhook( secret = secrets.token_hex(32) provider_name = self.PROVIDER_NAME ingress_url = ( - f"{app_config.platform_base_url}/api/integrations/{provider_name}" + f"{app_config.platform_base_url}/api/integrations/{provider_name.value}" f"/webhooks/{id}/ingress" ) provider_webhook_id, config = await self._register_webhook( diff --git a/autogpt_platform/backend/backend/integrations/webhooks/github.py b/autogpt_platform/backend/backend/integrations/webhooks/github.py index 2393437d209a..3e1d3cd7ea51 100644 --- a/autogpt_platform/backend/backend/integrations/webhooks/github.py +++ b/autogpt_platform/backend/backend/integrations/webhooks/github.py @@ -8,6 +8,7 @@ from backend.data import integrations from backend.data.model import Credentials +from backend.integrations.providers import ProviderName from .base import BaseWebhooksManager @@ -20,7 +21,7 @@ class GithubWebhookType(StrEnum): class GithubWebhooksManager(BaseWebhooksManager): - PROVIDER_NAME = "github" + PROVIDER_NAME = ProviderName.GITHUB WebhookType = GithubWebhookType diff --git a/autogpt_platform/backend/backend/integrations/webhooks/graph_lifecycle_hooks.py b/autogpt_platform/backend/backend/integrations/webhooks/graph_lifecycle_hooks.py index 363bf535c501..6fb728104857 100644 --- a/autogpt_platform/backend/backend/integrations/webhooks/graph_lifecycle_hooks.py +++ b/autogpt_platform/backend/backend/integrations/webhooks/graph_lifecycle_hooks.py @@ -95,11 +95,18 @@ async def on_node_activate( if not block.webhook_config: return node + provider = block.webhook_config.provider + if provider not in WEBHOOK_MANAGERS_BY_NAME: + raise ValueError( + f"Block #{block.id} has webhook_config for provider {provider} " + "which does not support webhooks" + ) + logger.debug( f"Activating webhook node #{node.id} with config {block.webhook_config}" ) - webhooks_manager = WEBHOOK_MANAGERS_BY_NAME[block.webhook_config.provider]() + webhooks_manager = WEBHOOK_MANAGERS_BY_NAME[provider]() try: resource = block.webhook_config.resource_format.format(**node.input_default) @@ -167,7 +174,14 @@ async def on_node_deactivate( if not block.webhook_config: return node - webhooks_manager = WEBHOOK_MANAGERS_BY_NAME[block.webhook_config.provider]() + provider = block.webhook_config.provider + if provider not in WEBHOOK_MANAGERS_BY_NAME: + raise ValueError( + f"Block #{block.id} has webhook_config for provider {provider} " + "which does not support webhooks" + ) + + webhooks_manager = WEBHOOK_MANAGERS_BY_NAME[provider]() if node.webhook_id: logger.debug(f"Node #{node.id} has webhook_id {node.webhook_id}") diff --git a/autogpt_platform/backend/backend/server/integrations/router.py b/autogpt_platform/backend/backend/server/integrations/router.py index e91125649a41..6b445ee99998 100644 --- a/autogpt_platform/backend/backend/server/integrations/router.py +++ b/autogpt_platform/backend/backend/server/integrations/router.py @@ -46,7 +46,9 @@ class LoginResponse(BaseModel): @router.get("/{provider}/login") def login( - provider: Annotated[str, Path(title="The provider to initiate an OAuth flow for")], + provider: Annotated[ + ProviderName, Path(title="The provider to initiate an OAuth flow for") + ], user_id: Annotated[str, Depends(get_user_id)], request: Request, scopes: Annotated[ @@ -78,7 +80,9 @@ class CredentialsMetaResponse(BaseModel): @router.post("/{provider}/callback") def callback( - provider: Annotated[str, Path(title="The target provider for this OAuth exchange")], + provider: Annotated[ + ProviderName, Path(title="The target provider for this OAuth exchange") + ], code: Annotated[str, Body(title="Authorization code acquired by user login")], state_token: Annotated[str, Body(title="Anti-CSRF nonce")], user_id: Annotated[str, Depends(get_user_id)], @@ -107,11 +111,12 @@ def callback( if not set(scopes).issubset(set(credentials.scopes)): # For now, we'll just log the warning and continue logger.warning( - f"Granted scopes {credentials.scopes} for {provider}do not include all requested scopes {scopes}" + f"Granted scopes {credentials.scopes} for provider {provider.value} " + f"do not include all requested scopes {scopes}" ) except Exception as e: - logger.error(f"Code->Token exchange failed for provider {provider}: {e}") + logger.error(f"Code->Token exchange failed for provider {provider.value}: {e}") raise HTTPException( status_code=400, detail=f"Failed to exchange code for tokens: {str(e)}" ) @@ -120,7 +125,8 @@ def callback( creds_manager.create(user_id, credentials) logger.debug( - f"Successfully processed OAuth callback for user {user_id} and provider {provider}" + f"Successfully processed OAuth callback for user {user_id} " + f"and provider {provider.value}" ) return CredentialsMetaResponse( id=credentials.id, @@ -152,7 +158,9 @@ def list_credentials( @router.get("/{provider}/credentials") def list_credentials_by_provider( - provider: Annotated[str, Path(title="The provider to list credentials for")], + provider: Annotated[ + ProviderName, Path(title="The provider to list credentials for") + ], user_id: Annotated[str, Depends(get_user_id)], ) -> list[CredentialsMetaResponse]: credentials = creds_manager.store.get_creds_by_provider(user_id, provider) @@ -171,7 +179,9 @@ def list_credentials_by_provider( @router.get("/{provider}/credentials/{cred_id}") def get_credential( - provider: Annotated[str, Path(title="The provider to retrieve credentials for")], + provider: Annotated[ + ProviderName, Path(title="The provider to retrieve credentials for") + ], cred_id: Annotated[str, Path(title="The ID of the credentials to retrieve")], user_id: Annotated[str, Depends(get_user_id)], ) -> Credentials: @@ -188,7 +198,9 @@ def get_credential( @router.post("/{provider}/credentials", status_code=201) def create_api_key_credentials( user_id: Annotated[str, Depends(get_user_id)], - provider: Annotated[str, Path(title="The provider to create credentials for")], + provider: Annotated[ + ProviderName, Path(title="The provider to create credentials for") + ], api_key: Annotated[str, Body(title="The API key to store")], title: Annotated[str, Body(title="Optional title for the credentials")], expires_at: Annotated[ @@ -229,7 +241,9 @@ class CredentialsDeletionNeedsConfirmationResponse(BaseModel): @router.delete("/{provider}/credentials/{cred_id}") async def delete_credentials( request: Request, - provider: Annotated[str, Path(title="The provider to delete credentials for")], + provider: Annotated[ + ProviderName, Path(title="The provider to delete credentials for") + ], cred_id: Annotated[str, Path(title="The ID of the credentials to delete")], user_id: Annotated[str, Depends(get_user_id)], force: Annotated[ @@ -273,12 +287,15 @@ async def webhook_ingress_generic( ], webhook_id: Annotated[str, Path(title="Our ID for the webhook")], ): - logger.debug(f"Received {provider} webhook ingress for ID {webhook_id}") + logger.debug(f"Received {provider.value} webhook ingress for ID {webhook_id}") webhook_manager = WEBHOOK_MANAGERS_BY_NAME[provider]() webhook = await get_webhook(webhook_id) logger.debug(f"Webhook #{webhook_id}: {webhook}") payload, event_type = await webhook_manager.validate_payload(webhook, request) - logger.debug(f"Validated {provider} {event_type} event with payload {payload}") + logger.debug( + f"Validated {provider.value} {webhook.webhook_type} {event_type} event " + f"with payload {payload}" + ) webhook_event = WebhookEvent( provider=provider, @@ -339,6 +356,14 @@ async def remove_all_webhooks_for_credentials( NeedConfirmation: If any of the webhooks are still in use and `force` is `False` """ webhooks = await get_all_webhooks(credentials.id) + if credentials.provider not in WEBHOOK_MANAGERS_BY_NAME: + if webhooks: + logger.error( + f"Credentials #{credentials.id} for provider {credentials.provider} " + f"are attached to {len(webhooks)} webhooks, " + f"but there is no available WebhooksHandler for {credentials.provider}" + ) + return if any(w.attached_nodes for w in webhooks) and not force: raise NeedConfirmation( "Some webhooks linked to these credentials are still in use by an agent" @@ -357,18 +382,23 @@ async def remove_all_webhooks_for_credentials( logger.warning(f"Webhook #{webhook.id} failed to prune") -def _get_provider_oauth_handler(req: Request, provider_name: str) -> "BaseOAuthHandler": +def _get_provider_oauth_handler( + req: Request, provider_name: ProviderName +) -> "BaseOAuthHandler": if provider_name not in HANDLERS_BY_NAME: raise HTTPException( - status_code=404, detail=f"Unknown provider '{provider_name}'" + status_code=404, + detail=f"Provider '{provider_name.value}' does not support OAuth", ) - client_id = getattr(settings.secrets, f"{provider_name}_client_id") - client_secret = getattr(settings.secrets, f"{provider_name}_client_secret") + client_id = getattr(settings.secrets, f"{provider_name.value}_client_id") + client_secret = getattr(settings.secrets, f"{provider_name.value}_client_secret") if not (client_id and client_secret): raise HTTPException( status_code=501, - detail=f"Integration with provider '{provider_name}' is not configured", + detail=( + f"Integration with provider '{provider_name.value}' is not configured" + ), ) handler_class = HANDLERS_BY_NAME[provider_name] From 8fb15cc8da95d6a9b580346c17e3b0957e46eecf Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Tue, 10 Dec 2024 13:48:26 +0100 Subject: [PATCH 4/6] Change type of `Webhook.provider` to `ProviderName` --- autogpt_platform/backend/backend/data/integrations.py | 7 ++++--- .../backend/integrations/webhooks/graph_lifecycle_hooks.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/autogpt_platform/backend/backend/data/integrations.py b/autogpt_platform/backend/backend/data/integrations.py index 4d5197f693a8..4f444fb03505 100644 --- a/autogpt_platform/backend/backend/data/integrations.py +++ b/autogpt_platform/backend/backend/data/integrations.py @@ -7,6 +7,7 @@ from backend.data.includes import INTEGRATION_WEBHOOK_INCLUDE from backend.data.queue import AsyncRedisEventBus +from backend.integrations.providers import ProviderName from .db import BaseDbModel @@ -18,7 +19,7 @@ class Webhook(BaseDbModel): user_id: str - provider: str + provider: ProviderName credentials_id: str webhook_type: str resource: str @@ -37,7 +38,7 @@ def from_db(webhook: IntegrationWebhook): return Webhook( id=webhook.id, user_id=webhook.userId, - provider=webhook.provider, + provider=ProviderName(webhook.provider), credentials_id=webhook.credentialsId, webhook_type=webhook.webhookType, resource=webhook.resource, @@ -61,7 +62,7 @@ async def create_webhook(webhook: Webhook) -> Webhook: data={ "id": webhook.id, "userId": webhook.user_id, - "provider": webhook.provider, + "provider": webhook.provider.value, "credentialsId": webhook.credentials_id, "webhookType": webhook.webhook_type, "resource": webhook.resource, diff --git a/autogpt_platform/backend/backend/integrations/webhooks/graph_lifecycle_hooks.py b/autogpt_platform/backend/backend/integrations/webhooks/graph_lifecycle_hooks.py index 6fb728104857..c241bc3a4a41 100644 --- a/autogpt_platform/backend/backend/integrations/webhooks/graph_lifecycle_hooks.py +++ b/autogpt_platform/backend/backend/integrations/webhooks/graph_lifecycle_hooks.py @@ -203,7 +203,7 @@ async def on_node_deactivate( logger.warning( f"Cannot deregister webhook #{webhook.id}: credentials " f"#{webhook.credentials_id} not available " - f"({webhook.provider} webhook ID: {webhook.provider_webhook_id})" + f"({webhook.provider.value} webhook ID: {webhook.provider_webhook_id})" ) return updated_node From 401cb141f6be65dc259989c7beec23be13e4e3f8 Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Tue, 10 Dec 2024 14:26:25 +0100 Subject: [PATCH 5/6] fix discriminator check --- autogpt_platform/backend/backend/data/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt_platform/backend/backend/data/model.py b/autogpt_platform/backend/backend/data/model.py index 92d860987069..9e79fd7da394 100644 --- a/autogpt_platform/backend/backend/data/model.py +++ b/autogpt_platform/backend/backend/data/model.py @@ -277,7 +277,7 @@ def validate_credentials_field_schema(cls, model: type["BlockSchema"]): if ( len(schema_extra.credentials_provider) > 1 - and not schema_extra.credentials_provider + and not schema_extra.discriminator ): raise TypeError("Multi-provider CredentialsField requires discriminator!") From 11f82b921230a00f425b4fd90c11ff12e74139df Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Wed, 11 Dec 2024 20:10:57 +0100 Subject: [PATCH 6/6] fix `BlockSchema.cached_jsonschema` accidental inheritance issue --- autogpt_platform/backend/backend/data/block.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/autogpt_platform/backend/backend/data/block.py b/autogpt_platform/backend/backend/data/block.py index 071129056627..096195e2c97c 100644 --- a/autogpt_platform/backend/backend/data/block.py +++ b/autogpt_platform/backend/backend/data/block.py @@ -65,7 +65,7 @@ def dict(self) -> dict[str, str]: class BlockSchema(BaseModel): - cached_jsonschema: ClassVar[dict[str, Any]] = {} + cached_jsonschema: ClassVar[dict[str, Any]] @classmethod def jsonschema(cls) -> dict[str, Any]: @@ -145,6 +145,10 @@ def __pydantic_init_subclass__(cls, **kwargs): - A field that is called `credentials` MUST be a `CredentialsMetaInput`. """ super().__pydantic_init_subclass__(**kwargs) + + # Reset cached JSON schema to prevent inheriting it from parent class + cls.cached_jsonschema = {} + credentials_fields = [ field_name for field_name, info in cls.model_fields.items()