From bb468841d2249fea93f7cc8883d685ae1d8b81cc Mon Sep 17 00:00:00 2001 From: "Adam Ling (MSFT)" Date: Thu, 4 Mar 2021 11:03:06 -0800 Subject: [PATCH 1/2] [EventHubs] add logging.info to warn the usage of partition key of non-string type (#17057) * add a logging to warn the usage of partition key of non-string type * move the logging info into eventdatabatch to avoid duplicate logging when sending list, also updated the wording * remove unused six module --- .../azure-eventhub/azure/eventhub/_common.py | 9 +++++++++ .../azure/eventhub/_producer_client.py | 13 +++++++++---- .../azure/eventhub/aio/_producer_client_async.py | 13 +++++++++---- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/sdk/eventhub/azure-eventhub/azure/eventhub/_common.py b/sdk/eventhub/azure-eventhub/azure/eventhub/_common.py index d15196dd8cf3..7f7f967ac63f 100644 --- a/sdk/eventhub/azure-eventhub/azure/eventhub/_common.py +++ b/sdk/eventhub/azure-eventhub/azure/eventhub/_common.py @@ -350,6 +350,15 @@ class EventDataBatch(object): def __init__(self, max_size_in_bytes=None, partition_id=None, partition_key=None): # type: (Optional[int], Optional[str], Optional[Union[str, bytes]]) -> None + + if partition_key and not isinstance(partition_key, (six.text_type, six.binary_type)): + _LOGGER.info( + "WARNING: Setting partition_key of non-string value on the events to be sent is discouraged " + "as the partition_key will be ignored by the Event Hub service and events will be assigned " + "to all partitions using round-robin. Furthermore, there are SDKs for consuming events which expect " + "partition_key to only be string type, they might fail to parse the non-string value." + ) + self.max_size_in_bytes = max_size_in_bytes or constants.MAX_MESSAGE_LENGTH_BYTES self.message = BatchMessage(data=[], multi_messages=False, properties=None) self._partition_id = partition_id diff --git a/sdk/eventhub/azure-eventhub/azure/eventhub/_producer_client.py b/sdk/eventhub/azure-eventhub/azure/eventhub/_producer_client.py index c3f139337eb9..9ae9dad761e4 100644 --- a/sdk/eventhub/azure-eventhub/azure/eventhub/_producer_client.py +++ b/sdk/eventhub/azure-eventhub/azure/eventhub/_producer_client.py @@ -222,8 +222,10 @@ def send_batch(self, event_data_batch, **kwargs): A `TypeError` will be raised if partition_key is specified and event_data_batch is an `EventDataBatch` because `EventDataBatch` itself has partition_key. If both partition_id and partition_key are provided, the partition_id will take precedence. - **WARNING: Please DO NOT pass a partition_key of non-string type. The Event Hub service ignores partition_key - of non-string type, in which case events will be assigned to all partitions using round-robin.** + **WARNING: Setting partition_key of non-string value on the events to be sent is discouraged + as the partition_key will be ignored by the Event Hub service and events will be assigned + to all partitions using round-robin. Furthermore, there are SDKs for consuming events which expect + partition_key to only be string type, they might fail to parse the non-string value.** :rtype: None :raises: :class:`AuthenticationError` :class:`ConnectError` @@ -246,6 +248,7 @@ def send_batch(self, event_data_batch, **kwargs): """ partition_id = kwargs.get("partition_id") partition_key = kwargs.get("partition_key") + if isinstance(event_data_batch, EventDataBatch): if partition_id or partition_key: raise TypeError("partition_id and partition_key should be None when sending an EventDataBatch " @@ -283,8 +286,10 @@ def create_batch(self, **kwargs): :keyword str partition_key: With the given partition_key, event data will be sent to a particular partition of the Event Hub decided by the service. If both partition_id and partition_key are provided, the partition_id will take precedence. - **WARNING: Please DO NOT pass a partition_key of non-string type. The Event Hub service ignores partition_key - of non-string type, in which case events will be assigned to all partitions using round-robin.** + **WARNING: Setting partition_key of non-string value on the events to be sent is discouraged + as the partition_key will be ignored by the Event Hub service and events will be assigned + to all partitions using round-robin. Furthermore, there are SDKs for consuming events which expect + partition_key to only be string type, they might fail to parse the non-string value.** :keyword int max_size_in_bytes: The maximum size of bytes data that an EventDataBatch object can hold. By default, the value is determined by your Event Hubs tier. :rtype: ~azure.eventhub.EventDataBatch diff --git a/sdk/eventhub/azure-eventhub/azure/eventhub/aio/_producer_client_async.py b/sdk/eventhub/azure-eventhub/azure/eventhub/aio/_producer_client_async.py index 6caa8e9f5144..7035060a75ff 100644 --- a/sdk/eventhub/azure-eventhub/azure/eventhub/aio/_producer_client_async.py +++ b/sdk/eventhub/azure-eventhub/azure/eventhub/aio/_producer_client_async.py @@ -253,8 +253,10 @@ async def send_batch( A `TypeError` will be raised if partition_key is specified and event_data_batch is an `EventDataBatch` because `EventDataBatch` itself has partition_key. If both partition_id and partition_key are provided, the partition_id will take precedence. - **WARNING: Please DO NOT pass a partition_key of non-string type. The Event Hub service ignores partition_key - of non-string type, in which case events will be assigned to all partitions using round-robin.** + **WARNING: Setting partition_key of non-string value on the events to be sent is discouraged + as the partition_key will be ignored by the Event Hub service and events will be assigned + to all partitions using round-robin. Furthermore, there are SDKs for consuming events which expect + partition_key to only be string type, they might fail to parse the non-string value.** :rtype: None :raises: :class:`AuthenticationError` :class:`ConnectError` @@ -277,6 +279,7 @@ async def send_batch( """ partition_id = kwargs.get("partition_id") partition_key = kwargs.get("partition_key") + if isinstance(event_data_batch, EventDataBatch): if partition_id or partition_key: raise TypeError("partition_id and partition_key should be None when sending an EventDataBatch " @@ -318,8 +321,10 @@ async def create_batch( :param str partition_key: With the given partition_key, event data will be sent to a particular partition of the Event Hub decided by the service. If both partition_id and partition_key are provided, the partition_id will take precedence. - **WARNING: Please DO NOT pass a partition_key of non-string type. The Event Hub service ignores partition_key - of non-string type, in which case events will be assigned to all partitions using round-robin.** + **WARNING: Setting partition_key of non-string value on the events to be sent is discouraged + as the partition_key will be ignored by the Event Hub service and events will be assigned + to all partitions using round-robin. Furthermore, there are SDKs for consuming events which expect + partition_key to only be string type, they might fail to parse the non-string value.** :param int max_size_in_bytes: The maximum size of bytes data that an EventDataBatch object can hold. By default, the value is determined by your Event Hubs tier. :rtype: ~azure.eventhub.EventDataBatch From 4f18a58e89858b7955a43f0a580469575729ba2e Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 4 Mar 2021 11:51:10 -0800 Subject: [PATCH 2/2] EG - more docs imrpovement (#17079) * docs improvement * commenbts * Update sdk/eventgrid/azure-eventgrid/README.md Co-authored-by: swathipil <76007337+swathipil@users.noreply.github.com> Co-authored-by: swathipil <76007337+swathipil@users.noreply.github.com> --- sdk/eventgrid/azure-eventgrid/README.md | 81 +++++++++++++++++-- .../azure/eventgrid/_models.py | 16 ++-- .../azure/eventgrid/_publisher_client.py | 15 ++-- .../eventgrid/aio/_publisher_client_async.py | 10 ++- ..._publish_custom_schema_to_a_topic_async.py | 4 +- .../sample_publish_eg_event_using_dict.py | 2 +- 6 files changed, 100 insertions(+), 28 deletions(-) diff --git a/sdk/eventgrid/azure-eventgrid/README.md b/sdk/eventgrid/azure-eventgrid/README.md index 813cf6208a5e..2ff9d24b800e 100644 --- a/sdk/eventgrid/azure-eventgrid/README.md +++ b/sdk/eventgrid/azure-eventgrid/README.md @@ -59,24 +59,22 @@ endpoint = "https://..eventgrid.azure.net" credential = AzureKeyCredential("") eg_publisher_client = EventGridPublisherClient(endpoint, credential) ``` -> **Note:** A client may also be authenticated via SAS signature, using the `AzureSasCredential`. A sample demonstrating this, is available [here][python-eg-sample-publish-sas-signature] ([async_version][python-eg-sample-publish-sas-signature-async]). +> **Note:** A client may also be authenticated via SAS signature, using the `AzureSasCredential`. A sample demonstrating this, is available [here][python-eg-sample-send-using-sas] ([async_version][python-eg-sample-send-using-sas-async]). > **Note:** The `generate_sas` method can be used to generate a shared access signature. A sample demonstrating this can be seen [here][python-eg-generate-sas]. ## Key concepts -Information about the key concepts on Event Grid, see [Concepts in Azure Event Grid][publisher-service-doc] - ### Topic -A [topic](https://docs.microsoft.com/azure/event-grid/concepts#topics) is a channel within the EventGrid service to send events. The event schema that a topic accepts is decided at topic creation time. If events of a schema type are sent to a topic that requires a different schema type, errors will be raised. +A **[topic](https://docs.microsoft.com/azure/event-grid/concepts#topics)** is a channel within the EventGrid service to send events. The event schema that a topic accepts is decided at topic creation time. If events of a schema type are sent to a topic that requires a different schema type, errors will be raised. ### Domain -An event [domain](https://docs.microsoft.com/azure/event-grid/event-domains) is a management tool for large numbers of Event Grid topics related to the same application. They allow you to publish events to thousands of topics. Domains also give you authorization and authentication control over each topic. For more information, visit [Event domain overview](https://docs.microsoft.com/azure/event-grid/event-domains). +An event **[domain](https://docs.microsoft.com/azure/event-grid/event-domains)** is a management tool for large numbers of Event Grid topics related to the same application. They allow you to publish events to thousands of topics. Domains also give you authorization and authentication control over each topic. For more information, visit [Event domain overview](https://docs.microsoft.com/azure/event-grid/event-domains). When you create an event domain, a publishing endpoint for this domain is made available to you. This process is similar to creating an Event Grid Topic. The only difference is that, when publishing to a domain, you must specify the topic within the domain that you'd like the event to be delivered to. ### Event schemas -An [**event**](https://docs.microsoft.com/azure/event-grid/concepts#events) is the smallest amount of information that fully describes something that happened in the system. When a custom topic or domain is created, you must specify the schema that will be used when publishing events. +An **[event](https://docs.microsoft.com/azure/event-grid/concepts#events)** is the smallest amount of information that fully describes something that happened in the system. When a custom topic or domain is created, you must specify the schema that will be used when publishing events. Event Grid supports multiple schemas for encoding events. @@ -102,7 +100,9 @@ The following formats of events are allowed to be sent: Please have a look at the [samples](https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/eventgrid/azure-eventgrid/samples) for detailed examples. - **Note:** It is important to know if your topic supports Cloud or EventGrid events before publishing. If you send to a topic that does not support the schema of the event you are sending, send() will throw an exception. + **Note:** It is important to know if your topic supports CloudEvents or EventGridEvents before publishing. If you send to a topic that does not support the schema of the event you are sending, send() will throw an exception. + + For more information about the key concepts on Event Grid, see [Concepts in Azure Event Grid][publisher-service-doc]. ## Examples @@ -110,6 +110,8 @@ The following sections provide several code snippets covering some of the most c * [Send an Event Grid Event](#send-an-event-grid-event) * [Send a Cloud Event](#send-a-cloud-event) +* [Send Multiple Events](#send-multiple-events) +* [Send events as Dictionaries](#send-events-as-dictionaries) * [Consume a payload from storage queue](#consume-from-storage-queue) * [Consume from ServiceBus](#consume-from-servicebus) @@ -162,6 +164,70 @@ client = EventGridPublisherClient(endpoint, credential) client.send(event) ``` + +### Send Multiple events + +It is possible to send events as a batch when sending multiple events to a topic or a domain. This example sends a list of CloudEvents using the send method. + +**WARNING:** When sending a list of multiple events at one time, iterating over and sending each event will not result in optimal performance. For best performance, it is highly recommended to send a list of events. + +```Python +import os +from azure.core.credentials import AzureKeyCredential +from azure.core.messaging import CloudEvent +from azure.eventgrid import EventGridPublisherClient + +key = os.environ["CLOUD_ACCESS_KEY"] +endpoint = os.environ["CLOUD_TOPIC_HOSTNAME"] + +event0 = CloudEvent( + type="Azure.Sdk.Sample", + source="https://egsample.dev/sampleevent", + data={"team": "azure-sdk"} +) +event1 = CloudEvent( + type="Azure.Sdk.Sample", + source="https://egsample.dev/sampleevent", + data={"team2": "azure-eventgrid"} +) + +events = [event0, event1] + +credential = AzureKeyCredential(key) +client = EventGridPublisherClient(endpoint, credential) + +client.send(events) +``` + +### Send events as dictionaries + +A dict representation of respective serialized models can also be used to publish CloudEvent(s) or EventGridEvent(s) apart from the strongly typed objects. + +Use a dict-like representation to send to a topic with custom schema as shown below. + +```Python +import os +from azure.core.credentials import AzureKeyCredential +from azure.eventgrid import EventGridPublisherClient + +key = os.environ["CUSTOM_SCHEMA_ACCESS_KEY"] +endpoint = os.environ["CUSTOM_SCHEMA_TOPIC_HOSTNAME"] + +event = custom_schema_event = { + "customSubject": "sample", + "customEventType": "sample.event", + "customDataVersion": "2.0", + "customId": uuid.uuid4(), + "customEventTime": dt.datetime.now(UTC()).isoformat(), + "customData": "sample data" + } + +credential = AzureKeyCredential(key) +client = EventGridPublisherClient(endpoint, credential) + +client.send(event) +``` + ### Consume from storage queue This example consumes a message received from storage queue and deserializes it to a CloudEvent object. @@ -328,6 +394,7 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con [python-eg-pypi]: https://pypi.org/project/azure-eventgrid [python-eg-product-docs]: https://docs.microsoft.com/azure/event-grid/overview [python-eg-ref-docs]: https://azuresdkdocs.blob.core.windows.net/$web/python/azure-eventgrid/latest/index.html +[publisher-service-doc]: https://docs.microsoft.com/azure/event-grid/concepts [python-eg-samples]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/eventgrid/azure-eventgrid/samples [python-eg-changelog]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/eventgrid/azure-eventgrid/CHANGELOG.md [pip]: https://pypi.org/project/pip/ diff --git a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_models.py b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_models.py index 3deb712dd9b1..636fed5ddc78 100644 --- a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_models.py +++ b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_models.py @@ -28,17 +28,17 @@ class EventGridEvent(InternalEventGridEvent): :param data_version: Required. The schema version of the data object. If not provided, will be stamped with an empty value. :type data_version: str - :keyword topic: Optional. The resource path of the event source. If not provided, Event Grid will + :keyword topic: The resource path of the event source. If not provided, Event Grid will stamp onto the event. This is required when sending event(s) to a domain. - :type topic: str - :keyword metadata_version: Optional. The schema version of the event metadata. If provided, + :paramtype topic: Optional[str] + :keyword metadata_version: The schema version of the event metadata. If provided, must match Event Grid Schema exactly. If not provided, EventGrid will stamp onto event. - :type metadata_version: str - :keyword id: Optional. An identifier for the event. In not provided, a random UUID will be generated and used. - :type id: Optional[str] - :keyword event_time: Optional.The time (in UTC) of the event. If not provided, + :paramtype metadata_version: Optional[str] + :keyword id: An identifier for the event. In not provided, a random UUID will be generated and used. + :paramtype id: Optional[str] + :keyword event_time: The time (in UTC) of the event. If not provided, it will be the time (in UTC) the event was generated. - :type event_time: Optional[~datetime.datetime] + :paramtype event_time: Optional[~datetime.datetime] :ivar subject: A resource path relative to the topic path. :vartype subject: str :ivar event_type: The type of the event that occurred. diff --git a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_publisher_client.py b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_publisher_client.py index 482699f9068a..1bf0a9eee2f2 100644 --- a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_publisher_client.py +++ b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_publisher_client.py @@ -62,6 +62,7 @@ class EventGridPublisherClient(object): :param credential: The credential object used for authentication which implements SAS key authentication or SAS token authentication. :type credential: ~azure.core.credentials.AzureKeyCredential or ~azure.core.credentials.AzureSasCredential + :rtype: None .. admonition:: Example: @@ -143,8 +144,8 @@ def send(self, events, **kwargs): :start-after: [START publish_eg_event_dict] :end-before: [END publish_eg_event_dict] :language: python - :dedent: 0 - :caption: Publishing an EventGridEvent using a dict-like representation. + :dedent: 4 + :caption: Publishing a list of EventGridEvents using a dict-like representation. .. literalinclude:: ../samples/sync_samples/sample_publish_cloud_event_using_dict.py :start-after: [START publish_cloud_event_dict] @@ -162,14 +163,16 @@ def send(self, events, **kwargs): :start-after: [START publish_custom_schema] :end-before: [END publish_custom_schema] :language: python - :dedent: 0 + :dedent: 4 :caption: Publishing a Custom Schema event. - **WARNING**: To gain the best performance when sending multiple events at one time, - it is highly recommended to send a list of events instead of iterating over and sending each event in a loop. + **WARNING**: When sending a list of multiple events at one time, iterating over and sending each event + will not result in optimal performance. For best performance, it is highly recommended to send + a list of events. :param events: A single instance or a list of dictionaries/CloudEvent/EventGridEvent to be sent. - :type events: SendType + :type events: ~azure.core.messaging.CloudEvent, ~azure.eventgrid.EventGridEvent, Dict, + list[~azure.core.messaging.CloudEvent], list[~azure.eventgrid.EventGridEvent] or list[Dict] :keyword str content_type: The type of content to be used to send the events. Has default value "application/json; charset=utf-8" for EventGridEvents, with "cloudevents-batch+json" for CloudEvents diff --git a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/aio/_publisher_client_async.py b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/aio/_publisher_client_async.py index b22c97c280ef..410cfba60243 100644 --- a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/aio/_publisher_client_async.py +++ b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/aio/_publisher_client_async.py @@ -137,7 +137,7 @@ async def send(self, events: SendType, **kwargs: Any) -> None: :end-before: [END publish_eg_event_dict_async] :language: python :dedent: 4 - :caption: Publishing an EventGridEvent using a dict-like representation. + :caption: Publishing a list of EventGridEvents using a dict-like representation. .. literalinclude:: ../samples/async_samples/sample_publish_cloud_event_using_dict_async.py :start-after: [START publish_cloud_event_dict_async] @@ -158,11 +158,13 @@ async def send(self, events: SendType, **kwargs: Any) -> None: :dedent: 4 :caption: Publishing a Custom Schema event. - **WARNING**: To gain the best performance when sending multiple events at one time, - it is highly recommended to send a list of events instead of iterating over and sending each event in a loop. + **WARNING**: When sending a list of multiple events at one time, iterating over and sending each event + will not result in optimal performance. For best performance, it is highly recommended to send + a list of events. :param events: A single instance or a list of dictionaries/CloudEvent/EventGridEvent to be sent. - :type events: SendType + :type events: ~azure.core.messaging.CloudEvent, ~azure.eventgrid.EventGridEvent, Dict, + list[~azure.core.messaging.CloudEvent], list[~azure.eventgrid.EventGridEvent] or list[Dict] :keyword str content_type: The type of content to be used to send the events. Has default value "application/json; charset=utf-8" for EventGridEvents, with "cloudevents-batch+json" for CloudEvents diff --git a/sdk/eventgrid/azure-eventgrid/samples/async_samples/sample_publish_custom_schema_to_a_topic_async.py b/sdk/eventgrid/azure-eventgrid/samples/async_samples/sample_publish_custom_schema_to_a_topic_async.py index 26306a839329..a25ac1c9798e 100644 --- a/sdk/eventgrid/azure-eventgrid/samples/async_samples/sample_publish_custom_schema_to_a_topic_async.py +++ b/sdk/eventgrid/azure-eventgrid/samples/async_samples/sample_publish_custom_schema_to_a_topic_async.py @@ -30,7 +30,7 @@ async def publish_event(): # authenticate client - # [START publish_custon_schema_async] + # [START publish_custom_schema_async] credential = AzureKeyCredential(key) client = EventGridPublisherClient(endpoint, credential) @@ -46,7 +46,7 @@ async def publish_event(): # publish list of events await client.send(custom_schema_event) - # [END publish_custon_schema_async] + # [END publish_custom_schema_async] if __name__ == '__main__': loop = asyncio.get_event_loop() diff --git a/sdk/eventgrid/azure-eventgrid/samples/sync_samples/sample_publish_eg_event_using_dict.py b/sdk/eventgrid/azure-eventgrid/samples/sync_samples/sample_publish_eg_event_using_dict.py index 68eb336419fe..63524e494453 100644 --- a/sdk/eventgrid/azure-eventgrid/samples/sync_samples/sample_publish_eg_event_using_dict.py +++ b/sdk/eventgrid/azure-eventgrid/samples/sync_samples/sample_publish_eg_event_using_dict.py @@ -26,10 +26,10 @@ endpoint = os.environ["EG_TOPIC_HOSTNAME"] def publish(): + # [START publish_eg_event_dict] credential = AzureKeyCredential(topic_key) client = EventGridPublisherClient(endpoint, credential) - # [START publish_eg_event_dict] event0 = { "eventType": "Contoso.Items.ItemReceived", "data": {