diff --git a/HISTORY.rst b/HISTORY.rst index 8ff577038..02cd361cd 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -25,6 +25,10 @@ Release History - az dt endpoint create - az dt private-endpoint create +* Added `az dt data-history` command group, which will allow users to configure a data history connection + for a Digital Twins instance using an Event Hub and Azure Data Explorer database. Once configured, + changes to the Digital Twins instance can be seen in the Azure Data Explorer database. + **IoT Central updates** * Added commands for Edge devices and modules: diff --git a/azext_iot/digitaltwins/_help.py b/azext_iot/digitaltwins/_help.py index 77426a20a..08dad3be2 100644 --- a/azext_iot/digitaltwins/_help.py +++ b/azext_iot/digitaltwins/_help.py @@ -7,11 +7,8 @@ Help definitions for IoT Hub commands. """ -import os from knack.help_files import helps -enabled_data_history = os.environ.get("ADT_DH_PREVIEW_ENABLE") - def load_digitaltwins_help(): @@ -143,124 +140,122 @@ def load_digitaltwins_help(): az dt reset -n {instance_name} """ - # TODO: uncomment once data history is public - if enabled_data_history: - helps["dt data-history"] = """ - type: group - short-summary: Manage and configure data history. - """ - - helps["dt data-history connection"] = """ - type: group - short-summary: Manage and configure data history connections. - """ - - helps["dt data-history connection create"] = """ - type: group - short-summary: Creates a data history connection between a Digital Twins instance and supported resources. - """ - - helps["dt data-history connection create adx"] = """ - type: command - short-summary: Creates a data history connection between a Digital Twins instance and an Azure - Data Explorer database. Requires pre-created Azure Data Explorer and Event Hub resources. - long-summary: | - Will prompt the user to add the following roles and permissions on the Digital Twins instance needed to successfully create the connection: - - 'Contributor' role for the Azure Data Explorer Database scope - - 'Database Admin' permission for the Azure Data Explorer Database scope - - 'Azure Event Hubs Data Owner' role for the Event Hub scope - - examples: - - name: Adds a data history connection to a target Digital Twins instance with the $Default Event Hub - consumer group. - text: > - az dt data-history connection create adx -n {instance_name} - --cn {time_series_database_connection_name} - --adx-cluster-name {adx_cluster_name} - --adx-database-name {adx_database_name} - --eventhub {event_hub} - --eventhub-namespace {event_hub_namespace} - - name: Adds a data history connection to a target Digital Twins instance with a custom Azure Data Explorer - table name and Event Hub consumer group. - text: > - az dt data-history connection create adx -n {instance_name} - --cn {time_series_database_connection_name} - --adx-cluster-name {adx_cluster_name} - --adx-database-name {adx_database_name} - --adx-table-name {adx_table_name} - --eventhub {event_hub} - --eventhub-namespace {event_hub_namespace} - --eventhub-consumer-group {event_hub_consumer_group} - - name: Adds a data history connection to a target Digital Twins instance integrating with an Event Hub and Azure - Data Explorer instances in different resource groups and subscriptions from the target instance. - text: > - az dt data-history connection create adx -n {instance_name} - --cn {time_series_database_connection_name} - --adx-cluster-name {adx_cluster_name} - --adx-database-name {adx_database_name} - --adx-resource-group {adx_resource_group} - --adx-subscription {adx_subscription} - --eventhub {event_hub} - --eventhub-namespace {event_hub_namespace} - --eventhub-resource-group {event_hub_resource_group} - --eventhub-subscription {event_subscription} - - name: Adds a data history connection to a target Digital Twins instance with the $Default Event Hub consumer group - and skip the role assignment prompts. - text: > - az dt data-history connection create adx -n {instance_name} -y - --cn {time_series_database_connection_name} - --adx-cluster-name {adx_cluster_name} - --adx-database-name {adx_database_name} - --eventhub {event_hub} - --eventhub-namespace {event_hub_namespace} - """ - - helps["dt data-history connection list"] = """ - type: command - short-summary: List all data history connections configured on a Digital Twins instance. - examples: - - name: List all data history connections configured on an instance. - text: > - az dt data-history connection list -n {instance_name} - """ - - helps["dt data-history connection show"] = """ - type: command - short-summary: Show details of a data history connection configured on a Digital Twins instance. - examples: - - name: Show a data history connection configured on an instance. - text: > - az dt data-history connection show -n {instance_name} --cn {time_series_database_connection_name} - """ - - helps["dt data-history connection wait"] = """ - type: command - short-summary: Wait until an operation on a data history connection is complete. - examples: - - name: Wait until a data history connection is created. - text: > - az dt data-history connection wait -n {instance_name} --cn {time_series_database_connection_name} --created - - name: Wait until an existing data history connection is deleted. - text: > - az dt data-history connection wait -n {instance_name} --cn {time_series_database_connection_name} --deleted - """ - - helps["dt data-history connection delete"] = """ - type: command - short-summary: Delete a data history connection configured on a Digital Twins instance. - examples: - - name: Delete a data history connection configured on an instance - and block until the operation is complete. - text: > - az dt data-history connection delete -n {instance_name} - --cn {time_series_database_connection_name} - - name: Delete a data history connection configured on an instance - without confirmation or blocking. - text: > - az dt data-history connection delete -n {instance_name} - --cn {time_series_database_connection_name} - -y --no-wait - """ + helps["dt data-history"] = """ + type: group + short-summary: Manage and configure data history. + """ + + helps["dt data-history connection"] = """ + type: group + short-summary: Manage and configure data history connections. + """ + + helps["dt data-history connection create"] = """ + type: group + short-summary: Creates a data history connection between a Digital Twins instance and supported resources. + """ + + helps["dt data-history connection create adx"] = """ + type: command + short-summary: Creates a data history connection between a Digital Twins instance and an Azure + Data Explorer database. Requires pre-created Azure Data Explorer and Event Hub resources. + long-summary: | + Will prompt the user to add the following roles and permissions on the Digital Twins instance needed to successfully create the connection: + - 'Contributor' role for the Azure Data Explorer Database scope + - 'Database Admin' permission for the Azure Data Explorer Database scope + - 'Azure Event Hubs Data Owner' role for the Event Hub scope + + examples: + - name: Adds a data history connection to a target Digital Twins instance with the $Default Event Hub + consumer group. + text: > + az dt data-history connection create adx -n {instance_name} + --cn {time_series_database_connection_name} + --adx-cluster-name {adx_cluster_name} + --adx-database-name {adx_database_name} + --eventhub {event_hub} + --eventhub-namespace {event_hub_namespace} + - name: Adds a data history connection to a target Digital Twins instance with a custom Azure Data Explorer + table name and Event Hub consumer group. + text: > + az dt data-history connection create adx -n {instance_name} + --cn {time_series_database_connection_name} + --adx-cluster-name {adx_cluster_name} + --adx-database-name {adx_database_name} + --adx-table-name {adx_table_name} + --eventhub {event_hub} + --eventhub-namespace {event_hub_namespace} + --eventhub-consumer-group {event_hub_consumer_group} + - name: Adds a data history connection to a target Digital Twins instance integrating with an Event Hub and Azure + Data Explorer instances in different resource groups and subscriptions from the target instance. + text: > + az dt data-history connection create adx -n {instance_name} + --cn {time_series_database_connection_name} + --adx-cluster-name {adx_cluster_name} + --adx-database-name {adx_database_name} + --adx-resource-group {adx_resource_group} + --adx-subscription {adx_subscription} + --eventhub {event_hub} + --eventhub-namespace {event_hub_namespace} + --eventhub-resource-group {event_hub_resource_group} + --eventhub-subscription {event_subscription} + - name: Adds a data history connection to a target Digital Twins instance with the $Default Event Hub consumer group + and skip the role assignment prompts. + text: > + az dt data-history connection create adx -n {instance_name} -y + --cn {time_series_database_connection_name} + --adx-cluster-name {adx_cluster_name} + --adx-database-name {adx_database_name} + --eventhub {event_hub} + --eventhub-namespace {event_hub_namespace} + """ + + helps["dt data-history connection list"] = """ + type: command + short-summary: List all data history connections configured on a Digital Twins instance. + examples: + - name: List all data history connections configured on an instance. + text: > + az dt data-history connection list -n {instance_name} + """ + + helps["dt data-history connection show"] = """ + type: command + short-summary: Show details of a data history connection configured on a Digital Twins instance. + examples: + - name: Show a data history connection configured on an instance. + text: > + az dt data-history connection show -n {instance_name} --cn {time_series_database_connection_name} + """ + + helps["dt data-history connection wait"] = """ + type: command + short-summary: Wait until an operation on a data history connection is complete. + examples: + - name: Wait until a data history connection is created. + text: > + az dt data-history connection wait -n {instance_name} --cn {time_series_database_connection_name} --created + - name: Wait until an existing data history connection is deleted. + text: > + az dt data-history connection wait -n {instance_name} --cn {time_series_database_connection_name} --deleted + """ + + helps["dt data-history connection delete"] = """ + type: command + short-summary: Delete a data history connection configured on a Digital Twins instance. + examples: + - name: Delete a data history connection configured on an instance + and block until the operation is complete. + text: > + az dt data-history connection delete -n {instance_name} + --cn {time_series_database_connection_name} + - name: Delete a data history connection configured on an instance + without confirmation or blocking. + text: > + az dt data-history connection delete -n {instance_name} + --cn {time_series_database_connection_name} + -y --no-wait + """ helps["dt endpoint"] = """ type: group diff --git a/azext_iot/digitaltwins/command_map.py b/azext_iot/digitaltwins/command_map.py index d266daed8..f7f3ddc58 100644 --- a/azext_iot/digitaltwins/command_map.py +++ b/azext_iot/digitaltwins/command_map.py @@ -7,7 +7,6 @@ """ Load CLI commands """ -import os from azure.cli.core.commands import CliCommandType digitaltwins_resource_ops = CliCommandType( @@ -31,9 +30,6 @@ ) -enabled_data_history = os.environ.get("ADT_DH_PREVIEW_ENABLE") - - def load_digitaltwins_commands(self, _): """ Load CLI commands @@ -49,31 +45,29 @@ def load_digitaltwins_commands(self, _): cmd_group.wait_command("wait", "wait_instance") cmd_group.command("reset", "reset_instance", confirmation=True, is_preview=True) - # TODO: unhide in with Public preview for data history - if enabled_data_history: - with self.command_group( - "dt data-history", - command_type=digitaltwins_resource_ops, - is_preview=True, - ) as cmd_group: - pass - - with self.command_group( - "dt data-history connection", - command_type=digitaltwins_resource_ops, - ) as cmd_group: - cmd_group.show_command("show", "show_data_connection") - cmd_group.wait_command("wait", "wait_data_connection") - cmd_group.command("list", "list_data_connection") - cmd_group.command( - "delete", "delete_data_connection", confirmation=True, supports_no_wait=True - ) - - with self.command_group( - "dt data-history connection create", - command_type=digitaltwins_resource_ops, - ) as cmd_group: - cmd_group.command("adx", "create_adx_data_connection", supports_no_wait=True) + with self.command_group( + "dt data-history", + command_type=digitaltwins_resource_ops, + is_preview=True, + ) as cmd_group: + pass + + with self.command_group( + "dt data-history connection", + command_type=digitaltwins_resource_ops, + ) as cmd_group: + cmd_group.show_command("show", "show_data_connection") + cmd_group.wait_command("wait", "wait_data_connection") + cmd_group.command("list", "list_data_connection") + cmd_group.command( + "delete", "delete_data_connection", confirmation=True, supports_no_wait=True + ) + + with self.command_group( + "dt data-history connection create", + command_type=digitaltwins_resource_ops, + ) as cmd_group: + cmd_group.command("adx", "create_adx_data_connection", supports_no_wait=True) with self.command_group( "dt endpoint", command_type=digitaltwins_resource_ops diff --git a/azext_iot/digitaltwins/common.py b/azext_iot/digitaltwins/common.py index ef0517971..91de26e77 100644 --- a/azext_iot/digitaltwins/common.py +++ b/azext_iot/digitaltwins/common.py @@ -36,6 +36,10 @@ SKIP_ADD_ROLE_MSG = "Skipping addition of the {0}. This may prevent creation of the data history connection." +# Default Event Hub Consumer Group +DEFAULT_CONSUMER_GROUP = "$Default" + + # Enums class ADTEndpointType(Enum): """ diff --git a/azext_iot/digitaltwins/providers/connection/builders.py b/azext_iot/digitaltwins/providers/connection/builders.py index e51a651a9..6594ca6a8 100644 --- a/azext_iot/digitaltwins/providers/connection/builders.py +++ b/azext_iot/digitaltwins/providers/connection/builders.py @@ -24,7 +24,8 @@ FAIL_GENERIC_MSG, ABORT_MSG, ADD_ROLE_INPUT_MSG, - CONT_INPUT_MSG + CONT_INPUT_MSG, + DEFAULT_CONSUMER_GROUP ) logger = get_logger(__name__) @@ -42,7 +43,7 @@ def __init__( eh_entity_path: str, eh_resource_group: str, eh_subscription: str, - eh_consumer_group: str = '$Default', + eh_consumer_group: str = DEFAULT_CONSUMER_GROUP, yes: bool = False, ): self.cli = EmbeddedCLI() @@ -88,17 +89,19 @@ def validate_eventhub( eh_endpoint.error_prefix = ERROR_PREFIX + " find" self.eh_endpoint_uri = eh_endpoint.build_identity_based().endpoint_uri - eh_consumer_group_op = self.cli.invoke( - "eventhubs eventhub consumer-group show -n {} --eventhub-name {} --namespace-name {} -g {}".format( - eh_consumer_group, - eh_entity_path, - eh_namespace, - eh_resource_group, - ), - subscription=eh_subscription, - ) - if not eh_consumer_group_op.success(): - raise CLIInternalError("{} retrieve Event Hub Consumer Group.".format(ERROR_PREFIX)) + # Run check only if the consumer group is not the default. Default consumer group will always be present. + if eh_consumer_group.lower() != DEFAULT_CONSUMER_GROUP.lower(): + eh_consumer_group_op = self.cli.invoke( + "eventhubs eventhub consumer-group show -n {} --eventhub-name {} --namespace-name {} -g {}".format( + eh_consumer_group, + eh_entity_path, + eh_namespace, + eh_resource_group, + ), + subscription=eh_subscription, + ) + if not eh_consumer_group_op.success(): + raise CLIInternalError("{} retrieve Event Hub Consumer Group.".format(ERROR_PREFIX)) self.eh_namespace_resource_id = ( "/subscriptions/{}/resourceGroups/{}/providers/Microsoft.EventHub/namespaces/{}".format( @@ -226,7 +229,7 @@ def build_adx_connection_properties( adx_subscription: str = None, eh_resource_group: str = None, eh_subscription: str = None, - eh_consumer_group: str = "$Default", + eh_consumer_group: str = DEFAULT_CONSUMER_GROUP, yes: bool = False, ): validator = AdxConnectionValidator( diff --git a/azext_iot/digitaltwins/providers/resource.py b/azext_iot/digitaltwins/providers/resource.py index 6797450c8..db3cd2fb3 100644 --- a/azext_iot/digitaltwins/providers/resource.py +++ b/azext_iot/digitaltwins/providers/resource.py @@ -10,6 +10,7 @@ ResourceNotFoundError, ) from azext_iot.digitaltwins.common import ( + DEFAULT_CONSUMER_GROUP, MAX_ADT_DH_CREATE_RETRIES, ADTEndpointAuthType, ADTPublicNetworkAccessType, @@ -499,7 +500,7 @@ def create_adx_data_connection( adx_table_name=None, adx_resource_group=None, adx_subscription=None, - eh_consumer_group="$Default", + eh_consumer_group=DEFAULT_CONSUMER_GROUP, eh_resource_group=None, eh_subscription=None, resource_group_name=None, diff --git a/azext_iot/tests/digitaltwins/test_dt_data_history_lifecycle_int.py b/azext_iot/tests/digitaltwins/test_dt_data_history_lifecycle_int.py index d0ca3e9e3..4d48dd6b1 100644 --- a/azext_iot/tests/digitaltwins/test_dt_data_history_lifecycle_int.py +++ b/azext_iot/tests/digitaltwins/test_dt_data_history_lifecycle_int.py @@ -4,7 +4,6 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import os from time import sleep from knack.log import get_logger import pytest @@ -22,11 +21,8 @@ ) logger = get_logger(__name__) -ADH_REGION = "northeurope" -enabled_data_history = os.environ.get("ADT_DH_PREVIEW_ENABLE") -@pytest.mark.skipif(not enabled_data_history, reason="Wait until Data History Public Preview") class TestDTConnections(DTLiveScenarioTest): def __init__(self, test_case): super(TestDTConnections, self).__init__(test_case) @@ -74,7 +70,7 @@ def tearDownSuite(self): ) def test_dt_data_history_adx(self): - self.wait_for_capacity(region=ADH_REGION) + self.wait_for_capacity() instance_name = f"dt{generate_generic_id()}" connection_name = f"cn-{generate_generic_id()}" table_name = f"tb_{generate_generic_id()}" @@ -82,10 +78,9 @@ def test_dt_data_history_adx(self): self.add_eventhub_consumer_group(consumer_group=consumer_group) create_output = self.cmd( - "dt create -n {} -g {} -l {} --assign-identity".format( + "dt create -n {} -g {} --assign-identity".format( instance_name, self.rg, - ADH_REGION ) ).get_output_in_json() self.track_instance(create_output) @@ -216,15 +211,14 @@ def test_dt_data_history_adx(self): assert len(list_result) == 0 def test_dt_data_history_adx_create_incorrect_resource(self): - self.wait_for_capacity(region=ADH_REGION) + self.wait_for_capacity() instance_name = f"dt{generate_generic_id()}" connection_name = f"cn-{generate_generic_id()}" create_output = self.cmd( - "dt create -n {} -g {} -l {} --assign-identity".format( + "dt create -n {} -g {} --assign-identity".format( instance_name, self.rg, - ADH_REGION ) ).get_output_in_json() self.track_instance(create_output) @@ -333,17 +327,16 @@ def test_dt_data_history_adx_create_incorrect_resource(self): ) def test_dt_data_history_adx_wait(self): - self.wait_for_capacity(region=ADH_REGION) + self.wait_for_capacity() instance_name = f"dt{generate_generic_id()}" connection_name = f"cn-{generate_generic_id()}" consumer_group = f"cg-{generate_generic_id()}" self.add_eventhub_consumer_group(consumer_group=consumer_group) create_output = self.cmd( - "dt create -n {} -g {} -l {} --assign-identity".format( + "dt create -n {} -g {} --assign-identity".format( instance_name, self.rg, - ADH_REGION ) ).get_output_in_json() self.track_instance(create_output)