From 04635564d8bb61212c2d78392ac21516b75ccf63 Mon Sep 17 00:00:00 2001 From: Leszek Jakubowski Date: Tue, 14 Sep 2021 14:24:11 +0200 Subject: [PATCH 1/6] Adding support for the v2021-09-01-preview API - disk encryption sets, sdn selection, encryption at host options --- python/az/aro/HISTORY.rst | 4 ++ python/az/aro/azext_aro/_client_factory.py | 2 +- python/az/aro/azext_aro/_params.py | 15 ++++++ python/az/aro/azext_aro/_rbac.py | 11 +++-- python/az/aro/azext_aro/_validators.py | 34 ++++++++++++++ python/az/aro/azext_aro/commands.py | 2 +- python/az/aro/azext_aro/custom.py | 54 ++++++++++++++-------- 7 files changed, 96 insertions(+), 26 deletions(-) diff --git a/python/az/aro/HISTORY.rst b/python/az/aro/HISTORY.rst index 8a86c83c09c..f7f5057c81b 100644 --- a/python/az/aro/HISTORY.rst +++ b/python/az/aro/HISTORY.rst @@ -25,3 +25,7 @@ Release History 1.0.0 ++++++ * Remove preview flag. + +1.0.1 +++++++ +* Switch to new preview API diff --git a/python/az/aro/azext_aro/_client_factory.py b/python/az/aro/azext_aro/_client_factory.py index e03b5a16c46..9963abc2161 100644 --- a/python/az/aro/azext_aro/_client_factory.py +++ b/python/az/aro/azext_aro/_client_factory.py @@ -4,7 +4,7 @@ import urllib3 from azext_aro.custom import rp_mode_development -from azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2020_04_30 import AzureRedHatOpenShiftClient +from azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2021_09_01_preview import AzureRedHatOpenShiftClient from azure.cli.core.commands.client_factory import get_mgmt_service_client diff --git a/python/az/aro/azext_aro/_params.py b/python/az/aro/azext_aro/_params.py index db41c9fab90..6376319e541 100644 --- a/python/az/aro/azext_aro/_params.py +++ b/python/az/aro/azext_aro/_params.py @@ -4,8 +4,11 @@ from azext_aro._validators import validate_cidr from azext_aro._validators import validate_client_id from azext_aro._validators import validate_cluster_resource_group +from azext_aro._validators import validate_disk_encryption_set from azext_aro._validators import validate_domain +from azext_aro._validators import validate_encryption_at_host from azext_aro._validators import validate_pull_secret +from azext_aro._validators import validate_sdn from azext_aro._validators import validate_subnet from azext_aro._validators import validate_client_secret from azext_aro._validators import validate_visibility @@ -54,10 +57,22 @@ def load_arguments(self, _): c.argument('service_cidr', help='CIDR of service network. Must be a minimum of /18 or larger.', validator=validate_cidr('service_cidr')) + c.argument('software_defined_network', arg_type=get_enum_type(['OVNKubernetes', 'OpenShiftSDN']), + help='SDN type either "OVNKubernetes" (default) or "OpenShiftSDN"', + validator=validate_sdn) + c.argument('disk_encryption_set', + help='ResourceID of the DiskEncryptionSet to be used for master and worker VMs.', + validator=validate_disk_encryption_set) + c.argument('master_encryption_at_host', arg_type=get_enum_type(['Enabled', 'Disabled']), + help='Encryption at host flag for master VMs. Correct values are "Enabled" or "Disabled" (default)', + validator=validate_encryption_at_host('master_encryption_at_host')) c.argument('master_vm_size', help='Size of master VMs.') + c.argument('worker_encryption_at_host', arg_type=get_enum_type(['Enabled', 'Disabled']), + help='Encryption at host flag for worker VMs. Correct values are "Enabled" or "Disabled" (default)', + validator=validate_encryption_at_host('worker_encryption_at_host')) c.argument('worker_vm_size', help='Size of worker VMs.') c.argument('worker_vm_disk_size_gb', diff --git a/python/az/aro/azext_aro/_rbac.py b/python/az/aro/azext_aro/_rbac.py index adcd7ffec8e..f096004d7b8 100644 --- a/python/az/aro/azext_aro/_rbac.py +++ b/python/az/aro/azext_aro/_rbac.py @@ -11,7 +11,8 @@ from msrest.exceptions import ValidationError from msrestazure.tools import resource_id -NETWORK_CONTRIBUTOR = '4d97b98b-1d4f-4787-a291-c67834d212e7' +ROLE_NETWORK_CONTRIBUTOR = '4d97b98b-1d4f-4787-a291-c67834d212e7' +ROLE_READER = 'acdd72a7-3385-48ef-bd42-f606fba81ae7' logger = get_logger(__name__) @@ -34,7 +35,7 @@ def _create_role_assignment(auth_client, resource, params): logger.warning("%s; retry %d of %d", ex, retries, max_retries) -def assign_network_contributor_to_resource(cli_ctx, resource, object_id): +def assign_role_to_resource(cli_ctx, resource, object_id, role_name): auth_client = get_mgmt_service_client(cli_ctx, ResourceType.MGMT_AUTHORIZATION) RoleAssignmentCreateParameters = get_sdk(cli_ctx, ResourceType.MGMT_AUTHORIZATION, @@ -45,7 +46,7 @@ def assign_network_contributor_to_resource(cli_ctx, resource, object_id): subscription=get_subscription_id(cli_ctx), namespace='Microsoft.Authorization', type='roleDefinitions', - name=NETWORK_CONTRIBUTOR, + name=role_name, ) _create_role_assignment(auth_client, resource, RoleAssignmentCreateParameters( @@ -55,14 +56,14 @@ def assign_network_contributor_to_resource(cli_ctx, resource, object_id): )) -def has_network_contributor_on_resource(cli_ctx, resource, object_id): +def has_role_assignment_on_resource(cli_ctx, resource, object_id, role_name): auth_client = get_mgmt_service_client(cli_ctx, ResourceType.MGMT_AUTHORIZATION) role_definition_id = resource_id( subscription=get_subscription_id(cli_ctx), namespace='Microsoft.Authorization', type='roleDefinitions', - name=NETWORK_CONTRIBUTOR, + name=role_name, ) for assignment in auth_client.role_assignments.list_for_scope(resource): diff --git a/python/az/aro/azext_aro/_validators.py b/python/az/aro/azext_aro/_validators.py index e0a58082495..5bc22f21d5a 100644 --- a/python/az/aro/azext_aro/_validators.py +++ b/python/az/aro/azext_aro/_validators.py @@ -64,6 +64,21 @@ def validate_cluster_resource_group(cmd, namespace): namespace.cluster_resource_group) +def validate_disk_encryption_set(cmd, namespace): + if namespace.disk_encryption_set is not None: + if not is_valid_resource_id(namespace.disk_encryption_set): + raise InvalidArgumentValueError( + "Invalid --disk-encryption-set '%s', has to be a resource ID." % + namespace.disk_encryption_set) + + desid = parse_resource_id(namespace.disk_encryption_set) + compute_client = get_mgmt_service_client(cmd.cli_ctx, ResourceType.MGMT_COMPUTE) + try: + compute_client.disk_encryption_sets.get(resource_group_name=desid['resource_group'], disk_encryption_set_name=desid['name']) + except CloudError as err: + raise CLIInternalError(err.message) from err + + def validate_domain(namespace): if namespace.domain is not None: if not re.match(r'^' + @@ -74,6 +89,18 @@ def validate_domain(namespace): namespace.domain) +def validate_encryption_at_host(key): + def _validate_encryption_at_host(namespace): + eah = getattr(namespace, key) + if eah is not None: + eah = eah.capitalize() + if eah not in ['Enabled', 'Disabled']: + raise InvalidArgumentValueError("Invalid --%s '%s'." % + (key.replace('_', '-'), eah)) + + return _validate_encryption_at_host + + def validate_pull_secret(namespace): if namespace.pull_secret is None: # TODO: add aka.ms link here @@ -90,6 +117,13 @@ def validate_pull_secret(namespace): raise InvalidArgumentValueError("Invalid --pull-secret.") from e +def validate_sdn(namespace): + if namespace.software_defined_network is not None: + if namespace.software_defined_network not in ['OVNKubernetes', 'OpenshiftSDN']: + raise InvalidArgumentValueError("Invalid --software-defined-network '%s'." % + namespace.software_defined_network) + + def validate_subnet(key): def _validate_subnet(cmd, namespace): subnet = getattr(namespace, key) diff --git a/python/az/aro/azext_aro/commands.py b/python/az/aro/azext_aro/commands.py index 29665ba945e..f0ef86ae764 100644 --- a/python/az/aro/azext_aro/commands.py +++ b/python/az/aro/azext_aro/commands.py @@ -10,7 +10,7 @@ def load_command_table(self, _): aro_sdk = CliCommandType( - operations_tmpl='azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2020_04_30.operations#OpenShiftClustersOperations.{}', # pylint: disable=line-too-long + operations_tmpl='azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2021_09_01_preview.operations#OpenShiftClustersOperations.{}', # pylint: disable=line-too-long client_factory=cf_aro) with self.command_group('aro', aro_sdk, client_factory=cf_aro) as g: diff --git a/python/az/aro/azext_aro/custom.py b/python/az/aro/azext_aro/custom.py index fcdf60379c1..0c94ddad6c8 100644 --- a/python/az/aro/azext_aro/custom.py +++ b/python/az/aro/azext_aro/custom.py @@ -15,10 +15,10 @@ from msrest.exceptions import HttpOperationError from knack.log import get_logger -import azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2020_04_30.models as openshiftcluster +import azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2021_09_01_preview.models as openshiftcluster from azext_aro._aad import AADManager -from azext_aro._rbac import assign_network_contributor_to_resource, has_network_contributor_on_resource +from azext_aro._rbac import assign_role_to_resource, has_role_assignment_on_resource, ROLE_NETWORK_CONTRIBUTOR, ROLE_READER from azext_aro._validators import validate_subnets logger = get_logger(__name__) @@ -42,7 +42,11 @@ def aro_create(cmd, # pylint: disable=too-many-locals client_secret=None, pod_cidr=None, service_cidr=None, + software_defined_network=None, + disk_encryption_set=None, + master_encryption_at_host=None, master_vm_size=None, + worker_encryption_at_host=None, worker_vm_size=None, worker_vm_disk_size_gb=None, worker_count=None, @@ -104,10 +108,13 @@ def aro_create(cmd, # pylint: disable=too-many-locals network_profile=openshiftcluster.NetworkProfile( pod_cidr=pod_cidr or '10.128.0.0/14', service_cidr=service_cidr or '172.30.0.0/16', + software_defined_network=software_defined_network or 'OVNKubernetes' ), master_profile=openshiftcluster.MasterProfile( vm_size=master_vm_size or 'Standard_D8s_v3', subnet_id=master_subnet, + encryption_at_host=master_encryption_at_host or 'Disabled', + disk_encryption_set_id=disk_encryption_set, ), worker_profiles=[ openshiftcluster.WorkerProfile( @@ -116,6 +123,8 @@ def aro_create(cmd, # pylint: disable=too-many-locals disk_size_gb=worker_vm_disk_size_gb or 128, subnet_id=worker_subnet, count=worker_count or 3, + encryption_at_host=worker_encryption_at_host or 'Disabled', + disk_encryption_set_id=disk_encryption_set, ) ], apiserver_profile=openshiftcluster.APIServerProfile( @@ -278,6 +287,11 @@ def get_network_resources(cli_ctx, subnets, vnet): return resources +def get_disk_encryption_resources(cli_ctx, oc): + disk_encryption_set = oc.master_profile.disk_encryption_set_id + resources = set() + resources.add(disk_encryption_set) + return resources # cluster_application_update manages cluster application & service principal update # If called without parameters it should be best-effort @@ -363,8 +377,9 @@ def resolve_rp_client_id(): def ensure_resource_permissions(cli_ctx, oc, fail, sp_obj_ids): try: - # Get cluster resources we need to assign network contributor on - resources = get_cluster_network_resources(cli_ctx, oc) + # Get cluster resources we need to assign permissions on, sort to ensure the same order of operations + resources = {ROLE_NETWORK_CONTRIBUTOR: sorted(get_cluster_network_resources(cli_ctx, oc)), + ROLE_READER: sorted(get_disk_encryption_resources(cli_ctx, oc))} except (CloudError, HttpOperationError) as e: if fail: logger.error(e.message) @@ -373,18 +388,19 @@ def ensure_resource_permissions(cli_ctx, oc, fail, sp_obj_ids): return for sp_id in sp_obj_ids: - for resource in sorted(resources): - # Create the role assignment if it doesn't exist - # Assume that the role assignment exists if we fail to look it up - resource_contributor_exists = True - - try: - resource_contributor_exists = has_network_contributor_on_resource(cli_ctx, resource, sp_id) - except CloudError as e: - if fail: - logger.error(e.message) - raise - logger.info(e.message) - - if not resource_contributor_exists: - assign_network_contributor_to_resource(cli_ctx, resource, sp_id) + for role in resources: + for resource in resources[role]: + # Create the role assignment if it doesn't exist + # Assume that the role assignment exists if we fail to look it up + resource_contributor_exists = True + + try: + resource_contributor_exists = has_role_assignment_on_resource(cli_ctx, resource, sp_id, role) + except CloudError as e: + if fail: + logger.error(e.message) + raise + logger.info(e.message) + + if not resource_contributor_exists: + assign_role_to_resource(cli_ctx, resource, sp_id, role) From 6d6c922d8f6f9091e4b0f4bb81b6f17768ecdabd Mon Sep 17 00:00:00 2001 From: Leszek Jakubowski Date: Wed, 15 Sep 2021 13:08:16 +0200 Subject: [PATCH 2/6] Adding doc about how to use the disk-encryption-set --- docs/disk-encryption-set.md | 62 +++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 docs/disk-encryption-set.md diff --git a/docs/disk-encryption-set.md b/docs/disk-encryption-set.md new file mode 100644 index 00000000000..f25f560e1d6 --- /dev/null +++ b/docs/disk-encryption-set.md @@ -0,0 +1,62 @@ +# Using a custom Disk Encryption Set + +## What is the Disk Encryption Set used for? + +In summary: it allows the customer to control the keys that are used to encrypt/decrypt the VM disks. +See https://docs.microsoft.com/en-us/azure/virtual-machines/disks-enable-host-based-encryption-portal#deploy-a-vm-with-customer-managed-keys for more information. + +## How to deploy? +First install and use the AzureCLI extension with +``` +make az +``` + +You can check if the extension is in use by running +``` +$ az extension list + +[ + { + "experimental": false, + "extensionType": "dev", + "name": "aro", + "path": "/github.com/Azure/ARO-RP/python/az/aro", + "preview": true, + "version": "1.0.1" + } +``` + +Follow https://docs.microsoft.com/en-us/azure/openshift/tutorial-create-cluster but don't run the `az aro create` command. Instead: + + - set additional env variables +``` +export KEYVAULT_NAME=$USER-enckv +export KEYVAULT_KEY_NAME=$USER-key +export DISK_ENCRYPTION_SET_NAME=$USER-des +``` + - create the KeyVault and Key +``` + +az keyvault create -n $KEYVAULT_NAME -g $RESOURCEGROUP -l $LOCATION --enable-purge-protection true --enable-soft-delete true + +az keyvault key create --vault-name $KEYVAULT_NAME -n $KEYVAULT_KEY_NAME --protection software + +KEYVAULT_ID=$(az keyvault show --name $KEYVAULT_NAME --query "[id]" -o tsv) + +KEYVAULT_KEY_URL=$(az keyvault key show --vault-name $KEYVAULT_NAME --name $KEYVAULT_KEY_NAME --query "[key.kid]" -o tsv) +``` + - create the DES and add permissions to use the KeyVault +``` +az disk-encryption-set create -n $DISK_ENCRYPTION_SET_NAME -l $LOCATION -g $RESOURCEGROUP --source-vault $KEYVAULT_ID --key-url $KEYVAULT_KEY_URL + +DES_IDENTITY=$(az disk-encryption-set show -n $DISK_ENCRYPTION_SET_NAME -g $RESOURCEGROUP --query "[identity.principalId]" -o tsv) + +az keyvault set-policy -n $KEYVAULT_NAME -g $RESOURCEGROUP --object-id $DES_IDENTITY --key-permissions wrapkey unwrapkey get +``` + - run the az aro create command +``` +az aro create --resource-group $RESOURCEGROUP --name $CLUSTER --vnet aro-vnet --master-subnet master-subnet --worker-subnet worker-subnet --disk-encryption-set $DES_ID +``` + +After creating the cluster all VMs should have the customer controlled Disk Encryption Set. +Remember to delete the disk-encryption-set and keyvault after you're done. From 417dad32181644edf6ccc6095b468b9772b8ef74 Mon Sep 17 00:00:00 2001 From: Leszek Jakubowski Date: Thu, 16 Sep 2021 15:40:40 +0200 Subject: [PATCH 3/6] Shortening az parameter names, switching default SDN to OpenShiftSDN --- python/az/aro/azext_aro/_params.py | 5 ++++- python/az/aro/azext_aro/_validators.py | 3 ++- python/az/aro/azext_aro/custom.py | 11 +++++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/python/az/aro/azext_aro/_params.py b/python/az/aro/azext_aro/_params.py index 6376319e541..0b309a75e81 100644 --- a/python/az/aro/azext_aro/_params.py +++ b/python/az/aro/azext_aro/_params.py @@ -58,19 +58,22 @@ def load_arguments(self, _): help='CIDR of service network. Must be a minimum of /18 or larger.', validator=validate_cidr('service_cidr')) c.argument('software_defined_network', arg_type=get_enum_type(['OVNKubernetes', 'OpenShiftSDN']), - help='SDN type either "OVNKubernetes" (default) or "OpenShiftSDN"', + options_list=['--sdn-type'], + help='SDN type either "OVNKubernetes" or "OpenShiftSDN (default)"', validator=validate_sdn) c.argument('disk_encryption_set', help='ResourceID of the DiskEncryptionSet to be used for master and worker VMs.', validator=validate_disk_encryption_set) c.argument('master_encryption_at_host', arg_type=get_enum_type(['Enabled', 'Disabled']), + options_list=['--master-enc-at-host'], help='Encryption at host flag for master VMs. Correct values are "Enabled" or "Disabled" (default)', validator=validate_encryption_at_host('master_encryption_at_host')) c.argument('master_vm_size', help='Size of master VMs.') c.argument('worker_encryption_at_host', arg_type=get_enum_type(['Enabled', 'Disabled']), + options_list=['--worker-enc-at-host'], help='Encryption at host flag for worker VMs. Correct values are "Enabled" or "Disabled" (default)', validator=validate_encryption_at_host('worker_encryption_at_host')) c.argument('worker_vm_size', diff --git a/python/az/aro/azext_aro/_validators.py b/python/az/aro/azext_aro/_validators.py index 5bc22f21d5a..f4fc77eddca 100644 --- a/python/az/aro/azext_aro/_validators.py +++ b/python/az/aro/azext_aro/_validators.py @@ -74,7 +74,8 @@ def validate_disk_encryption_set(cmd, namespace): desid = parse_resource_id(namespace.disk_encryption_set) compute_client = get_mgmt_service_client(cmd.cli_ctx, ResourceType.MGMT_COMPUTE) try: - compute_client.disk_encryption_sets.get(resource_group_name=desid['resource_group'], disk_encryption_set_name=desid['name']) + compute_client.disk_encryption_sets.get(resource_group_name=desid['resource_group'], + disk_encryption_set_name=desid['name']) except CloudError as err: raise CLIInternalError(err.message) from err diff --git a/python/az/aro/azext_aro/custom.py b/python/az/aro/azext_aro/custom.py index 0c94ddad6c8..3972d1ea1d5 100644 --- a/python/az/aro/azext_aro/custom.py +++ b/python/az/aro/azext_aro/custom.py @@ -18,7 +18,8 @@ import azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2021_09_01_preview.models as openshiftcluster from azext_aro._aad import AADManager -from azext_aro._rbac import assign_role_to_resource, has_role_assignment_on_resource, ROLE_NETWORK_CONTRIBUTOR, ROLE_READER +from azext_aro._rbac import assign_role_to_resource, has_role_assignment_on_resource +from azext_aro._rbac import ROLE_NETWORK_CONTRIBUTOR, ROLE_READER from azext_aro._validators import validate_subnets logger = get_logger(__name__) @@ -108,7 +109,7 @@ def aro_create(cmd, # pylint: disable=too-many-locals network_profile=openshiftcluster.NetworkProfile( pod_cidr=pod_cidr or '10.128.0.0/14', service_cidr=service_cidr or '172.30.0.0/16', - software_defined_network=software_defined_network or 'OVNKubernetes' + software_defined_network=software_defined_network or 'OpenShiftSDN' ), master_profile=openshiftcluster.MasterProfile( vm_size=master_vm_size or 'Standard_D8s_v3', @@ -287,12 +288,14 @@ def get_network_resources(cli_ctx, subnets, vnet): return resources -def get_disk_encryption_resources(cli_ctx, oc): + +def get_disk_encryption_resources(oc): disk_encryption_set = oc.master_profile.disk_encryption_set_id resources = set() resources.add(disk_encryption_set) return resources + # cluster_application_update manages cluster application & service principal update # If called without parameters it should be best-effort # If called with parameters it fails if something is not possible @@ -379,7 +382,7 @@ def ensure_resource_permissions(cli_ctx, oc, fail, sp_obj_ids): try: # Get cluster resources we need to assign permissions on, sort to ensure the same order of operations resources = {ROLE_NETWORK_CONTRIBUTOR: sorted(get_cluster_network_resources(cli_ctx, oc)), - ROLE_READER: sorted(get_disk_encryption_resources(cli_ctx, oc))} + ROLE_READER: sorted(get_disk_encryption_resources(oc))} except (CloudError, HttpOperationError) as e: if fail: logger.error(e.message) From fd22ba2b467107287bfc4dba6b541cae1ed90997 Mon Sep 17 00:00:00 2001 From: Peter Kostyukov Date: Tue, 21 Sep 2021 14:10:24 -0500 Subject: [PATCH 4/6] Documentation cleanup --- docs/disk-encryption-set.md | 68 ++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/docs/disk-encryption-set.md b/docs/disk-encryption-set.md index f25f560e1d6..e4ae331d481 100644 --- a/docs/disk-encryption-set.md +++ b/docs/disk-encryption-set.md @@ -2,19 +2,18 @@ ## What is the Disk Encryption Set used for? -In summary: it allows the customer to control the keys that are used to encrypt/decrypt the VM disks. -See https://docs.microsoft.com/en-us/azure/virtual-machines/disks-enable-host-based-encryption-portal#deploy-a-vm-with-customer-managed-keys for more information. +In summary, it allows a customer to control the keys that are used to encrypt/decrypt VM disks. +See [deploy-a-vm-with-customer-managed-keys](https://docs.microsoft.com/en-us/azure/virtual-machines/disks-enable-host-based-encryption-portal#deploy-a-vm-with-customer-managed-keys) for more information. ## How to deploy? -First install and use the AzureCLI extension with -``` +First, install and use the AzureCLI extension with +```bash make az ``` -You can check if the extension is in use by running -``` -$ az extension list - +>You can check if the extension is in use by running: +```bash +az extension list [ { "experimental": false, @@ -24,39 +23,62 @@ $ az extension list "preview": true, "version": "1.0.1" } +] ``` -Follow https://docs.microsoft.com/en-us/azure/openshift/tutorial-create-cluster but don't run the `az aro create` command. Instead: +Follow [tutorial-create-cluster](https://docs.microsoft.com/en-us/azure/openshift/tutorial-create-cluster) but don't run the `az aro create` command, instead proceed as follows: - set additional env variables -``` +```bash export KEYVAULT_NAME=$USER-enckv export KEYVAULT_KEY_NAME=$USER-key export DISK_ENCRYPTION_SET_NAME=$USER-des ``` - create the KeyVault and Key -``` - -az keyvault create -n $KEYVAULT_NAME -g $RESOURCEGROUP -l $LOCATION --enable-purge-protection true --enable-soft-delete true +```bash +az keyvault create -n $KEYVAULT_NAME \ + -g $RESOURCEGROUP \ + -l $LOCATION \ + --enable-purge-protection true \ + --enable-soft-delete true -az keyvault key create --vault-name $KEYVAULT_NAME -n $KEYVAULT_KEY_NAME --protection software +az keyvault key create --vault-name $KEYVAULT_NAME \ + -n $KEYVAULT_KEY_NAME \ + --protection software KEYVAULT_ID=$(az keyvault show --name $KEYVAULT_NAME --query "[id]" -o tsv) -KEYVAULT_KEY_URL=$(az keyvault key show --vault-name $KEYVAULT_NAME --name $KEYVAULT_KEY_NAME --query "[key.kid]" -o tsv) +KEYVAULT_KEY_URL=$(az keyvault key show --vault-name $KEYVAULT_NAME \ + --name $KEYVAULT_KEY_NAME \ + --query "[key.kid]" -o tsv) ``` - create the DES and add permissions to use the KeyVault -``` -az disk-encryption-set create -n $DISK_ENCRYPTION_SET_NAME -l $LOCATION -g $RESOURCEGROUP --source-vault $KEYVAULT_ID --key-url $KEYVAULT_KEY_URL +```bash +az disk-encryption-set create -n $DISK_ENCRYPTION_SET_NAME \ + -l $LOCATION \ + -g $RESOURCEGROUP \ + --source-vault $KEYVAULT_ID \ + --key-url $KEYVAULT_KEY_URL -DES_IDENTITY=$(az disk-encryption-set show -n $DISK_ENCRYPTION_SET_NAME -g $RESOURCEGROUP --query "[identity.principalId]" -o tsv) +DES_IDENTITY=$(az disk-encryption-set show -n $DISK_ENCRYPTION_SET_NAME \ + -g $RESOURCEGROUP \ + --query "[identity.principalId]" \ + -o tsv) -az keyvault set-policy -n $KEYVAULT_NAME -g $RESOURCEGROUP --object-id $DES_IDENTITY --key-permissions wrapkey unwrapkey get -``` - - run the az aro create command +az keyvault set-policy -n $KEYVAULT_NAME \ + -g $RESOURCEGROUP \ + --object-id $DES_IDENTITY \ + --key-permissions wrapkey unwrapkey get ``` -az aro create --resource-group $RESOURCEGROUP --name $CLUSTER --vnet aro-vnet --master-subnet master-subnet --worker-subnet worker-subnet --disk-encryption-set $DES_ID + - run the az aro create command +```bash +az aro create --resource-group $RESOURCEGROUP \ + --name $CLUSTER \ + --vnet aro-vnet \ + --master-subnet master-subnet \ + --worker-subnet worker-subnet \ + --disk-encryption-set $DES_ID ``` After creating the cluster all VMs should have the customer controlled Disk Encryption Set. -Remember to delete the disk-encryption-set and keyvault after you're done. +>Remember to delete the disk-encryption-set and keyvault when done. From b6451f3e640a0dd263cd172a8de4e26a74a25a72 Mon Sep 17 00:00:00 2001 From: Leszek Jakubowski Date: Mon, 4 Oct 2021 14:20:49 +0200 Subject: [PATCH 5/6] fixing params and validation --- python/az/aro/azext_aro/_params.py | 19 ++++++++----------- python/az/aro/azext_aro/_validators.py | 18 +++--------------- python/az/aro/azext_aro/custom.py | 10 +++++----- 3 files changed, 16 insertions(+), 31 deletions(-) diff --git a/python/az/aro/azext_aro/_params.py b/python/az/aro/azext_aro/_params.py index 0b309a75e81..01f48f5c371 100644 --- a/python/az/aro/azext_aro/_params.py +++ b/python/az/aro/azext_aro/_params.py @@ -6,7 +6,6 @@ from azext_aro._validators import validate_cluster_resource_group from azext_aro._validators import validate_disk_encryption_set from azext_aro._validators import validate_domain -from azext_aro._validators import validate_encryption_at_host from azext_aro._validators import validate_pull_secret from azext_aro._validators import validate_sdn from azext_aro._validators import validate_subnet @@ -58,24 +57,22 @@ def load_arguments(self, _): help='CIDR of service network. Must be a minimum of /18 or larger.', validator=validate_cidr('service_cidr')) c.argument('software_defined_network', arg_type=get_enum_type(['OVNKubernetes', 'OpenShiftSDN']), - options_list=['--sdn-type'], - help='SDN type either "OVNKubernetes" or "OpenShiftSDN (default)"', + options_list=['--software-defined-network-type', '--sdn-type'], + help='SDN type either "OpenShiftSDN (default)" or "OVNKubernetes"', validator=validate_sdn) c.argument('disk_encryption_set', help='ResourceID of the DiskEncryptionSet to be used for master and worker VMs.', validator=validate_disk_encryption_set) - c.argument('master_encryption_at_host', arg_type=get_enum_type(['Enabled', 'Disabled']), - options_list=['--master-enc-at-host'], - help='Encryption at host flag for master VMs. Correct values are "Enabled" or "Disabled" (default)', - validator=validate_encryption_at_host('master_encryption_at_host')) + c.argument('master_encryption_at_host', arg_type=get_three_state_flag(), + options_list=['--master-encryption-at-host', '--master-enc-host'], + help='Encryption at host flag for master VMs.') c.argument('master_vm_size', help='Size of master VMs.') - c.argument('worker_encryption_at_host', arg_type=get_enum_type(['Enabled', 'Disabled']), - options_list=['--worker-enc-at-host'], - help='Encryption at host flag for worker VMs. Correct values are "Enabled" or "Disabled" (default)', - validator=validate_encryption_at_host('worker_encryption_at_host')) + c.argument('worker_encryption_at_host', arg_type=get_three_state_flag(), + options_list=['--worker-encryption-at-host', '--worker-enc-host'], + help='Encryption at host flag for worker VMs.') c.argument('worker_vm_size', help='Size of worker VMs.') c.argument('worker_vm_disk_size_gb', diff --git a/python/az/aro/azext_aro/_validators.py b/python/az/aro/azext_aro/_validators.py index f4fc77eddca..8e52f3adf21 100644 --- a/python/az/aro/azext_aro/_validators.py +++ b/python/az/aro/azext_aro/_validators.py @@ -9,7 +9,7 @@ from azure.cli.core.commands.client_factory import get_mgmt_service_client from azure.cli.core.commands.client_factory import get_subscription_id from azure.cli.core.profiles import ResourceType -from azure.cli.core.azclierror import CLIInternalError, InvalidArgumentValueError, \ +from azure.cli.core.azclierror import InvalidArgumentValueError, \ RequiredArgumentMissingError from knack.log import get_logger from msrestazure.azure_exceptions import CloudError @@ -77,7 +77,7 @@ def validate_disk_encryption_set(cmd, namespace): compute_client.disk_encryption_sets.get(resource_group_name=desid['resource_group'], disk_encryption_set_name=desid['name']) except CloudError as err: - raise CLIInternalError(err.message) from err + raise InvalidArgumentValueError(err.message) from err def validate_domain(namespace): @@ -90,18 +90,6 @@ def validate_domain(namespace): namespace.domain) -def validate_encryption_at_host(key): - def _validate_encryption_at_host(namespace): - eah = getattr(namespace, key) - if eah is not None: - eah = eah.capitalize() - if eah not in ['Enabled', 'Disabled']: - raise InvalidArgumentValueError("Invalid --%s '%s'." % - (key.replace('_', '-'), eah)) - - return _validate_encryption_at_host - - def validate_pull_secret(namespace): if namespace.pull_secret is None: # TODO: add aka.ms link here @@ -171,7 +159,7 @@ def _validate_subnet(cmd, namespace): client.subnets.get(parts['resource_group'], parts['name'], parts['child_name_1']) except CloudError as err: - raise CLIInternalError(err.message) from err + raise InvalidArgumentValueError(err.message) from err return _validate_subnet diff --git a/python/az/aro/azext_aro/custom.py b/python/az/aro/azext_aro/custom.py index 3972d1ea1d5..f557fc37d9f 100644 --- a/python/az/aro/azext_aro/custom.py +++ b/python/az/aro/azext_aro/custom.py @@ -45,9 +45,9 @@ def aro_create(cmd, # pylint: disable=too-many-locals service_cidr=None, software_defined_network=None, disk_encryption_set=None, - master_encryption_at_host=None, + master_encryption_at_host=False, master_vm_size=None, - worker_encryption_at_host=None, + worker_encryption_at_host=False, worker_vm_size=None, worker_vm_disk_size_gb=None, worker_count=None, @@ -114,7 +114,7 @@ def aro_create(cmd, # pylint: disable=too-many-locals master_profile=openshiftcluster.MasterProfile( vm_size=master_vm_size or 'Standard_D8s_v3', subnet_id=master_subnet, - encryption_at_host=master_encryption_at_host or 'Disabled', + encryption_at_host='Enabled' if master_encryption_at_host else 'Disabled', disk_encryption_set_id=disk_encryption_set, ), worker_profiles=[ @@ -124,7 +124,7 @@ def aro_create(cmd, # pylint: disable=too-many-locals disk_size_gb=worker_vm_disk_size_gb or 128, subnet_id=worker_subnet, count=worker_count or 3, - encryption_at_host=worker_encryption_at_host or 'Disabled', + encryption_at_host='Enabled' if worker_encryption_at_host else 'Disabled', disk_encryption_set_id=disk_encryption_set, ) ], @@ -391,7 +391,7 @@ def ensure_resource_permissions(cli_ctx, oc, fail, sp_obj_ids): return for sp_id in sp_obj_ids: - for role in resources: + for role in sorted(resources): for resource in resources[role]: # Create the role assignment if it doesn't exist # Assume that the role assignment exists if we fail to look it up From d8d13bd1aa12e674ceed2fdb5e627164a0db35f2 Mon Sep 17 00:00:00 2001 From: Leszek Jakubowski Date: Mon, 4 Oct 2021 15:40:09 +0200 Subject: [PATCH 6/6] changed Exception messages and fixed a --help typo --- python/az/aro/azext_aro/_params.py | 2 +- python/az/aro/azext_aro/_validators.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/python/az/aro/azext_aro/_params.py b/python/az/aro/azext_aro/_params.py index 01f48f5c371..9b6e331718f 100644 --- a/python/az/aro/azext_aro/_params.py +++ b/python/az/aro/azext_aro/_params.py @@ -58,7 +58,7 @@ def load_arguments(self, _): validator=validate_cidr('service_cidr')) c.argument('software_defined_network', arg_type=get_enum_type(['OVNKubernetes', 'OpenShiftSDN']), options_list=['--software-defined-network-type', '--sdn-type'], - help='SDN type either "OpenShiftSDN (default)" or "OVNKubernetes"', + help='SDN type either "OpenShiftSDN" (default) or "OVNKubernetes"', validator=validate_sdn) c.argument('disk_encryption_set', diff --git a/python/az/aro/azext_aro/_validators.py b/python/az/aro/azext_aro/_validators.py index 8e52f3adf21..07a1c24e524 100644 --- a/python/az/aro/azext_aro/_validators.py +++ b/python/az/aro/azext_aro/_validators.py @@ -77,7 +77,8 @@ def validate_disk_encryption_set(cmd, namespace): compute_client.disk_encryption_sets.get(resource_group_name=desid['resource_group'], disk_encryption_set_name=desid['name']) except CloudError as err: - raise InvalidArgumentValueError(err.message) from err + raise InvalidArgumentValueError("Invald --disc-encryption-set, error when getting '%s': %s" % + (namespace.disk_encryption_set, err.message)) from err def validate_domain(namespace): @@ -159,7 +160,8 @@ def _validate_subnet(cmd, namespace): client.subnets.get(parts['resource_group'], parts['name'], parts['child_name_1']) except CloudError as err: - raise InvalidArgumentValueError(err.message) from err + raise InvalidArgumentValueError("Invald --%s, error when getting '%s': %s" % + (key.replace('_', '-'), subnet, err.message)) from err return _validate_subnet