From a0eb7879f12a5fdcf41c4d554ee2efa04cb3dcca Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Mon, 8 Jul 2019 12:38:34 +0200 Subject: [PATCH 01/12] Added files for boto3_elasticsearch: utils/boto3_elasticsearch.py: Contains custom waiter definitions that we will need to use until they get implemented in botocore (not present in 1.12.183). modules/boto3_elasticsearch.py: Contains all functions from boto3 that interact with AWS. Added a few helper functions. states/boto3_elasticsearch.py: Added states for the AWS ES service. --- salt/modules/boto3_elasticsearch.py | 1250 +++++++++++++++++++++++++++ salt/states/boto3_elasticsearch.py | 663 ++++++++++++++ salt/utils/boto3_elasticsearch.py | 140 +++ 3 files changed, 2053 insertions(+) create mode 100644 salt/modules/boto3_elasticsearch.py create mode 100644 salt/states/boto3_elasticsearch.py create mode 100644 salt/utils/boto3_elasticsearch.py diff --git a/salt/modules/boto3_elasticsearch.py b/salt/modules/boto3_elasticsearch.py new file mode 100644 index 000000000000..5a248b8c1e46 --- /dev/null +++ b/salt/modules/boto3_elasticsearch.py @@ -0,0 +1,1250 @@ +# -*- coding: utf-8 -*- +''' +Connection module for Amazon Elasticsearch Service + +.. versionadded:: Natrium + +:configuration: This module accepts explicit IAM credentials but can also + utilize IAM roles assigned to the instance trough Instance Profiles. + Dynamic credentials are then automatically obtained from AWS API and no + further configuration is necessary. More Information available at: + + .. code-block:: text + + http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + + If IAM roles are not used you need to specify them either in a pillar or + in the minion's config file: + + .. code-block:: yaml + + es.keyid: GKTADJGHEIQSXMKKRBJ08H + es.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + + A region may also be specified in the configuration: + + .. code-block:: yaml + + es.region: us-east-1 + + If a region is not specified, the default is us-east-1. + + It's also possible to specify key, keyid and region via a profile, either + as a passed in dict, or as a string to pull from pillars or minion config: + + .. code-block:: yaml + + myprofile: + keyid: GKTADJGHEIQSXMKKRBJ08H + key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + region: us-east-1 + + All methods return a dict with: + 'result' key containing a boolean indicating success or failure, + 'error' key containing the errormessage returned by boto on error, + 'response' key containing the data of the response returned by boto on success. + +:codeauthor: Herbert Buurman +:depends: boto3 +''' +# keep lint from choking on _get_conn and _cache_id +# pylint: disable=E0602 + +# Import Python libs +from __future__ import absolute_import, print_function, unicode_literals +import logging + +# Import Salt libs +from salt.ext import six +import salt.utils.compat +import salt.utils.json +import salt.utils.versions +from salt.exceptions import SaltInvocationError +from salt.utils.decorators import depends + +# Import third party libs + +try: + # Disable unused import-errors as these are only used for dependency checking + # pylint: disable=unused-import + import boto3 + import botocore + # pylint: enable=unused-import + from botocore.exceptions import ClientError, ParamValidationError, WaiterError + logging.getLogger('boto3').setLevel(logging.INFO) +except ImportError: + pass + +log = logging.getLogger(__name__) + + +def __virtual__(): + ''' + Only load if boto libraries exist and if boto libraries are greater than + a given version. + ''' + return salt.utils.versions.check_boto_reqs(boto3_ver='1.2.7') + + +def __init__(opts): + _ = opts + salt.utils.compat.pack_dunder(__name__) + __utils__['boto3.assign_funcs'](__name__, 'es') + + +def add_tags( + domain_name=None, + arn=None, + tags=None, + region=None, key=None, keyid=None, profile=None): + ''' + Attaches tags to an existing Elasticsearch domain. + Tags are a set of case-sensitive key value pairs. + An Elasticsearch domain may have up to 10 tags. + + :param str domain_name: The name of the Elasticsearch domain you want to add tags to. + :param str arn: The ARN of the Elasticsearch domain you want to add tags to. + Specifying this overrides ``domain_name``. + :param dict tags: The dict of tags to add to the Elasticsearch domain. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon failure, also contains a key 'error' with the error message as value. + + As a special case, tags whose key starts with `__` are ignored. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto_elasticsearch_domain.add_tags mydomain tags='{"foo": "bar", "baz": "qux"}' + + ''' + if not any((arn, domain_name)): + raise SaltInvocationError('At least one of domain_name or arn must be specified.') + ret = {'result': False} + if arn is None: + res = describe_elasticsearch_domain( + domain_name=domain_name, + region=region, key=key, keyid=keyid, profile=profile) + if 'error' in res: + ret.update(res) + elif not res['result']: + ret.update({'error': 'The domain with name "{}" does not exist.'.format(domain_name)}) + else: + arn = res['response'].get('ARN') + if arn: + boto_params = { + 'ARN': arn, + 'TagList': [{'Key': k, 'Value': value} for k, value in six.iteritems(tags or {})] + } + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + conn.add_tags(**boto_params) + ret['result'] = True + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.12.21') +def cancel_elasticsearch_service_software_update( + domain_name, + region=None, keyid=None, key=None, profile=None): + ''' + Cancels a scheduled service software update for an Amazon ES domain. You can + only perform this operation before the AutomatedUpdateDate and when the UpdateStatus + is in the PENDING_UPDATE state. + + :param str domain_name: The name of the domain that you want to stop the latest + service software update on. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the current service software options. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.cancel_elasticsearch_service_software_update(DomainName=domain_name) + ret['result'] = True + res['response'] = res['ServiceSoftwareOptions'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def create_elasticsearch_domain( + domain_name, + elasticsearch_version=None, + elasticsearch_cluster_config=None, + ebs_options=None, + access_policies=None, + snapshot_options=None, + vpc_options=None, + cognito_options=None, + encryption_at_rest_options=None, + node_to_node_encryption_options=None, + advanced_options=None, + log_publishing_options=None, + blocking=False, + region=None, key=None, keyid=None, profile=None): + ''' + Given a valid config, create a domain. + + :param str domain_name: The name of the Elasticsearch domain that you are creating. + Domain names are unique across the domains owned by an account within an + AWS region. Domain names must start with a letter or number and can contain + the following characters: a-z (lowercase), 0-9, and - (hyphen). + :param str elasticsearch_version: String of format X.Y to specify version for + the Elasticsearch domain eg. "1.5" or "2.3". + :param dict elasticsearch_cluster_config: Dict specifying the configuration + options for an Elasticsearch domain. Sub-options contained here are: + :param str InstanceType: The instance type for an Elasticsearch cluster. + :param int InstanceCount: The instance type for an Elasticsearch cluster. + :param bool DedicatedMasterEnabled: Indicate whether a dedicated master + node is enabled. + :param bool ZoneAwarenessEnabled: Indicate whether zone awareness is enabled. + If this is not enabled, the Elasticsearch domain will only be in one + availability zone. + :param dict ZoneAwarenessConfig: Specifies the zone awareness configuration + for a domain when zone awareness is enabled. Sub-options contained + here are: + :param int AvailabilityZoneCount: An integer value to indicate the + number of availability zones for a domain when zone awareness is + enabled. This should be equal to number of subnets if VPC endpoints + is enabled. Allowed values: 2, 3 + :param str DedicatedMasterType: The instance type for a dedicated master node. + :param int DedicatedMasterCount: Total number of dedicated master nodes, + active and on standby, for the cluster. + :param dict ebs_options: Dict specifying the options to enable or disable and + specifying the type and size of EBS storage volumes. + Sub-options contained here are: + :param bool EBSEnabled: Specifies whether EBS-based storage is enabled. + :param str VolumeType: Specifies the volume type for EBS-based storage. + :param int VolumeSize: Integer to specify the size of an EBS volume. + :param int Iops: Specifies the IOPD for a Provisioned IOPS EBS volume (SSD). + :param str/dict access_policies: Dict or JSON string with the IAM access policy. + :param dict snapshot_options: Dict specifying the snapshot options. + Sub-options contained here are: + :param int AutomatedSnapshotStartHour: Specifies the time, in UTC format, + when the service takes a daily automated snapshot of the specified + Elasticsearch domain. Default value is 0 hours. + :param dict vpc_options: Dict with the options to specify the subnets and security + groups for the VPC endpoint. Sub-options contained here are: + :param list SubnetIds: The list of subnets for the VPC endpoint. + :param list SecurityGroupIds: The list of security groups for the VPC endpoint. + :param dict cognito_options: Dict with options to specify the cognito user and + identity pools for Kibana authentication. Sub-options contained here are: + :param bool Enabled: Specifies the option to enable Cognito for Kibana authentication. + :param str UserPoolId: Specifies the Cognito user pool ID for Kibana authentication. + :param str IdentityPoolId: Specifies the Cognito identity pool ID for Kibana authentication. + :param str RoleArn: Specifies the role ARN that provides Elasticsearch permissions + for accessing Cognito resources. + :param dict encryption_at_rest_options: Dict specifying the encryption at rest + options. Sub-options contained here are: + :param bool Enabled: Specifies the option to enable Encryption At Rest. + :param str KmsKeyId: Specifies the KMS Key ID for Encryption At Rest options. + :param dict node_to_node_encryption_options: Dict specifying the node to node + encryption options. Sub-options contained here are: + :param bool Enabled: Specify True to enable node-to-node encryption. + :param dict advanced_options: Dict with option to allow references to indices + in an HTTP request body. Must be False when configuring access to individual + sub-resources. By default, the value is True. + See http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide\ + /es-createupdatedomains.html#es-createdomain-configure-advanced-options + for more information. + :param dict log_publishing_options: Dict with options for various type of logs. + The keys denote the type of log file and can be one of the following: + INDEX_SLOW_LOGS, SEARCH_SLOW_LOGS, ES_APPLICATION_LOGS. + The value assigned to each key is a dict with the following sub-options: + :param str CloudWatchLogsLogGroupArn: The ARN of the Cloudwatch log + group to which the log needs to be published. + :param bool Enabled: Specifies whether given log publishing option is enabled or not. + :param bool blocking: Whether or not to wait (block) until the Elasticsearch + domain has been created. + + Note: Not all instance types allow enabling encryption at rest. See https://docs.aws.amazon.com\ + /elasticsearch-service/latest/developerguide/aes-supported-instance-types.html + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the domain status configuration. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto_elasticsearch_domain.create mydomain \\ + elasticsearch_cluster_config='{ \\ + "InstanceType": "t2.micro.elasticsearch", \\ + "InstanceCount": 1, \\ + "DedicatedMasterEnabled": False, \\ + "ZoneAwarenessEnabled": False}' \\ + ebs_options='{ \\ + "EBSEnabled": True, \\ + "VolumeType": "gp2", \\ + "VolumeSize": 10, \\ + "Iops": 0}' \\ + access_policies='{ + "Version": "2012-10-17", \\ + "Statement": [ \\ + {"Effect": "Allow", \\ + "Principal": {"AWS": "*"}, \\ + "Action": "es:*", \\ + "Resource": "arn:aws:es:us-east-1:111111111111:domain/mydomain/*", \\ + "Condition": {"IpAddress": {"aws:SourceIp": ["127.0.0.1"]}}}]} \\ + snapshot_options='{"AutomatedSnapshotStartHour": 0}' \\ + advanced_options='{"rest.action.multi.allow_explicit_index": "true"}' + ''' + boto_kwargs = salt.utils.data.filter_falsey({ + 'DomainName': domain_name, + 'ElasticsearchVersion': six.text_type(elasticsearch_version or ''), + 'ElasticsearchClusterConfig': elasticsearch_cluster_config, + 'EBSOptions': ebs_options, + 'AccessPolicies': (salt.utils.json.dumps(access_policies) + if isinstance(access_policies, dict) + else access_policies), + 'SnapshotOptions': snapshot_options, + 'VPCOptions': vpc_options, + 'CognitoOptions': cognito_options, + 'EncryptionAtRestOptions': encryption_at_rest_options, + 'NodeToNodeEncryptionOptions': node_to_node_encryption_options, + 'AdvancedOptions': advanced_options, + 'LogPublishingOptions': log_publishing_options, + }) + ret = {'result': False} + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + res = conn.create_elasticsearch_domain(**boto_kwargs) + if res and 'DomainStatus' in res: + ret['result'] = True + ret['response'] = res['DomainStatus'] + if blocking: + waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESDomainAvailable') + waiter.wait(DomainName=domain_name) + except (ParamValidationError, ClientError, WaiterError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def delete_elasticsearch_domain( + domain_name, + blocking=False, + region=None, key=None, keyid=None, profile=None): + ''' + Permanently deletes the specified Elasticsearch domain and all of its data. + Once a domain is deleted, it cannot be recovered. + + :param str domain_name: The name of the domain to delete. + :param bool blocking: Whether or not to wait (block) until the Elasticsearch + domain has been deleted. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto_elasticsearch_domain.delete mydomain + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + conn.delete_elasticsearch_domain(DomainName=domain_name) + ret['result'] = True + if blocking: + waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESDomainDeleted') + waiter.wait(DomainName=domain_name) + except (ParamValidationError, ClientError, WaiterError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.7.30') +def delete_elasticsearch_service_role( + region=None, keyid=None, key=None, profile=None): + ''' + Deletes the service-linked role that Elasticsearch Service uses to manage and + maintain VPC domains. Role deletion will fail if any existing VPC domains use + the role. You must delete any such Elasticsearch domains before deleting the role. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + conn.delete_elasticsearch_service_role() + ret['result'] = True + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def describe_elasticsearch_domain( + domain_name, + region=None, keyid=None, key=None, profile=None): + ''' + Given a domain name gets its status description. + + :param str domain_name: The name of the domain to get the status of. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the domain status information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto_elasticsearch_domain.status mydomain + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + res = conn.describe_elasticsearch_domain(DomainName=domain_name) + if res and 'DomainStatus' in res: + ret['result'] = True + ret['response'] = res['DomainStatus'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def describe_elasticsearch_domain_config( + domain_name, + region=None, keyid=None, key=None, profile=None): + ''' + Provides cluster configuration information about the specified Elasticsearch domain, + such as the state, creation date, update version, and update date for cluster options. + + :param str domain_name: The name of the domain to describe. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the current configuration information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto_elasticsearch_domain.describe mydomain + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + res = conn.describe_elasticsearch_domain_config(DomainName=domain_name) + if res and 'DomainConfig' in res: + ret['result'] = True + ret['response'] = res['DomainConfig'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def describe_elasticsearch_domains( + domain_names, + region=None, keyid=None, key=None, profile=None): + ''' + Returns domain configuration information about the specified Elasticsearch + domains, including the domain ID, domain endpoint, and domain ARN. + + :param list domain_names: List of domain names to get information for. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the list of domain status information. + Upon failure, also contains a key 'error' with the error message as value. + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.describe_elasticsearch_domains(DomainNames=domain_names) + if res and 'DomainStatusList' in res: + ret['result'] = True + ret['response'] = res['DomainStatusList'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.5.18') +def describe_elasticsearch_instance_type_limits( + instance_type, + elasticsearch_version, + domain_name=None, + region=None, keyid=None, key=None, profile=None): + ''' + Describe Elasticsearch Limits for a given InstanceType and ElasticsearchVersion. + When modifying existing Domain, specify the `` DomainName `` to know what Limits + are supported for modifying. + + :param str instance_type: The instance type for an Elasticsearch cluster for + which Elasticsearch ``Limits`` are needed. + :param str elasticsearch_version: Version of Elasticsearch for which ``Limits`` + are needed. + :param str domain_name: Represents the name of the Domain that we are trying + to modify. This should be present only if we are querying for Elasticsearch + ``Limits`` for existing domain. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the limits information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + boto_params = salt.utils.data.filter_falsey({ + 'DomainName': domain_name, + 'InstanceType': instance_type, + 'ElasticsearchVersion': six.text_type(elasticsearch_version), + }) + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.describe_elasticsearch_instance_type_limits(**boto_params) + if res and 'LimitsByRole' in res: + ret['result'] = True + ret['response'] = res['LimitsByRole'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.15') +def describe_reserved_elasticsearch_instance_offerings( + reserved_elasticsearch_instance_offering_id=None, + region=None, keyid=None, key=None, profile=None): + ''' + Lists available reserved Elasticsearch instance offerings. + + :param str reserved_elasticsearch_instance_offering_id: The offering identifier + filter value. Use this parameter to show only the available offering that + matches the specified reservation identifier. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the list of offerings information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + boto_params = { + 'ReservedElasticsearchInstanceOfferingId': reserved_elasticsearch_instance_offering_id + } + res = [] + for page in conn.get_paginator( + 'describe_reserved_elasticsearch_instance_offerings' + ).paginate(**boto_params): + res.extend(page['ReservedElasticsearchInstanceOfferings']) + if res: + ret['result'] = True + ret['response'] = res + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.15') +def describe_reserved_elasticsearch_instances( + reserved_elasticsearch_instance_id=None, + region=None, keyid=None, key=None, profile=None): + ''' + Returns information about reserved Elasticsearch instances for this account. + + :param str reserved_elasticsearch_instance_id: The reserved instance identifier + filter value. Use this parameter to show only the reservation that matches + the specified reserved Elasticsearch instance ID. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with a list of information on + reserved instances. + Upon failure, also contains a key 'error' with the error message as value. + + Note: Version 1.9.174 of boto3 has a bug in that reserved_elasticsearch_instance_id + is considered a required argument, even though the documentation says otherwise. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + boto_params = { + 'ReservedElasticsearchInstanceId': reserved_elasticsearch_instance_id, + } + res = [] + for page in conn.get_paginator( + 'describe_reserved_elasticsearch_instances' + ).paginate(**boto_params): + res.extend(page['ReservedElasticsearchInstances']) + if res: + ret['result'] = True + ret['response'] = res + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.77') +def get_compatible_elasticsearch_versions( + domain_name=None, + region=None, keyid=None, key=None, profile=None): + ''' + Returns a list of upgrade compatible Elastisearch versions. You can optionally + pass a ``domain_name`` to get all upgrade compatible Elasticsearch versions + for that specific domain. + + :param str domain_name: The name of an Elasticsearch domain. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with a list of compatible versions. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + boto_params = salt.utils.data.filter_falsey({ + 'DomainName': domain_name, + }) + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.get_compatible_elasticsearch_versions(**boto_params) + if res and 'CompatibleElasticsearchVersions' in res: + ret['result'] = True + ret['response'] = res['CompatibleElasticsearchVersions'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.77') +def get_upgrade_history( + domain_name, + region=None, keyid=None, key=None, profile=None): + ''' + Retrieves the complete history of the last 10 upgrades that were performed on the domain. + + :param str domain_name: The name of an Elasticsearch domain. Domain names are + unique across the domains owned by an account within an AWS region. Domain + names start with a letter or number and can contain the following characters: + a-z (lowercase), 0-9, and - (hyphen). + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with a list of upgrade histories. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + boto_params = {'DomainName': domain_name} + res = [] + for page in conn.get_paginator('get_upgrade_history').paginate(**boto_params): + res.extend(page['UpgradeHistories']) + if res: + ret['result'] = True + ret['response'] = res + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.77') +def get_upgrade_status( + domain_name, + region=None, keyid=None, key=None, profile=None): + ''' + Retrieves the latest status of the last upgrade or upgrade eligibility check + that was performed on the domain. + + :param str domain_name: The name of an Elasticsearch domain. Domain names are + unique across the domains owned by an account within an AWS region. Domain + names start with a letter or number and can contain the following characters: + a-z (lowercase), 0-9, and - (hyphen). + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with upgrade status information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + boto_params = {'DomainName': domain_name} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.get_upgrade_status(**boto_params) + ret['result'] = True + ret['response'] = res + del res['ResponseMetadata'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def list_domain_names( + region=None, keyid=None, key=None, profile=None): + ''' + Returns the name of all Elasticsearch domains owned by the current user's account. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with a list of domain names. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.list_domain_names() + if res and 'DomainNames' in res: + ret['result'] = True + ret['response'] = [item['DomainName'] for item in res['DomainNames']] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.5.18') +def list_elasticsearch_instance_types( + elasticsearch_version, + domain_name=None, + region=None, keyid=None, key=None, profile=None): + ''' + List all Elasticsearch instance types that are supported for given ElasticsearchVersion. + + :param str elasticsearch_version: Version of Elasticsearch for which list of + supported elasticsearch instance types are needed. + :param str domain_name: DomainName represents the name of the Domain that we + are trying to modify. This should be present only if we are querying for + list of available Elasticsearch instance types when modifying existing domain. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with a list of Elasticsearch instance types. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + boto_params = salt.utils.data.filter_falsey({ + 'ElasticsearchVersion': six.text_type(elasticsearch_version), + 'DomainName': domain_name, + }) + res = [] + for page in conn.get_paginator('list_elasticsearch_instance_types').paginate(**boto_params): + res.extend(page['ElasticsearchInstanceTypes']) + if res: + ret['result'] = True + ret['response'] = res + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.5.18') +def list_elasticsearch_versions( + region=None, keyid=None, key=None, profile=None): + ''' + List all supported Elasticsearch versions. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with a list of Elasticsearch versions. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = [] + for page in conn.get_paginator('list_elasticsearch_versions').paginate(): + res.extend(page['ElasticsearchVersions']) + if res: + ret['result'] = True + ret['response'] = res + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def list_tags( + domain_name=None, + arn=None, + region=None, key=None, keyid=None, profile=None): + ''' + Returns all tags for the given Elasticsearch domain. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with a dict of tags. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto_elasticsearch.list_tags my_domain + ''' + if not any((arn, domain_name)): + raise SaltInvocationError('At least one of domain_name or arn must be specified.') + ret = {'result': False} + if arn is None: + res = describe_elasticsearch_domain( + domain_name=domain_name, + region=region, key=key, keyid=keyid, profile=profile) + if 'error' in res: + ret.update(res) + elif not res['result']: + ret.update({'error': 'The domain with name "{}" does not exist.'.format(domain_name)}) + else: + arn = res['response'].get('ARN') + if arn: + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + res = conn.list_tags(ARN=arn) + ret['result'] = True + ret['response'] = {item['Key']: item['Value'] for item in res.get('TagList', [])} + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.15') +def purchase_reserved_elasticsearch_instance_offering( + reserved_elasticsearch_instance_offering_id, + reservation_name, + instance_count=None, + region=None, keyid=None, key=None, profile=None): + ''' + Allows you to purchase reserved Elasticsearch instances. + + :param str reserved_elasticsearch_instance_offering_id: The ID of the reserved + Elasticsearch instance offering to purchase. + :param str reservation_name: A customer-specified identifier to track this reservation. + :param int instance_count: The number of Elasticsearch instances to reserve. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with purchase information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + boto_params = salt.utils.data.filter_falsey({ + 'ReservedElasticsearchInstanceOfferingId': reserved_elasticsearch_instance_offering_id, + 'ReservationName': reservation_name, + 'InstanceCount': instance_count, + }) + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.purchase_reserved_elasticsearch_instance_offering(**boto_params) + if res: + ret['result'] = True + ret['response'] = res + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def remove_tags( + tag_keys, + domain_name=None, + arn=None, + region=None, key=None, keyid=None, profile=None): + ''' + Removes the specified set of tags from the specified Elasticsearch domain. + + :param list tag_keys: List with tag keys you want to remove from the Elasticsearch domain. + :param str domain_name: The name of the Elasticsearch domain you want to remove tags from. + :param str arn: The ARN of the Elasticsearch domain you want to remove tags from. + Specifying this overrides ``domain_name``. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto_cloudtrail.remove_tags my_trail tag_a=tag_value tag_b=tag_value + ''' + if not any((arn, domain_name)): + raise SaltInvocationError('At least one of domain_name or arn must be specified.') + ret = {'result': False} + if arn is None: + res = describe_elasticsearch_domain( + domain_name=domain_name, + region=region, key=key, keyid=keyid, profile=profile) + if 'error' in res: + ret.update(res) + elif not res['result']: + ret.update({'error': 'The domain with name "{}" does not exist.'.format(domain_name)}) + else: + arn = res['response'].get('ARN') + if arn: + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + conn.remove_tags(ARN=arn, + TagKeys=tag_keys) + ret['result'] = True + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.12.21') +def start_elasticsearch_service_software_update( + domain_name, + region=None, keyid=None, key=None, profile=None): + ''' + Schedules a service software update for an Amazon ES domain. + + :param str domain_name: The name of the domain that you want to update to the + latest service software. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with service software information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + boto_params = {'DomainName': domain_name} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.start_elasticsearch_service_software_update(**boto_params) + if res and 'ServiceSoftwareOptions' in res: + ret['result'] = True + ret['response'] = res['ServiceSoftwareOptions'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def update_elasticsearch_domain_config( + domain_name, + elasticsearch_cluster_config=None, + ebs_options=None, + vpc_options=None, + access_policies=None, + snapshot_options=None, + cognito_options=None, + advanced_options=None, + log_publishing_options=None, + blocking=False, + region=None, key=None, keyid=None, profile=None): + ''' + Modifies the cluster configuration of the specified Elasticsearch domain, + for example setting the instance type and the number of instances. + + param str domain_name: The name of the Elasticsearch domain that you are creating. + Domain names are unique across the domains owned by an account within an + AWS region. Domain names must start with a letter or number and can contain + the following characters: a-z (lowercase), 0-9, and - (hyphen). + param dict elasticsearch_cluster_config: Dict specifying the configuration + options for an Elasticsearch domain. Sub-options contained here are: + param str InstanceType: The instance type for an Elasticsearch cluster. + param int InstanceCount: The instance type for an Elasticsearch cluster. + param bool DedicatedMasterEnabled: Indicate whether a dedicated master + node is enabled. + param bool ZoneAwarenessEnabled: Indicate whether zone awareness is enabled. + param dict ZoneAwarenessConfig: Specifies the zone awareness configuration + for a domain when zone awareness is enabled. Sub-options contained + here are: + param int AvailabilityZoneCount: An integer value to indicate the + number of availability zones for a domain when zone awareness is + enabled. This should be equal to number of subnets if VPC endpoints + is enabled. + param str DedicatedMasterType: The instance type for a dedicated master node. + param int DedicatedMasterCount: Total number of dedicated master nodes, + active and on standby, for the cluster. + param dict ebs_options: Dict specifying the options to enable or disable and + specifying the type and size of EBS storage volumes. + Sub-options contained here are: + param bool EBSEnabled: Specifies whether EBS-based storage is enabled. + param str VolumeType: Specifies the volume type for EBS-based storage. + param int VolumeSize: Integer to specify the size of an EBS volume. + param int Iops: Specifies the IOPD for a Provisioned IOPS EBS volume (SSD). + param dict snapshot_options: Dict specifying the snapshot options. + Sub-options contained here are: + param int AutomatedSnapshotStartHour: Specifies the time, in UTC format, + when the service takes a daily automated snapshot of the specified + Elasticsearch domain. Default value is 0 hours. + param dict vpc_options: Dict with the options to specify the subnets and security + groups for the VPC endpoint. Sub-options contained here are: + param list SubnetIds: The list of subnets for the VPC endpoint. + param list SecurityGroupIds: The list of security groups for the VPC endpoint. + param dict cognito_options: Dict with options to specify the cognito user and + identity pools for Kibana authentication. Sub-options contained here are: + param bool Enabled: Specifies the option to enable Cognito for Kibana authentication. + param str UserPoolId: Specifies the Cognito user pool ID for Kibana authentication. + param str IdentityPoolId: Specifies the Cognito identity pool ID for Kibana authentication. + param str RoleArn: Specifies the role ARN that provides Elasticsearch permissions + for accessing Cognito resources. + param dict advanced_options: Dict with option to allow references to indices + in an HTTP request body. Must be False when configuring access to individual + sub-resources. By default, the value is True. + See http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide\ + /es-createupdatedomains.html#es-createdomain-configure-advanced-options + for more information. + param str/dict access_policies: Dict or JSON string with the IAM access policy. + param dict log_publishing_options: Dict with options for various type of logs. + The keys denote the type of log file and can be one of the following: + INDEX_SLOW_LOGS, SEARCH_SLOW_LOGS, ES_APPLICATION_LOGS. + The value assigned to each key is a dict with the following sub-options: + param str CloudWatchLogsLogGroupArn: The ARN of the Cloudwatch log + group to which the log needs to be published. + param bool Enabled: Specifies whether given log publishing option is enabled or not. + :param bool blocking: Whether or not to wait (block) until the Elasticsearch + domain has been updated. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the domain configuration. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto_elasticsearch_domain.update mydomain \\ + {'InstanceType': 't2.micro.elasticsearch', 'InstanceCount': 1, \\ + 'DedicatedMasterEnabled': false, 'ZoneAwarenessEnabled': false} \\ + {'EBSEnabled': true, 'VolumeType': 'gp2', 'VolumeSize': 10, \\ + 'Iops': 0} \\ + {"Version": "2012-10-17", "Statement": [{"Effect": "Allow", \\ + "Principal": {"AWS": "*"}, "Action": "es:*", \\ + "Resource": "arn:aws:es:us-east-1:111111111111:domain/mydomain/*", \\ + "Condition": {"IpAddress": {"aws:SourceIp": ["127.0.0.1"]}}}]} \\ + {"AutomatedSnapshotStartHour": 0} \\ + {"rest.action.multi.allow_explicit_index": "true"} + + ''' + ret = {'result': False} + boto_kwargs = salt.utils.data.filter_falsey({ + 'DomainName': domain_name, + 'ElasticsearchClusterConfig': elasticsearch_cluster_config, + 'EBSOptions': ebs_options, + 'SnapshotOptions': snapshot_options, + 'VPCOptions': vpc_options, + 'CognitoOptions': cognito_options, + 'AdvancedOptions': advanced_options, + 'AccessPolicies': (salt.utils.json.dumps(access_policies) + if isinstance(access_policies, dict) + else access_policies), + 'LogPublishingOptions': log_publishing_options, + }) + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.update_elasticsearch_domain_config(**boto_kwargs) + if not res or 'DomainConfig' not in res: + log.warning('Domain was not updated') + else: + ret['result'] = True + ret['response'] = res['DomainConfig'] + if blocking: + waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESDomainAvailable') + waiter.wait(DomainName=domain_name) + except (ParamValidationError, ClientError, WaiterError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.77') +def upgrade_elasticsearch_domain( + domain_name, + target_version, + perform_check_only=None, + blocking=False, + region=None, keyid=None, key=None, profile=None): + ''' + Allows you to either upgrade your domain or perform an Upgrade eligibility + check to a compatible Elasticsearch version. + + :param str domain_name: The name of an Elasticsearch domain. Domain names are + unique across the domains owned by an account within an AWS region. Domain + names start with a letter or number and can contain the following characters: + a-z (lowercase), 0-9, and - (hyphen). + :param str target_version: The version of Elasticsearch that you intend to + upgrade the domain to. + :param bool perform_check_only: This flag, when set to True, indicates that + an Upgrade Eligibility Check needs to be performed. This will not actually + perform the Upgrade. + :param bool blocking: Whether or not to wait (block) until the Elasticsearch + domain has been upgraded. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the domain configuration. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + boto_params = salt.utils.data.filter_falsey({ + 'DomainName': domain_name, + 'TargetVersion': six.text_type(target_version), + 'PerformCheckOnly': perform_check_only, + }) + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.upgrade_elasticsearch_domain(**boto_params) + if res: + ret['result'] = True + ret['response'] = res + if blocking: + waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESUpgradeFinished2') + waiter.wait(DomainName=domain_name) + frop = describe_elasticsearch_domain(domain_name, region=region, keyid=keyid, key=key, profile=profile) + log.debug(__name__ + ':upgrade_elasticsearch_domain:\n' + '\t\tfrop: {}'.format(frop)) + except (ParamValidationError, ClientError, WaiterError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def exists( + domain_name, + region=None, key=None, keyid=None, profile=None): + ''' + Given a domain name, check to see if the given domain exists. + + :param str domain_name: The name of the domain to check. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto_elasticsearch_domain.exists mydomain + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + conn.describe_elasticsearch_domain(DomainName=domain_name) + ret['result'] = True + except (ParamValidationError, ClientError) as exp: + if exp.response.get('Error', {}).get('Code') != 'ResourceNotFoundException': + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def wait_for_upgrade( + domain_name, + region=None, keyid=None, key=None, profile=None): + ''' + Block until an upgrade-in-progress for domain ``name`` is finished. + + :param str name: The name of the domain to wait for. + + :return dict: + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESUpgradeFinished2') + waiter.wait(DomainName=domain_name) + ret['result'] = True + except (ParamValidationError, ClientError, WaiterError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.77') +def check_upgrade_eligibility( + domain_name, + elasticsearch_version, + region=None, keyid=None, key=None, profile=None): + ''' + Helper function to determine in one call if an Elasticsearch domain can be + upgraded to the specified Elasticsearch version. + + This assumes that the Elasticsearch domain is at rest at the moment this function + is called. I.e. The domain is not in the process of : + - being created. + - being updated. + - another upgrade running, or a check thereof. + - being deleted. + + Behind the scenes, this does 3 things: + - Check if ``elasticsearch_version`` is among the compatible elasticsearch versions. + - Perform a check if the Elasticsearch domain is eligible for the upgrade. + - Check the result of the check and return the result as a boolean. + + :param str name: The Elasticsearch domain name to check. + :param str elasticsearch_version: The Elasticsearch version to upgrade to. + + :return dict: With key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with boolean result of the check. + Upon failure, also contains a key 'error' with the error message as value. + ''' + ret = {'result': False} + # Check if the desired version is in the list of compatible versions + res = get_compatible_elasticsearch_versions( + domain_name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + return res + compatible_versions = res['response'][0]['TargetVersions'] + if six.text_type(elasticsearch_version) not in compatible_versions: + ret['result'] = True + ret['response'] = False + ret['error'] = ('Desired version "{}" not in compatible versions: {}.' + ''.format(elasticsearch_version, compatible_versions)) + return ret + # Check if the domain is eligible to upgrade to the desired version + res = upgrade_elasticsearch_domain( + domain_name, + elasticsearch_version, + perform_check_only=True, + blocking=True, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + return res + res = wait_for_upgrade(domain_name, region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + return res + res = get_upgrade_status(domain_name, region=region, keyid=keyid, key=key, profile=profile) + ret['result'] = True + ret['response'] = (res['response']['UpgradeStep'] == 'PRE_UPGRADE_CHECK' and + res['response']['StepStatus'] == 'SUCCEEDED') + return ret diff --git a/salt/states/boto3_elasticsearch.py b/salt/states/boto3_elasticsearch.py new file mode 100644 index 000000000000..b8d1c511459a --- /dev/null +++ b/salt/states/boto3_elasticsearch.py @@ -0,0 +1,663 @@ +# -*- coding: utf-8 -*- +''' +Manage Elasticsearch Service +============================ + +.. versionadded:: Natrium + +:configuration: This module accepts explicit AWS credentials but can also + utilize IAM roles assigned to the instance trough Instance Profiles. + Dynamic credentials are then automatically obtained from AWS API and no + further configuration is necessary. More Information available at: + + .. code-block:: text + + http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + + If IAM roles are not used you need to specify them either in a pillar or + in the minion's config file: + + .. code-block:: yaml + + es.keyid: GKTADJGHEIQSXMKKRBJ08H + es.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + + A region may also be specified in the configuration: + + .. code-block:: yaml + + es.region: us-east-1 + + If a region is not specified, the default is us-east-1. + + It's also possible to specify key, keyid and region via a profile, either + as a passed in dict, or as a string to pull from pillars or minion config: + + .. code-block:: yaml + + myprofile: + keyid: GKTADJGHEIQSXMKKRBJ08H + key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + region: us-east-1 + +:codeauthor: Herbert Buurman +:depends: boto3 +''' + +# Import Python libs +from __future__ import absolute_import, print_function, unicode_literals +import logging + +# Import Salt libs +import salt.utils.json +from salt.utils.versions import LooseVersion + +# Import 3rd-party libs + +log = logging.getLogger(__name__) +__virtualname__ = 'boto3_elasticsearch' + + +def __virtual__(): + ''' + Only load if boto3 and the required module functions are available. + ''' + requirements = { + 'salt': [ + 'boto3_elasticsearch.describe_elasticsearch_domain', + 'boto3_elasticsearch.create_elasticsearch_domain', + 'boto3_elasticsearch.update_elasticsearch_domain_config', + 'boto3_elasticsearch.exists', + 'boto3_elasticsearch.get_upgrade_status', + 'boto3_elasticsearch.wait_for_upgrade', + 'boto3_elasticsearch.check_upgrade_eligibility', + 'boto3_elasticsearch.upgrade_elasticsearch_domain', + ], + } + for req in requirements['salt']: + if req not in __salt__: + return (False, 'A required function was not found in __salt__: {}'.format(req)) + return __virtualname__ + + +def present( + name, + elasticsearch_version=None, + elasticsearch_cluster_config=None, + ebs_options=None, + access_policies=None, + snapshot_options=None, + vpc_options=None, + cognito_options=None, + encryption_at_rest_options=None, + node_to_node_encryption_options=None, + advanced_options=None, + log_publishing_options=None, + blocking=True, + tags=None, + region=None, keyid=None, key=None, profile=None): + ''' + Ensure an Elasticsearch Domain exists. + + :param str name: The name of the Elasticsearch domain that you are creating. + Domain names are unique across the domains owned by an account within an + AWS region. Domain names must start with a letter or number and can contain + the following characters: a-z (lowercase), 0-9, and - (hyphen). + :param str elasticsearch_version: String of format X.Y to specify version for + the Elasticsearch domain eg. "1.5" or "2.3". + :param dict elasticsearch_cluster_config: Dict specifying the configuration + options for an Elasticsearch domain. Sub-options contained here are: + :param str InstanceType: The instance type for an Elasticsearch cluster. + :param int InstanceCount: The instance type for an Elasticsearch cluster. + :param bool DedicatedMasterEnabled: Indicate whether a dedicated master + node is enabled. + :param bool ZoneAwarenessEnabled: Indicate whether zone awareness is enabled. + :param dict ZoneAwarenessConfig: Specifies the zone awareness configuration + for a domain when zone awareness is enabled. Sub-options contained + here are: + :param int AvailabilityZoneCount: An integer value to indicate the + number of availability zones for a domain when zone awareness is + enabled. This should be equal to number of subnets if VPC endpoints + is enabled. + :param str DedicatedMasterType: The instance type for a dedicated master node. + :param int DedicatedMasterCount: Total number of dedicated master nodes, + active and on standby, for the cluster. + :param dict ebs_options: Dict specifying the options to enable or disable and + specifying the type and size of EBS storage volumes. + Sub-options contained here are: + :param bool EBSEnabled: Specifies whether EBS-based storage is enabled. + :param str VolumeType: Specifies the volume type for EBS-based storage. + :param int VolumeSize: Integer to specify the size of an EBS volume. + :param int Iops: Specifies the IOPD for a Provisioned IOPS EBS volume (SSD). + :param str/dict access_policies: Dict or JSON string with the IAM access policy. + :param dict snapshot_options: Dict specifying the snapshot options. + Sub-options contained here are: + :param int AutomatedSnapshotStartHour: Specifies the time, in UTC format, + when the service takes a daily automated snapshot of the specified + Elasticsearch domain. Default value is 0 hours. + :param dict vpc_options: Dict with the options to specify the subnets and security + groups for the VPC endpoint. Sub-options contained here are: + :param list SubnetIds: The list of subnets for the VPC endpoint. + :param list SecurityGroupIds: The list of security groups for the VPC endpoint. + :param dict cognito_options: Dict with options to specify the cognito user and + identity pools for Kibana authentication. Sub-options contained here are: + :param bool Enabled: Specifies the option to enable Cognito for Kibana authentication. + :param str UserPoolId: Specifies the Cognito user pool ID for Kibana authentication. + :param str IdentityPoolId: Specifies the Cognito identity pool ID for Kibana authentication. + :param str RoleArn: Specifies the role ARN that provides Elasticsearch permissions + for accessing Cognito resources. + :param dict encryption_at_rest_options: Dict specifying the encryption at rest + options. This option can only be used for the creation of a new Elasticsearch + domain. Sub-options contained here are: + :param bool Enabled: Specifies the option to enable Encryption At Rest. + :param str KmsKeyId: Specifies the KMS Key ID for Encryption At Rest options. + :param dict node_to_node_encryption_options: Dict specifying the node to node + encryption options. This option can only be used for the creation of + a new Elasticsearch domain. Sub-options contained here are: + :param bool Enabled: Specify True to enable node-to-node encryption. + :param dict advanced_options: Dict with option to allow references to indices + in an HTTP request body. Must be False when configuring access to individual + sub-resources. By default, the value is True. + See http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide\ + /es-createupdatedomains.html#es-createdomain-configure-advanced-options + for more information. + :param dict log_publishing_options: Dict with options for various type of logs. + The keys denote the type of log file and can be one of the following: + INDEX_SLOW_LOGS, SEARCH_SLOW_LOGS, ES_APPLICATION_LOGS. + The value assigned to each key is a dict with the following sub-options: + :param str CloudWatchLogsLogGroupArn: The ARN of the Cloudwatch log + group to which the log needs to be published. + :param bool Enabled: Specifies whether given log publishing option is enabled or not. + :param bool blocking: Whether or not the state should wait for all operations + (create/update/upgrade) to be completed. Default: ``True`` + :param dict tags: Dict of tags to ensure are present on the Elasticsearch domain. + + .. versionadded:: Natrium + + ''' + ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} + + action = None + current_domain = None + target_conf = salt.utils.data.filter_falsey({ + 'DomainName': name, + 'ElasticsearchClusterConfig': elasticsearch_cluster_config, + 'EBSOptions': ebs_options, + 'AccessPolicies': (salt.utils.json.dumps(access_policies) + if isinstance(access_policies, dict) + else access_policies), + 'SnapshotOptions': snapshot_options, + 'VPCOptions': vpc_options, + 'CognitoOptions': cognito_options, + 'AdvancedOptions': advanced_options, + 'LogPublishingOptions': log_publishing_options, + }, recurse_depth=3) + res = __salt__['boto3_elasticsearch.describe_elasticsearch_domain']( + name, + region=region, keyid=keyid, key=key, profile=profile) + if not res['result']: + ret['result'] = False + if 'ResourceNotFoundException' in res['error']: + action = 'create' + config_diff = {'old': None, 'new': target_conf} + else: + ret['comment'].append(res['error']) + else: + current_domain = salt.utils.data.filter_falsey(res['response'], recurse_depth=3) + current_domain_version = current_domain['ElasticsearchVersion'] + # Remove some values from current_domain that cannot be updated + for item in ['DomainId', 'UpgradeProcessing', 'Created', 'Deleted', 'Processing', + 'Endpoints', 'ARN', 'EncryptionAtRestOptions', 'NodeToNodeEncryptionOptions', + 'ElasticsearchVersion', 'ServiceSoftwareOptions']: + if item in current_domain: + del current_domain[item] + # Further remove values from VPCOptions (if present) that are read-only + for item in ['VPCId', 'AvailabilityZones']: + if item in current_domain.get('VPCOptions', {}): + del current_domain['VPCOptions'][item] + # Some special cases + if 'CognitoOptions' in current_domain: + if 'CognitoOptions' not in target_conf and not current_domain['CognitoOptions']['Enabled']: + del current_domain['CognitoOptions'] + if 'AdvancedOptions' not in target_conf and \ + 'rest.action.multi.allow_explicit_index' in current_domain['AdvancedOptions']: + del current_domain['AdvancedOptions']['rest.action.multi.allow_explicit_index'] + if not current_domain['AdvancedOptions']: + del current_domain['AdvancedOptions'] + + # Compare current configuration with provided configuration + config_diff = salt.utils.data.recursive_diff(current_domain, target_conf) + if config_diff: + action = 'update' + + # Compare ElasticsearchVersion separately, as the update procedure differs. + if elasticsearch_version and current_domain_version != elasticsearch_version: + action = 'upgrade' + + if action in ['create', 'update']: + if __opts__['test']: + ret['result'] = None + ret['comment'].append('The Elasticsearch Domain "{}" would have been {}d.' + ''.format(name, action)) + ret['changes'] = config_diff + else: + boto_kwargs = salt.utils.data.filter_falsey({ + 'elasticsearch_version': elasticsearch_version, + 'elasticsearch_cluster_config': elasticsearch_cluster_config, + 'ebs_options': ebs_options, + 'vpc_options': vpc_options, + 'access_policies': access_policies, + 'snapshot_options': snapshot_options, + 'cognito_options': cognito_options, + 'encryption_at_rest_options': encryption_at_rest_options, + 'node_to_node_encryption_options': node_to_node_encryption_options, + 'advanced_options': advanced_options, + 'log_publishing_options': log_publishing_options, + 'blocking': blocking, + 'region': region, 'keyid': keyid, 'key': key, 'profile': profile, + }) + if action == 'update': + # Drop certain kwargs that do not apply to updates. + for item in ['elasticsearch_version', 'encryption_at_rest_options', + 'node_to_node_encryption_options']: + if item in boto_kwargs: + del boto_kwargs[item] + res = __salt__['boto3_elasticsearch.{}_elasticsearch_domain{}' + ''.format(action, '_config' if action == 'update' else '')]( + name, + **boto_kwargs) + if 'error' in res: + ret['result'] = False + ret['comment'].append(res['error']) + else: + ret['result'] = True + ret['comment'].append('Elasticsearch Domain "{}" has been {}d.'.format(name, action)) + ret['changes'] = config_diff + elif action == 'upgrade': + res = upgraded( + name, + elasticsearch_version, + region=region, keyid=keyid, key=key, profile=profile) + ret['result'] = res['result'] + ret['comment'].extend(res['comment']) + if res['changes']: + salt.utils.dictupdate.set_dict_key_value( + ret, + 'changes:old:version', + res['changes']['old']) + salt.utils.dictupdate.set_dict_key_value( + ret, + 'changes:new:version', + res['changes']['new']) + + if tags is not None: + res = tagged( + name, + tags=tags, + replace=True, + region=region, keyid=keyid, key=key, profile=profile) + ret['result'] = res['result'] + ret['comment'].extend(res['comment']) + if 'old' in res['changes']: + salt.utils.dictupdate.update_dict_key_value( + ret, + 'changes:old:tags', + res['changes']['old'] + ) + if 'new' in res['changes']: + salt.utils.dictupdate.update_dict_key_value( + ret, + 'changes:new:tags', + res['changes']['new'] + ) + + if ret['result'] == 'oops': + ret['result'] = False + ret['comment'].append('An internal error has occurred: The result value was ' + 'not properly changed.') + return ret + + +def absent( + name, + blocking=True, + region=None, keyid=None, key=None, profile=None): + ''' + Ensure the Elasticsearch Domain specified does not exist. + + :param str name: The name of the Elasticsearch domain to be made absent. + :param bool blocking: Whether or not the state should wait for the deletion + to be completed. Default: ``True`` + + .. versionadded:: Natrium + + ''' + ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} + + res = __salt__['boto3_elasticsearch.exists']( + name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append(res['error']) + elif res['result']: + if __opts__['test']: + ret['result'] = None + ret['comment'].append('Elasticsearch domain "{}" would have been removed.' + ''.format(name)) + ret['changes'] = {'old': name, 'new': None} + else: + res = __salt__['boto3_elasticsearch.delete_elasticsearch_domain']( + domain_name=name, + blocking=blocking, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error deleting Elasticsearch domain "{}": {}' + ''.format(name, res['error'])) + else: + ret['result'] = True + ret['comment'].append('Elasticsearch domain "{}" has been deleted.' + ''.format(name)) + ret['changes'] = {'old': name, 'new': None} + else: + ret['result'] = True + ret['comment'].append('Elasticsearch domain "{}" is already absent.' + ''.format(name)) + if ret['result'] == 'oops': + ret['result'] = False + ret['comment'].append('An internal error has occurred: The result value was ' + 'not properly changed.') + return ret + + +def upgraded( + name, + elasticsearch_version, + blocking=True, + region=None, keyid=None, key=None, profile=None): + ''' + Ensures the Elasticsearch domain specified runs on the specified version of + elasticsearch. Only upgrades are possible as downgrades require a manual snapshot + and an S3 bucket to store them in. + + Note that this operation is blocking until the upgrade is complete. + + :param str name: The name of the Elasticsearch domain to upgrade. + :param str elasticsearch_version: String of format X.Y to specify version for + the Elasticsearch domain eg. "1.5" or "2.3". + ''' + ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} + current_domain = None + res = __salt__['boto3_elasticsearch.describe_elasticsearch_domain']( + name, + region=region, keyid=keyid, key=key, profile=profile) + if not res['result']: + ret['result'] = False + if 'ResourceNotFoundException' in res['error']: + ret['comment'].append('The Elasticsearch domain "{}" does not exist.' + ''.format(name)) + else: + ret['comment'].append(res['error']) + else: + current_domain = res['response'] + current_version = current_domain['ElasticsearchVersion'] + if elasticsearch_version and current_version == elasticsearch_version: + ret['result'] = True + ret['comment'].append('The Elasticsearch domain "{}" is already ' + 'at the desired version {}' + ''.format(name, elasticsearch_version)) + elif LooseVersion(elasticsearch_version) < LooseVersion(current_version): + ret['result'] = False + ret['comment'].append('Elasticsearch domain "{}" cannot be downgraded ' + 'to version "{}".' + ''.format(name, elasticsearch_version)) + if isinstance(ret['result'], bool): + return ret + log.debug(__name__ + ':upgraded: Check upgrade in progress') + # Check if an upgrade is already in progress + res = __salt__['boto3_elasticsearch.get_upgrade_status']( + name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error determining current upgrade status ' + 'of domain "{}": {}'.format(name, res['error'])) + return ret + if res['response'].get('StepStatus') == 'IN_PROGRESS': + if blocking: + # An upgrade is already in progress, wait for it to complete + res2 = __salt__['boto3_elasticsearch.wait_for_upgrade']( + name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res2: + ret['result'] = False + ret['comment'].append('Error waiting for upgrade of domain ' + '"{}" to complete: {}' + ''.format(name, res2['error'])) + elif res2['response'].get('UpgradeName', '').endswith(elasticsearch_version): + ret['result'] = True + ret['comment'].append('Elasticsearch Domain "{}" is ' + 'already at version "{}".' + ''.format(name, elasticsearch_version)) + else: + # We are not going to wait for it to complete, so bail. + ret['result'] = True + ret['comment'].append('An upgrade of Elasticsearch domain "{}" ' + 'is already underway: {}' + ''.format(name, res['response'].get('UpgradeName'))) + if isinstance(ret['result'], bool): + return ret + + log.debug(__name__ + ':upgraded: Check upgrade eligibility') + # Check if the domain is eligible for an upgrade + res = __salt__['boto3_elasticsearch.check_upgrade_eligibility']( + name, + elasticsearch_version, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error checking upgrade eligibility for ' + 'domain "{}": {}'.format(name, res['error'])) + elif not res['response']: + ret['result'] = False + ret['comment'].append('The Elasticsearch Domain "{}" is not eligible to ' + 'be upgraded to version {}.' + ''.format(name, elasticsearch_version)) + else: + log.debug(__name__ + ':upgraded: Start the upgrade') + # Start the upgrade + if __opts__['test']: + ret['result'] = None + ret['comment'].append('The Elasticsearch version for domain "{}" would have been upgraded.') + ret['changes'] = {'old': current_domain['ElasticsearchVersion'], + 'new': elasticsearch_version} + else: + res = __salt__['boto3_elasticsearch.upgrade_elasticsearch_domain']( + name, + elasticsearch_version, + blocking=blocking, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error upgrading Elasticsearch domain "{}": {}' + ''.format(name, res['error'])) + else: + ret['result'] = True + ret['comment'].append('The Elasticsearch domain "{}" has been ' + 'upgraded to version {}.' + ''.format(name, elasticsearch_version)) + ret['changes'] = {'old': current_domain['ElasticsearchVersion'], + 'new': elasticsearch_version} + if ret['result'] == 'oops': + ret['result'] = False + ret['comment'].append('An internal error has occurred: The result value was ' + 'not properly changed.') + return ret + + +def latest( + name, + minor_only=True, + region=None, keyid=None, key=None, profile=None): + ''' + Ensures the Elasticsearch domain specifies runs on the latest compatible + version of elasticsearch, upgrading it if it is not. + + Note that this operation is blocking until the upgrade is complete. + + :param str name: The name of the Elasticsearch domain to upgrade. + :param bool minor_only: Only upgrade to the latest minor version. + ''' + ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} + # Get current version + res = __salt__['boto3_elasticsearch.describe_elasticsearch_domain']( + domain_name=name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error getting information of Elasticsearch domain "{}": {}' + ''.format(name, res['error'])) + else: + current_version = res['response']['ElasticsearchVersion'] + # Get latest compatible version + latest_version = None + res = __salt__['boto3_elasticsearch.get_compatible_elasticsearch_versions']( + domain_name=name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error getting compatible Elasticsearch versions ' + 'for Elasticsearch domain "{}": {}' + ''.format(name, res['error'])) + if isinstance(ret['result'], bool): + return ret + try: + latest_version = res['response'][0]['TargetVersions'].pop(-1) + except IndexError: + pass + if not current_version: + ret['result'] = True + ret['comment'].append('The Elasticsearch domain "{}" can not be upgraded.' + ''.format(name)) + elif not latest_version: + ret['result'] = True + ret['comment'].append('The Elasticsearch domain "{}" is already at ' + 'the lastest version "{}".' + ''.format(name, current_version)) + else: + a_current_version = current_version.split('.') + a_latest_version = latest_version.split('.') + if not (minor_only and a_current_version[0] != a_latest_version[0]): + if __opts__['test']: + ret['result'] = None + ret['comment'].append('Elasticsearch domain "{}" would have been updated ' + 'to version "{}".'.format(name, latest_version)) + ret['changes'] = {'old': current_version, 'new': latest_version} + else: + ret = upgraded( + name, + latest_version, + region=region, keyid=keyid, key=key, profile=profile) + else: + ret['result'] = True + ret['comment'].append('Elasticsearch domain "{}" is already at its ' + 'latest minor version {}.' + ''.format(name, current_version)) + if ret['result'] == 'oops': + ret['result'] = False + ret['comment'].append('An internal error has occurred: The result value was ' + 'not properly changed.') + if ret['result'] and ret['changes'] and not minor_only: + # Try and see if we can upgrade again + res = latest(name, minor_only=minor_only, region=region, keyid=keyid, key=key, profile=profile) + if res['result'] and res['changes']: + ret['changes']['new'] = res['changes']['new'] + ret['comment'].extend(res['comment']) + return ret + + +def tagged( + name, + tags=None, + replace=False, + region=None, keyid=None, key=None, profile=None): + ''' + Ensures the Elasticsearch domain has the tags provided. + Adds tags to the domain unless ``replace`` is set to ``True``, in which + case all existing tags will be replaced with the tags provided in ``tags``. + (This will remove all tags if ``replace`` is ``True`` and ``tags`` is empty). + + :param str name: The Elasticsearch domain to work with. + :param dict tags: The tags to add to/replace on the Elasticsearch domain. + :param bool replace: Whether or not to replace (``True``) all existing tags + on the Elasticsearch domain, or add (``False``) tags to the ES domain. + ''' + ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} + current_tags = {} + # Check if the domain exists + res = __salt__['boto3_elasticsearch.exists']( + name, + region=region, keyid=keyid, key=key, profile=profile) + if res['result']: + res = __salt__['boto3_elasticsearch.list_tags']( + name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error fetching tags of Elasticsearch domain ' + '"{}": {}'.format(name, res['error'])) + else: + current_tags = res['response'] or {} + else: + ret['result'] = False + ret['comment'].append('Elasticsearch domain "{}" does not exist.' + ''.format(name)) + if isinstance(ret['result'], bool): + return ret + + diff_tags = salt.utils.dictdiffer.deep_diff(current_tags, tags) + if not diff_tags: + ret['result'] = True + ret['comment'].append('Elasticsearch domain "{}" already has the specified ' + 'tags.'.format(name)) + else: + if replace: + ret['changes'] = diff_tags + else: + ret['changes'] = {'old': current_tags, 'new': current_tags.update(tags)} + if __opts__['test']: + ret['result'] = None + ret['comment'].append('Tags on Elasticsearch domain "{}" would have ' + 'been {}ed.'.format(name, 'replac' if replace else 'add')) + else: + if replace: + res = __salt__['boto3_elasticsearch.remove_tags']( + tag_keys=current_tags.keys(), + domain_name=name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error removing current tags from Elasticsearch ' + 'domain "{}": {}'.format(name, res['error'])) + ret['changes'] = {} + if isinstance(ret['result'], bool): + return ret + res = __salt__['boto3_elasticsearch.add_tags']( + domain_name=name, + tags=tags, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error tagging Elasticsearch domain ' + '"{}": {}'.format(name, res['error'])) + ret['changes'] = {} + else: + ret['result'] = True + ret['comment'].append('Tags on Elasticsearch domain "{}" have been ' + '{}ed.'.format(name, 'replac' if replace else 'add')) + if ret['result'] == 'oops': + ret['result'] = False + ret['comment'].append('An internal error has occurred: The result value was ' + 'not properly changed.') + return ret diff --git a/salt/utils/boto3_elasticsearch.py b/salt/utils/boto3_elasticsearch.py new file mode 100644 index 000000000000..bdb5ac4308c6 --- /dev/null +++ b/salt/utils/boto3_elasticsearch.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +''' +Botocore waiters for elasticsearch that are not present in boto3+botocore (yet). + +:codeauthor: Herbert Buurman +:depends: boto3 +''' +# Import Python libs +from __future__ import absolute_import, print_function, unicode_literals + +# Import Salt libs +from salt.exceptions import SaltInvocationError +import salt.utils.versions + +try: + import botocore.waiter +except ImportError: + pass + + +WAITER_CONFIGS = { + 'ESDomainAvailable': { + 'delay': 60, + 'operation': 'DescribeElasticsearchDomainConfig', + 'maxAttempts': 60, + 'acceptors': [{ + 'expected': 'Active', + 'matcher': 'path', + 'state': 'success', + 'argument': 'DomainConfig.ElasticsearchClusterConfig.Status.State', + }, { + 'expected': True, + 'matcher': 'pathAny', + 'state': 'failure', + 'argument': 'DomainConfig.*.Status.PendingDeletion', + }], + }, + 'ESUpgradeFinished2': { + 'delay': 60, + 'operation': 'DescribeElasticsearchDomain', + 'maxAttempts': 60, + 'acceptors': [{ + 'expected': False, + 'matcher': 'path', + 'state': 'success', + 'argument': 'DomainStatus.UpgradeProcessing', + }], + }, + 'ESUpgradeFinished': { + 'delay': 60, + 'operation': 'GetUpgradeStatus', + 'maxAttempts': 60, + 'acceptors': [{ + 'expected': 'SUCCEEDED', + 'matcher': 'path', + 'state': 'success', + 'argument': 'StepStatus' + }, { + 'expected': 'FAILED', + 'matcher': 'path', + 'state': 'success', + 'argument': 'StepStatus', + }, { + 'expected': 'SUCCEEDED_WITH_ISSUES', + 'matcher': 'path', + 'state': 'success', + 'argument': 'StepStatus', + }], + }, + 'ESDomainDeleted': { + 'delay': 30, + 'operation': 'DescribeElasticsearchDomain', + 'maxAttempts': 60, + 'acceptors': [{ + 'expected': True, + 'matcher': 'path', + 'state': 'retry', + 'argument': 'DomainStatus.Deleted', + }, { + 'expected': False, + 'matcher': 'path', + 'state': 'failure', + 'argument': 'DomainStatus.Processing', + }, { + 'expected': 'ResourceNotFoundException', + 'matcher': 'error', + 'state': 'success', + }], + }, + 'ESDomainCreated': { + 'delay': 30, + 'operation': 'DescribeElasticsearchDomain', + 'maxAttempts': 60, + 'acceptors': [{ + 'expected': True, + 'matcher': 'path', + 'state': 'success', + 'argument': 'DomainStatus.Created', + }], + }, +} + + +def __virtual__(): + ''' + Only load if botocore libraries exist. + ''' + return salt.utils.versions.check_boto_reqs(check_boto=False) + + +def get_waiter(client, waiter=None, waiter_config=None): + ''' + Gets a botocore waiter using either one of the preconfigured models by name + ``waiter``, or with a manually supplied ``waiter_config``. + + :param botoclient client: The botocore client to use. + :param str waiter: The name of the waiter config to use. + Either ``waiter`` or ``waiter_config`` must be supplied. + If both ``waiter`` and ``waiter_config`` are supplied, ``waiter`` takes + presedence, unless no configuration for ``waiter`` exists. + :param dict waiter_config: The manual waiter config to use. + Either waiter or waiter_config must be supplied. + + :returns botocore.waiter + ''' + if not any((waiter, waiter_config)): + raise SaltInvocationError('At least one of waiter or waiter_config must be specified.') + waiter_model = botocore.waiter.WaiterModel( + {'version': 2, 'waiters': {waiter: WAITER_CONFIGS.get(waiter, waiter_config)}} + ) + return botocore.waiter.create_waiter_with_client(waiter, waiter_model, client) + + +def list_waiters(): + ''' + Lists the builtin waiter configuration names. + + :returns list + ''' + return WAITER_CONFIGS.keys() From 6a873eaa869180dba5af6c997cb9eec1913ae990 Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Mon, 8 Jul 2019 12:38:56 +0200 Subject: [PATCH 02/12] Added unittests for boto3_elasticsearch. --- .../unit/modules/test_boto3_elasticsearch.py | 1152 +++++++++++++++++ 1 file changed, 1152 insertions(+) create mode 100644 tests/unit/modules/test_boto3_elasticsearch.py diff --git a/tests/unit/modules/test_boto3_elasticsearch.py b/tests/unit/modules/test_boto3_elasticsearch.py new file mode 100644 index 000000000000..37b7b3cebcc3 --- /dev/null +++ b/tests/unit/modules/test_boto3_elasticsearch.py @@ -0,0 +1,1152 @@ +# -*- coding: utf-8 -*- +''' + Tests for salt.modules.boto3_elasticsearch +''' + +# Import Python libs +from __future__ import absolute_import, print_function, unicode_literals +import random +import string +import datetime +import textwrap + +# Import Salt Testing libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import skipIf, TestCase +from tests.support.mock import (NO_MOCK, NO_MOCK_REASON, MagicMock, patch) + +# Import Salt libs +import salt.loader +from salt.utils.versions import LooseVersion +import salt.modules.boto3_elasticsearch as boto3_elasticsearch +from salt.ext.six.moves import range + +# Import 3rd-party libs +try: + import boto3 + from botocore.exceptions import ClientError + HAS_BOTO3 = True +except ImportError: + HAS_BOTO3 = False + +# the boto3_elasticsearch module relies on the connect_to_region() method +# which was added in boto 2.8.0 +# https://github.com/boto/boto/commit/33ac26b416fbb48a60602542b4ce15dcc7029f12 +REQUIRED_BOTO3_VERSION = '1.2.1' + + +def __virtual__(): + ''' + Returns True/False boolean depending on if Boto3 is installed and correct + version. + ''' + if not HAS_BOTO3: + return False + if LooseVersion(boto3.__version__) < LooseVersion(REQUIRED_BOTO3_VERSION): + return False, ('The boto3 module must be greater or equal to version {}' + ''.format(REQUIRED_BOTO3_VERSION)) + return True + + +REGION = 'us-east-1' +ACCESS_KEY = 'GKTADJGHEIQSXMKKRBJ08H' +SECRET_KEY = 'askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs' +CONN_PARAMETERS = {'region': REGION, 'key': ACCESS_KEY, 'keyid': SECRET_KEY, 'profile': {}} +ERROR_MESSAGE = 'An error occurred ({}) when calling the {} operation: Test-defined error' +ERROR_CONTENT = { + 'Error': { + 'Code': 101, + 'Message': "Test-defined error" + } +} +NOT_FOUND_ERROR = ClientError({ + 'Error': { + 'Code': 'ResourceNotFoundException', + 'Message': "Test-defined error" + } +}, 'msg') +DOMAIN_RET = { + 'DomainId': 'accountno/testdomain', + 'DomainName': 'testdomain', + 'ARN': 'arn:aws:es:region:accountno:domain/testdomain', + 'Created': True, + 'Deleted': False, + 'Endpoints': { + 'vpc': 'vpc-testdomain-1234567890.region.es.amazonaws.com' + }, + 'Processing': False, + 'UpgradeProcessing': False, + 'ElasticsearchVersion': '6.3', + 'ElasticsearchClusterConfig': { + 'InstanceType': 't2.medium.elasticsearch', + 'InstanceCount': 1, + 'DedicatedMasterEnabled': False, + 'ZoneAwarenessEnabled': False, + }, + 'EBSOptions': { + 'EBSEnabled': True, + 'VolumeType': 'gp2', + 'VolumeSize': 123, + 'Iops': 12 + }, + 'AccessPolicies': textwrap.dedent(''' + {"Version":"2012-10-17","Statement":[{"Effect":"Allow", + "Principal":{"AWS":"*"},"Action":"es:*", + "Resource":"arn:aws:es:region:accountno:domain/testdomain/*"}]}'''), + 'SnapshotOptions': { + 'AutomatedSnapshotStartHour': 1 + }, + 'VPCOptions': { + 'VPCId': 'vpc-12345678', + 'SubnetIds': [ + 'subnet-deadbeef', + ], + 'AvailabilityZones': [ + 'regiona', + ], + 'SecurityGroupIds': [ + 'sg-87654321', + ] + }, + 'CognitoOptions': { + 'Enabled': False, + }, + 'EncryptionAtRestOptions': { + 'Enabled': False, + }, + 'NodeToNodeEncryptionOptions': { + 'Enabled': False + }, + 'AdvancedOptions': { + 'rest.action.multi.allow_explicit_index': 'true' + }, + 'ServiceSoftwareOptions': { + 'CurrentVersion': 'R20190221-P1', + 'NewVersion': 'R20190418', + 'UpdateAvailable': True, + 'Cancellable': False, + 'UpdateStatus': 'ELIGIBLE', + 'Description': ('A newer release R20190418 is available. This release ' + 'will be automatically deployed after somedate'), + 'AutomatedUpdateDate': None + } +} + + +@skipIf(HAS_BOTO3 is False, 'The boto module must be installed.') +@skipIf(LooseVersion(boto3.__version__) < LooseVersion(REQUIRED_BOTO3_VERSION), + 'The boto3 module must be greater or equal to version {}'.format(REQUIRED_BOTO3_VERSION)) +@skipIf(NO_MOCK, NO_MOCK_REASON) +class Boto3ElasticsearchTestCase(TestCase, LoaderModuleMockMixin): + ''' + TestCase for salt.modules.boto3_elasticsearch module + ''' + conn = None + + def setup_loader_modules(self): + self.opts = salt.config.DEFAULT_MINION_OPTS.copy() + utils = salt.loader.utils( + self.opts, + whitelist=['boto3', 'args', 'systemd', 'path', 'platform'], + context={}) + return {boto3_elasticsearch: {'__utils__': utils}} + + def setUp(self): + super(Boto3ElasticsearchTestCase, self).setUp() + boto3_elasticsearch.__init__(self.opts) + del self.opts + + # Set up MagicMock to replace the boto3 session + # connections keep getting cached from prior tests, can't find the + # correct context object to clear it. So randomize the cache key, to prevent any + # cache hits + CONN_PARAMETERS['key'] = ''.join(random.choice(string.ascii_lowercase + string.digits) + for _ in range(50)) + + self.conn = MagicMock() + self.addCleanup(delattr, self, 'conn') + self.patcher = patch('boto3.session.Session') + self.addCleanup(self.patcher.stop) + self.addCleanup(delattr, self, 'patcher') + mock_session = self.patcher.start() + session_instance = mock_session.return_value + session_instance.configure_mock(client=MagicMock(return_value=self.conn)) + self.paginator = MagicMock() + self.addCleanup(delattr, self, 'paginator') + self.conn.configure_mock(get_paginator=MagicMock(return_value=self.paginator)) + + def test_describe_elasticsearch_domain_positive(self): + ''' + Test that when describing a domain when the domain actually exists, + the .exists method returns a dict with 'result': True + and 'response' with the domain status information. + ''' + # The patch below is not neccesary per se, + # as .exists returns positive as long as no exception is raised. + with patch.object(self.conn, + 'describe_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}): + self.assertEqual( + boto3_elasticsearch.describe_elasticsearch_domain( + domain_name='testdomain', + **CONN_PARAMETERS), + {'result': True, 'response': DOMAIN_RET} + ) + + def test_describe_elasticsearch_domain_error(self): + ''' + Test that when describing a domain when the domain does not exist, + the .exists method returns a dict with 'result': False + and 'error' with boto's ResourceNotFoundException. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domain', + side_effect=NOT_FOUND_ERROR): + result = boto3_elasticsearch.describe_elasticsearch_domain( + domain_name='testdomain', + **CONN_PARAMETERS) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format('ResourceNotFoundException', 'msg') + ) + self.assertFalse(result['result']) + + def test_create_elasticsearch_domain_positive(self): + ''' + Test that when creating a domain, and it succeeds, + the .create method returns a dict with 'result': True + and 'response' with the newly created domain's status information. + ''' + with patch.object(self.conn, + 'create_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}): + kwargs = { + 'elasticsearch_version': DOMAIN_RET['ElasticsearchVersion'], + 'elasticsearch_cluster_config': DOMAIN_RET['ElasticsearchClusterConfig'], + 'ebs_options': DOMAIN_RET['EBSOptions'], + 'access_policies': DOMAIN_RET['AccessPolicies'], + 'snapshot_options': DOMAIN_RET['SnapshotOptions'], + 'vpc_options': DOMAIN_RET['VPCOptions'], + 'cognito_options': DOMAIN_RET['CognitoOptions'], + 'encryption_at_rest_options': DOMAIN_RET['EncryptionAtRestOptions'], + 'advanced_options': DOMAIN_RET['AdvancedOptions'], + } + kwargs.update(CONN_PARAMETERS) + self.assertEqual( + boto3_elasticsearch.create_elasticsearch_domain(domain_name='testdomain', **kwargs), + {'result': True, 'response': DOMAIN_RET} + ) + + def test_create_elasticsearch_domain_error(self): + ''' + Test that when creating a domain, and boto3 returns an error, + the .create method returns a dict with 'result': False + and 'error' with the error reported by boto3. + ''' + with patch.object(self.conn, + 'create_elasticsearch_domain', + side_effect=ClientError(ERROR_CONTENT, 'create_domain')): + kwargs = { + 'elasticsearch_version': DOMAIN_RET['ElasticsearchVersion'], + 'elasticsearch_cluster_config': DOMAIN_RET['ElasticsearchClusterConfig'], + 'ebs_options': DOMAIN_RET['EBSOptions'], + 'access_policies': DOMAIN_RET['AccessPolicies'], + 'snapshot_options': DOMAIN_RET['SnapshotOptions'], + 'vpc_options': DOMAIN_RET['VPCOptions'], + 'cognito_options': DOMAIN_RET['CognitoOptions'], + 'encryption_at_rest_options': DOMAIN_RET['EncryptionAtRestOptions'], + 'advanced_options': DOMAIN_RET['AdvancedOptions'], + } + kwargs.update(CONN_PARAMETERS) + result = boto3_elasticsearch.create_elasticsearch_domain('testdomain', **kwargs) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'create_domain') + ) + + def test_delete_domain_positive(self): + ''' + Test that when deleting a domain, and it succeeds, + the .delete method returns {'result': True}. + ''' + with patch.object(self.conn, 'delete_elasticsearch_domain'): + self.assertEqual( + boto3_elasticsearch.delete_elasticsearch_domain('testdomain', **CONN_PARAMETERS), + {'result': True} + ) + + def test_delete_domain_error(self): + ''' + Test that when deleting a domain, and boto3 returns an error, + the .delete method returns {'result': False, 'error' :'the error'}. + ''' + with patch.object(self.conn, + 'delete_elasticsearch_domain', + side_effect=ClientError(ERROR_CONTENT, 'delete_domain')): + result = boto3_elasticsearch.delete_elasticsearch_domain('testdomain', **CONN_PARAMETERS) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'delete_domain') + ) + + def test_update_domain_positive(self): + ''' + Test that when updating a domain succeeds, the .update method returns {'result': True}. + ''' + with patch.object(self.conn, + 'update_elasticsearch_domain_config', + return_value={'DomainConfig': DOMAIN_RET}): + kwargs = { + 'elasticsearch_cluster_config': DOMAIN_RET['ElasticsearchClusterConfig'], + 'ebs_options': DOMAIN_RET['EBSOptions'], + 'snapshot_options': DOMAIN_RET['SnapshotOptions'], + 'vpc_options': DOMAIN_RET['VPCOptions'], + 'cognito_options': DOMAIN_RET['CognitoOptions'], + 'advanced_options': DOMAIN_RET['AdvancedOptions'], + 'access_policies': DOMAIN_RET['AccessPolicies'], + 'log_publishing_options': {}, + } + + kwargs.update(CONN_PARAMETERS) + self.assertEqual( + boto3_elasticsearch.update_elasticsearch_domain_config('testdomain', **kwargs), + {'result': True, 'response': DOMAIN_RET} + ) + + def test_update_domain_error(self): + ''' + Test that when updating a domain fails, and boto3 returns an error, + the .update method returns the error. + ''' + with patch.object(self.conn, + 'update_elasticsearch_domain_config', + side_effect=ClientError(ERROR_CONTENT, 'update_domain')): + kwargs = { + 'elasticsearch_cluster_config': DOMAIN_RET['ElasticsearchClusterConfig'], + 'ebs_options': DOMAIN_RET['EBSOptions'], + 'snapshot_options': DOMAIN_RET['SnapshotOptions'], + 'vpc_options': DOMAIN_RET['VPCOptions'], + 'cognito_options': DOMAIN_RET['CognitoOptions'], + 'advanced_options': DOMAIN_RET['AdvancedOptions'], + 'access_policies': DOMAIN_RET['AccessPolicies'], + 'log_publishing_options': {}, + } + kwargs.update(CONN_PARAMETERS) + result = boto3_elasticsearch.update_elasticsearch_domain_config('testdomain', **kwargs) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'update_domain') + ) + + def test_add_tags_positive(self): + ''' + Test that when adding tags is succesful, the .add_tags method returns {'result': True}. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}): + self.assertEqual( + boto3_elasticsearch.add_tags( + 'testdomain', + tags={'foo': 'bar', 'baz': 'qux'}, + **CONN_PARAMETERS + ), + {'result': True} + ) + + def test_add_tags_error(self): + ''' + Test that when adding tags fails, and boto3 returns an error, + the .add_tags function returns {'tagged': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'add_tags', + side_effect=ClientError(ERROR_CONTENT, 'add_tags')), \ + patch.object(self.conn, + 'describe_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}): + result = boto3_elasticsearch.add_tags( + 'testdomain', + tags={'foo': 'bar', 'baz': 'qux'}, + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'add_tags') + ) + + def test_remove_tags_positive(self): + ''' + Test that when removing tags is succesful, the .remove_tags method returns {'tagged': True}. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}): + self.assertEqual( + boto3_elasticsearch.remove_tags( + tag_keys=['foo', 'bar'], + domain_name='testdomain', + **CONN_PARAMETERS), + {'result': True} + ) + + def test_remove_tag_error(self): + ''' + Test that when removing tags fails, and boto3 returns an error, + the .remove_tags method returns {'tagged': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'remove_tags', + side_effect=ClientError(ERROR_CONTENT, 'remove_tags')), \ + patch.object(self.conn, + 'describe_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}): + result = boto3_elasticsearch.remove_tags( + tag_keys=['foo', 'bar'], + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'remove_tags') + ) + + def test_list_tags_positive(self): + ''' + Test that when listing tags is succesful, + the .list_tags method returns a dict with key 'tags'. + Also test that the tags returned are manipulated properly (i.e. transformed + into a dict with tags). + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}), \ + patch.object(self.conn, + 'list_tags', + return_value={'TagList': [{'Key': 'foo', 'Value': 'bar'}]}): + result = boto3_elasticsearch.list_tags( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertEqual(result, { + 'result': True, + 'response': {'foo': 'bar'} + }) + + def test_list_tags_error(self): + ''' + Test that when listing tags causes boto3 to return an error, + the .list_tags method returns the error. + ''' + with patch.object(self.conn, + 'list_tags', + side_effect=ClientError(ERROR_CONTENT, 'list_tags')), \ + patch.object(self.conn, + 'describe_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}): + result = boto3_elasticsearch.list_tags( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'list_tags') + ) + + def test_cancel_elasticsearch_service_software_update_positive(self): + ''' + Test that when calling cancel_elasticsearch_service_software_update and + it is succesful, it returns {'result': True}. + ''' + retval = { + 'ServiceSoftwareOptions': { + 'CurrentVersion': 'string', + 'NewVersion': 'string', + 'UpdateAvailable': True, + 'Cancellable': True, + 'UpdateStatus': 'ELIGIBLE', + 'Description': 'string', + 'AutomatedUpdateDate': datetime.datetime(2015, 1, 1), + } + } + with patch.object(self.conn, + 'cancel_elasticsearch_service_software_update', + return_value=retval): + result = boto3_elasticsearch.cancel_elasticsearch_service_software_update( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertEqual(result, { + 'result': True, + }) + + def test_cancel_elasticsearch_service_software_update_error(self): + ''' + Test that when calling cancel_elasticsearch_service_software_update and + boto3 returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'cancel_elasticsearch_service_software_update', + side_effect=ClientError(ERROR_CONTENT, 'cancel_elasticsearch_service_software_update')): + result = boto3_elasticsearch.cancel_elasticsearch_service_software_update( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'cancel_elasticsearch_service_software_update') + ) + + def test_delete_elasticsearch_service_role_positive(self): + ''' + Test that when calling delete_elasticsearch_service_role and + it is succesful, it returns {'result': True}. + ''' + with patch.object(self.conn, + 'delete_elasticsearch_service_role', + return_value=None): + result = boto3_elasticsearch.delete_elasticsearch_service_role( + **CONN_PARAMETERS + ) + self.assertEqual(result, { + 'result': True, + }) + + def test_delete_elasticsearch_service_role_error(self): + ''' + Test that when calling delete_elasticsearch_service_role and boto3 returns + an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'delete_elasticsearch_service_role', + side_effect=ClientError(ERROR_CONTENT, 'delete_elasticsearch_service_role')): + result = boto3_elasticsearch.delete_elasticsearch_service_role( + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'delete_elasticsearch_service_role') + ) + + def test_describe_elasticsearch_domain_config_positive(self): + ''' + Test that when calling describe_elasticsearch_domain_config and + it is succesful, it returns {'result': True}. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domain_config', + return_value={'DomainConfig': DOMAIN_RET}): + self.assertEqual( + boto3_elasticsearch.describe_elasticsearch_domain_config('testdomain', **CONN_PARAMETERS), + {'result': True, 'response': DOMAIN_RET} + ) + + def test_describe_elasticsearch_domain_config_error(self): + ''' + Test that when calling describe_elasticsearch_domain_config and boto3 returns + an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domain_config', + side_effect=ClientError(ERROR_CONTENT, 'describe_elasticsearch_domain_config')): + result = boto3_elasticsearch.describe_elasticsearch_domain_config( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'describe_elasticsearch_domain_config') + ) + + def test_describe_elasticsearch_domains_positive(self): + ''' + Test that when calling describe_elasticsearch_domains and it is succesful, + it returns {'result': True, 'response': some_data}. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domains', + return_value={'DomainStatusList': [DOMAIN_RET]}): + self.assertEqual( + boto3_elasticsearch.describe_elasticsearch_domains( + domain_names=['test_domain'], + **CONN_PARAMETERS + ), + {'result': True, 'response': [DOMAIN_RET]} + ) + + def test_describe_elasticsearch_domains_error(self): + ''' + Test that when calling describe_elasticsearch_domains and boto3 returns + an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domains', + side_effect=ClientError(ERROR_CONTENT, 'describe_elasticsearch_domains')): + result = boto3_elasticsearch.describe_elasticsearch_domains( + domain_names=['testdomain'], + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'describe_elasticsearch_domains') + ) + + def test_describe_elasticsearch_instance_type_limits_positive(self): + ''' + Test that when calling describe_elasticsearch_instance_type_limits and + it succeeds, it returns {'result': True, 'response' some_value}. + ''' + ret_val = { + 'LimitsByRole': { + 'string': { + 'StorageTypes': [{ + 'StorageTypeName': 'string', + 'StorageSubTypeName': 'string', + 'StorageTypeLimits': [{ + 'LimitName': 'string', + 'LimitValues': ['string'], + }], + }], + 'InstanceLimits': { + 'InstanceCountLimits': { + 'MinimumInstanceCount': 123, + 'MaximumInstanceCount': 123 + } + }, + 'AdditionalLimits': [{ + 'LimitName': 'string', + 'LimitValues': ['string'] + }], + } + } + } + with patch.object(self.conn, + 'describe_elasticsearch_instance_type_limits', + return_value=ret_val): + self.assertEqual( + boto3_elasticsearch.describe_elasticsearch_instance_type_limits( + domain_name='testdomain', + instance_type='foo', + elasticsearch_version='1.0', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['LimitsByRole']} + ) + + def test_describe_elasticsearch_instance_type_limits_error(self): + ''' + Test that when calling describe_elasticsearch_instance_type_limits and boto3 returns + an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_instance_type_limits', + side_effect=ClientError(ERROR_CONTENT, 'describe_elasticsearch_instance_type_limits')): + result = boto3_elasticsearch.describe_elasticsearch_instance_type_limits( + domain_name='testdomain', + instance_type='foo', + elasticsearch_version='1.0', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'describe_elasticsearch_instance_type_limits') + ) + + def test_describe_reserved_elasticsearch_instance_offerings_positive(self): + ''' + Test that when calling describe_reserved_elasticsearch_instance_offerings + and it succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'NextToken': 'string', + 'ReservedElasticsearchInstanceOfferings': [{ + 'ReservedElasticsearchInstanceOfferingId': 'string', + 'ElasticsearchInstanceType': 't2.medium.elasticsearch', + 'Duration': 123, + 'FixedPrice': 123.0, + 'UsagePrice': 123.0, + 'CurrencyCode': 'string', + 'PaymentOption': 'NO_UPFRONT', + 'RecurringCharges': [{ + 'RecurringChargeAmount': 123.0, + 'RecurringChargeFrequency': 'string' + }] + }] + } + with patch.object(self.paginator, + 'paginate', + return_value=[ret_val]): + self.assertEqual( + boto3_elasticsearch.describe_reserved_elasticsearch_instance_offerings( + reserved_elasticsearch_instance_offering_id='foo', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['ReservedElasticsearchInstanceOfferings']} + ) + + def test_describe_reserved_elasticsearch_instance_offerings_error(self): + ''' + Test that when calling describe_reserved_elasticsearch_instance_offerings + and boto3 returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.paginator, + 'paginate', + side_effect=ClientError(ERROR_CONTENT, 'describe_reserved_elasticsearch_instance_offerings')): + result = boto3_elasticsearch.describe_reserved_elasticsearch_instance_offerings( + reserved_elasticsearch_instance_offering_id='foo', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'describe_reserved_elasticsearch_instance_offerings') + ) + + def test_describe_reserved_elasticsearch_instances_positive(self): + ''' + Test that when calling describe_reserved_elasticsearch_instances and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'NextToken': 'string', + 'ReservedElasticsearchInstances': [{ + 'ReservationName': 'string', + 'ReservedElasticsearchInstanceId': 'string', + 'ReservedElasticsearchInstanceOfferingId': 'string', + 'ElasticsearchInstanceType': 't2.medium.elasticsearch', + 'StartTime': datetime.datetime(2015, 1, 1), + 'Duration': 123, + 'FixedPrice': 123.0, + 'UsagePrice': 123.0, + 'CurrencyCode': 'string', + 'ElasticsearchInstanceCount': 123, + 'State': 'string', + 'PaymentOption': 'ALL_UPFRONT', + 'RecurringCharges': [{ + 'RecurringChargeAmount': 123.0, + 'RecurringChargeFrequency': 'string' + }, + ] + }, + ] + } + with patch.object(self.paginator, + 'paginate', + return_value=[ret_val]): + self.assertEqual( + boto3_elasticsearch.describe_reserved_elasticsearch_instances( + reserved_elasticsearch_instance_id='foo', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['ReservedElasticsearchInstances']} + ) + + def test_describe_reserved_elasticsearch_instances_error(self): + ''' + Test that when calling describe_reserved_elasticsearch_instances and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.paginator, + 'paginate', + side_effect=ClientError(ERROR_CONTENT, 'describe_reserved_elasticsearch_instances')): + result = boto3_elasticsearch.describe_reserved_elasticsearch_instances( + reserved_elasticsearch_instance_id='foo', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'describe_reserved_elasticsearch_instances') + ) + + def test_get_compatible_elasticsearch_versions_positive(self): + ''' + Test that when calling get_compatible_elasticsearch_versions and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'CompatibleElasticsearchVersions': [{ + 'SourceVersion': 'string', + 'TargetVersions': [ + 'string', + ] + }] + } + with patch.object(self.conn, + 'get_compatible_elasticsearch_versions', + return_value=ret_val): + self.assertEqual( + boto3_elasticsearch.get_compatible_elasticsearch_versions( + domain_name='testdomain', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['CompatibleElasticsearchVersions']} + ) + + def test_get_compatible_elasticsearch_versions_error(self): + ''' + Test that when calling get_compatible_elasticsearch_versions and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'get_compatible_elasticsearch_versions', + side_effect=ClientError(ERROR_CONTENT, 'get_compatible_elasticsearch_versions')): + result = boto3_elasticsearch.get_compatible_elasticsearch_versions( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'get_compatible_elasticsearch_versions') + ) + + def test_get_upgrade_history_positive(self): + ''' + Test that when calling get_upgrade_history and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'UpgradeHistories': [{ + 'UpgradeName': 'string', + 'StartTimestamp': datetime.datetime(2015, 1, 1), + 'UpgradeStatus': 'IN_PROGRESS', + 'StepsList': [{ + 'UpgradeStep': 'PRE_UPGRADE_CHECK', + 'UpgradeStepStatus': 'IN_PROGRESS', + 'Issues': [ + 'string', + ], + 'ProgressPercent': 123.0 + }] + }], + 'NextToken': 'string' + } + with patch.object(self.paginator, + 'paginate', + return_value=[ret_val]): + self.assertEqual( + boto3_elasticsearch.get_upgrade_history( + domain_name='testdomain', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['UpgradeHistories']} + ) + + def test_get_upgrade_history_error(self): + ''' + Test that when calling get_upgrade_history and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.paginator, + 'paginate', + side_effect=ClientError(ERROR_CONTENT, 'get_upgrade_history')): + result = boto3_elasticsearch.get_upgrade_history( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'get_upgrade_history') + ) + + def test_get_upgrade_status_positive(self): + ''' + Test that when calling get_upgrade_status and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'UpgradeStep': 'PRE_UPGRADE_CHECK', + 'StepStatus': 'IN_PROGRESS', + 'UpgradeName': 'string' + } + with patch.object(self.conn, + 'get_upgrade_status', + return_value=ret_val): + self.assertEqual( + boto3_elasticsearch.get_upgrade_status( + domain_name='testdomain', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val} + ) + + def test_get_upgrade_status_error(self): + ''' + Test that when calling get_upgrade_status and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'get_upgrade_status', + side_effect=ClientError(ERROR_CONTENT, 'get_upgrade_status')): + result = boto3_elasticsearch.get_upgrade_status( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'get_upgrade_status') + ) + + def test_list_domain_names_positive(self): + ''' + Test that when calling list_domain_names and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'DomainNames': [{ + 'DomainName': 'string' + }] + } + with patch.object(self.conn, + 'list_domain_names', + return_value=ret_val): + self.assertEqual( + boto3_elasticsearch.list_domain_names( + **CONN_PARAMETERS + ), + {'result': True, 'response': [item['DomainName'] for item in ret_val['DomainNames']]} + ) + + def test_list_domain_names_error(self): + ''' + Test that when calling list_domain_names and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'list_domain_names', + side_effect=ClientError(ERROR_CONTENT, 'list_domain_names')): + result = boto3_elasticsearch.list_domain_names( + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'list_domain_names') + ) + + def test_list_elasticsearch_instance_types_positive(self): + ''' + Test that when calling list_elasticsearch_instance_types and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'ElasticsearchInstanceTypes': [ + 'm3.medium.elasticsearch', 'm3.large.elasticsearch', 'm3.xlarge.elasticsearch', + 'm3.2xlarge.elasticsearch', 'm4.large.elasticsearch', 'm4.xlarge.elasticsearch', + 'm4.2xlarge.elasticsearch', 'm4.4xlarge.elasticsearch', 'm4.10xlarge.elasticsearch', + 't2.micro.elasticsearch', 't2.small.elasticsearch', 't2.medium.elasticsearch', + 'r3.large.elasticsearch', 'r3.xlarge.elasticsearch', 'r3.2xlarge.elasticsearch', + 'r3.4xlarge.elasticsearch', 'r3.8xlarge.elasticsearch', 'i2.xlarge.elasticsearch', + 'i2.2xlarge.elasticsearch', 'd2.xlarge.elasticsearch', 'd2.2xlarge.elasticsearch', + 'd2.4xlarge.elasticsearch', 'd2.8xlarge.elasticsearch', 'c4.large.elasticsearch', + 'c4.xlarge.elasticsearch', 'c4.2xlarge.elasticsearch', 'c4.4xlarge.elasticsearch', + 'c4.8xlarge.elasticsearch', 'r4.large.elasticsearch', 'r4.xlarge.elasticsearch', + 'r4.2xlarge.elasticsearch', 'r4.4xlarge.elasticsearch', 'r4.8xlarge.elasticsearch', + 'r4.16xlarge.elasticsearch', 'i3.large.elasticsearch', 'i3.xlarge.elasticsearch', + 'i3.2xlarge.elasticsearch', 'i3.4xlarge.elasticsearch', 'i3.8xlarge.elasticsearch', + 'i3.16xlarge.elasticsearch', + ], + 'NextToken': 'string' + } + with patch.object(self.paginator, + 'paginate', + return_value=[ret_val]): + self.assertEqual( + boto3_elasticsearch.list_elasticsearch_instance_types( + elasticsearch_version='1.0', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['ElasticsearchInstanceTypes']} + ) + + def test_list_elasticsearch_instance_types_error(self): + ''' + Test that when calling list_elasticsearch_instance_types and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.paginator, + 'paginate', + side_effect=ClientError(ERROR_CONTENT, 'list_elasticsearch_instance_types')): + result = boto3_elasticsearch.list_elasticsearch_instance_types( + elasticsearch_version='1.0', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'list_elasticsearch_instance_types') + ) + + def test_list_elasticsearch_versions_positive(self): + ''' + Test that when calling list_elasticsearch_versions and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'ElasticsearchVersions': ['string'], + 'NextToken': 'string' + } + with patch.object(self.paginator, + 'paginate', + return_value=[ret_val]): + self.assertEqual( + boto3_elasticsearch.list_elasticsearch_versions( + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['ElasticsearchVersions']} + ) + + def test_list_elasticsearch_versions_error(self): + ''' + Test that when calling list_elasticsearch_versions and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.paginator, + 'paginate', + side_effect=ClientError(ERROR_CONTENT, 'list_elasticsearch_versions')): + result = boto3_elasticsearch.list_elasticsearch_versions( + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'list_elasticsearch_versions') + ) + + def test_purchase_reserved_elasticsearch_instance_offering_positive(self): + ''' + Test that when calling purchase_reserved_elasticsearch_instance_offering and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'ReservedElasticsearchInstanceId': 'string', + 'ReservationName': 'string' + } + with patch.object(self.conn, + 'purchase_reserved_elasticsearch_instance_offering', + return_value=ret_val): + self.assertEqual( + boto3_elasticsearch.purchase_reserved_elasticsearch_instance_offering( + reserved_elasticsearch_instance_offering_id='foo', + reservation_name='bar', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val} + ) + + def test_purchase_reserved_elasticsearch_instance_offering_error(self): + ''' + Test that when calling purchase_reserved_elasticsearch_instance_offering and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'purchase_reserved_elasticsearch_instance_offering', + side_effect=ClientError(ERROR_CONTENT, 'purchase_reserved_elasticsearch_instance_offering')): + result = boto3_elasticsearch.purchase_reserved_elasticsearch_instance_offering( + reserved_elasticsearch_instance_offering_id='foo', + reservation_name='bar', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'purchase_reserved_elasticsearch_instance_offering') + ) + + def test_start_elasticsearch_service_software_update_positive(self): + ''' + Test that when calling start_elasticsearch_service_software_update and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'ServiceSoftwareOptions': { + 'CurrentVersion': 'string', + 'NewVersion': 'string', + 'UpdateAvailable': True, + 'Cancellable': True, + 'UpdateStatus': 'PENDING_UPDATE', + 'Description': 'string', + 'AutomatedUpdateDate': datetime.datetime(2015, 1, 1) + } + } + with patch.object(self.conn, + 'start_elasticsearch_service_software_update', + return_value=ret_val): + self.assertEqual( + boto3_elasticsearch.start_elasticsearch_service_software_update( + domain_name='testdomain', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['ServiceSoftwareOptions']} + ) + + def test_start_elasticsearch_service_software_update_error(self): + ''' + Test that when calling start_elasticsearch_service_software_update and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'start_elasticsearch_service_software_update', + side_effect=ClientError(ERROR_CONTENT, 'start_elasticsearch_service_software_update')): + result = boto3_elasticsearch.start_elasticsearch_service_software_update( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'start_elasticsearch_service_software_update') + ) + + def test_upgrade_elasticsearch_domain_positive(self): + ''' + Test that when calling upgrade_elasticsearch_domain and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'DomainName': 'string', + 'TargetVersion': 'string', + 'PerformCheckOnly': True + } + with patch.object(self.conn, + 'upgrade_elasticsearch_domain', + return_value=ret_val): + self.assertEqual( + boto3_elasticsearch.upgrade_elasticsearch_domain( + domain_name='testdomain', + target_version='1.1', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val} + ) + + def test_upgrade_elasticsearch_domain_error(self): + ''' + Test that when calling upgrade_elasticsearch_domain and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'upgrade_elasticsearch_domain', + side_effect=ClientError(ERROR_CONTENT, 'upgrade_elasticsearch_domain')): + result = boto3_elasticsearch.upgrade_elasticsearch_domain( + domain_name='testdomain', + target_version='1.1', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', {}).get('message'), + ERROR_MESSAGE.format(101, 'upgrade_elasticsearch_domain') + ) From be82c10d4edb2279b6e74b2733bd082983345685 Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Mon, 8 Jul 2019 12:40:02 +0200 Subject: [PATCH 03/12] salt/modules/boto3_elasticsearch.py: Removed debugcode. --- salt/modules/boto3_elasticsearch.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/modules/boto3_elasticsearch.py b/salt/modules/boto3_elasticsearch.py index 5a248b8c1e46..d356cf3d5352 100644 --- a/salt/modules/boto3_elasticsearch.py +++ b/salt/modules/boto3_elasticsearch.py @@ -1130,9 +1130,6 @@ def upgrade_elasticsearch_domain( if blocking: waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESUpgradeFinished2') waiter.wait(DomainName=domain_name) - frop = describe_elasticsearch_domain(domain_name, region=region, keyid=keyid, key=key, profile=profile) - log.debug(__name__ + ':upgrade_elasticsearch_domain:\n' - '\t\tfrop: {}'.format(frop)) except (ParamValidationError, ClientError, WaiterError) as exp: ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) return ret From 6e569481fdcbc4b3139257970425d1f4d10c4f34 Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Mon, 8 Jul 2019 13:15:52 +0200 Subject: [PATCH 04/12] Added yaml code examples in docstrings. --- salt/states/boto3_elasticsearch.py | 73 ++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/salt/states/boto3_elasticsearch.py b/salt/states/boto3_elasticsearch.py index b8d1c511459a..f189e627d92c 100644 --- a/salt/states/boto3_elasticsearch.py +++ b/salt/states/boto3_elasticsearch.py @@ -174,6 +174,40 @@ def present( .. versionadded:: Natrium + Example: + + This will create an elasticsearch domain consisting of a single t2.small instance + in the eu-west-1 region (Ireland) and will wait until the instance is available + before returning from the state. + + .. code-block:: yaml + + Create new domain: + boto3_elasticsearch.present: + - name: my_domain + - elasticsearch_version: '5.1' + - elasticsearch_cluster_config: + InstanceType: t2.small.elasticsearch + InstanceCount: 1 + DedicatedMasterEnabled: False + ZoneAwarenessEnabled: False + - ebs_options: + EBSEnabled: True + VolumeType: gp2 + VolumeSize: 10 + - snapshot_options: + AutomatedSnapshotStartHour: 3 + - vpc_options: + SubnetIds: + - subnet-12345678 + SecurityGroupIds: + - sg-12345678 + - node_to_node_encryption_options: + Enabled: False + - region: eu-west-1 + - tags: + foo: bar + baz: qux ''' ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} @@ -331,6 +365,14 @@ def absent( .. versionadded:: Natrium + Example: + + .. code-block:: yaml + + Remove Elasticsearch Domain: + boto3_elasticsearch.absent: + - name: my_domain + - region: eu-west-1 ''' ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} @@ -386,6 +428,18 @@ def upgraded( :param str name: The name of the Elasticsearch domain to upgrade. :param str elasticsearch_version: String of format X.Y to specify version for the Elasticsearch domain eg. "1.5" or "2.3". + + .. versionadded:: Natrium + + Example: + + .. code-block:: yaml + + Upgrade Elasticsearch Domain: + boto3_elasticsearch.upgraded: + - name: my_domain + - elasticsearch_version: '7.2' + - region: eu-west-1 ''' ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} current_domain = None @@ -508,6 +562,22 @@ def latest( :param str name: The name of the Elasticsearch domain to upgrade. :param bool minor_only: Only upgrade to the latest minor version. + + .. versionadded:: Natrium + + Example: + + The following example will ensure the elasticsearch domain ``my_domain`` is + upgraded to the latest minor version. So if it is currently 5.1 it will be + upgraded to 5.6. + + .. code-block:: yaml + + Upgrade Elasticsearch Domain: + boto3_elasticsearch.latest: + - name: my_domain + - minor_only: True + - region: eu-west-1 ''' ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} # Get current version @@ -592,6 +662,9 @@ def tagged( :param dict tags: The tags to add to/replace on the Elasticsearch domain. :param bool replace: Whether or not to replace (``True``) all existing tags on the Elasticsearch domain, or add (``False``) tags to the ES domain. + + .. versionadded:: Natrium + ''' ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} current_tags = {} From 2ed357e2dcdd98e7a81f5a7ba96ac0beb4db82cb Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Mon, 8 Jul 2019 14:12:10 +0200 Subject: [PATCH 05/12] Removed ESUpgradeFinished waiter definition as this was not accurate enough. Renamed ESUpgradeFinished2 to ESUpgradeFinished instead. Renamed references to ESUpgradeFInished2 in the execution module. --- salt/modules/boto3_elasticsearch.py | 4 ++-- salt/utils/boto3_elasticsearch.py | 23 +---------------------- 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/salt/modules/boto3_elasticsearch.py b/salt/modules/boto3_elasticsearch.py index d356cf3d5352..ce08c9741c18 100644 --- a/salt/modules/boto3_elasticsearch.py +++ b/salt/modules/boto3_elasticsearch.py @@ -1128,7 +1128,7 @@ def upgrade_elasticsearch_domain( ret['result'] = True ret['response'] = res if blocking: - waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESUpgradeFinished2') + waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESUpgradeFinished') waiter.wait(DomainName=domain_name) except (ParamValidationError, ClientError, WaiterError) as exp: ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) @@ -1178,7 +1178,7 @@ def wait_for_upgrade( ret = {'result': False} try: conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) - waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESUpgradeFinished2') + waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESUpgradeFinished') waiter.wait(DomainName=domain_name) ret['result'] = True except (ParamValidationError, ClientError, WaiterError) as exp: diff --git a/salt/utils/boto3_elasticsearch.py b/salt/utils/boto3_elasticsearch.py index bdb5ac4308c6..0aa84f45edb0 100644 --- a/salt/utils/boto3_elasticsearch.py +++ b/salt/utils/boto3_elasticsearch.py @@ -35,7 +35,7 @@ 'argument': 'DomainConfig.*.Status.PendingDeletion', }], }, - 'ESUpgradeFinished2': { + 'ESUpgradeFinished': { 'delay': 60, 'operation': 'DescribeElasticsearchDomain', 'maxAttempts': 60, @@ -46,27 +46,6 @@ 'argument': 'DomainStatus.UpgradeProcessing', }], }, - 'ESUpgradeFinished': { - 'delay': 60, - 'operation': 'GetUpgradeStatus', - 'maxAttempts': 60, - 'acceptors': [{ - 'expected': 'SUCCEEDED', - 'matcher': 'path', - 'state': 'success', - 'argument': 'StepStatus' - }, { - 'expected': 'FAILED', - 'matcher': 'path', - 'state': 'success', - 'argument': 'StepStatus', - }, { - 'expected': 'SUCCEEDED_WITH_ISSUES', - 'matcher': 'path', - 'state': 'success', - 'argument': 'StepStatus', - }], - }, 'ESDomainDeleted': { 'delay': 30, 'operation': 'DescribeElasticsearchDomain', From 310e2d1730deba808303b99d826f622159941765 Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Mon, 8 Jul 2019 15:13:03 +0200 Subject: [PATCH 06/12] Fixed tests with the changed structure of errormessages. --- .../unit/modules/test_boto3_elasticsearch.py | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/tests/unit/modules/test_boto3_elasticsearch.py b/tests/unit/modules/test_boto3_elasticsearch.py index 37b7b3cebcc3..563cee5a7e01 100644 --- a/tests/unit/modules/test_boto3_elasticsearch.py +++ b/tests/unit/modules/test_boto3_elasticsearch.py @@ -206,7 +206,7 @@ def test_describe_elasticsearch_domain_error(self): domain_name='testdomain', **CONN_PARAMETERS) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format('ResourceNotFoundException', 'msg') ) self.assertFalse(result['result']) @@ -260,7 +260,7 @@ def test_create_elasticsearch_domain_error(self): kwargs.update(CONN_PARAMETERS) result = boto3_elasticsearch.create_elasticsearch_domain('testdomain', **kwargs) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'create_domain') ) @@ -286,7 +286,7 @@ def test_delete_domain_error(self): result = boto3_elasticsearch.delete_elasticsearch_domain('testdomain', **CONN_PARAMETERS) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'delete_domain') ) @@ -335,7 +335,7 @@ def test_update_domain_error(self): kwargs.update(CONN_PARAMETERS) result = boto3_elasticsearch.update_elasticsearch_domain_config('testdomain', **kwargs) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'update_domain') ) @@ -373,7 +373,7 @@ def test_add_tags_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'add_tags') ) @@ -410,7 +410,7 @@ def test_remove_tag_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'remove_tags') ) @@ -453,7 +453,7 @@ def test_list_tags_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'list_tags') ) @@ -498,7 +498,7 @@ def test_cancel_elasticsearch_service_software_update_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'cancel_elasticsearch_service_software_update') ) @@ -530,7 +530,7 @@ def test_delete_elasticsearch_service_role_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'delete_elasticsearch_service_role') ) @@ -561,7 +561,7 @@ def test_describe_elasticsearch_domain_config_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'describe_elasticsearch_domain_config') ) @@ -595,7 +595,7 @@ def test_describe_elasticsearch_domains_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'describe_elasticsearch_domains') ) @@ -657,7 +657,7 @@ def test_describe_elasticsearch_instance_type_limits_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'describe_elasticsearch_instance_type_limits') ) @@ -707,7 +707,7 @@ def test_describe_reserved_elasticsearch_instance_offerings_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'describe_reserved_elasticsearch_instance_offerings') ) @@ -764,7 +764,7 @@ def test_describe_reserved_elasticsearch_instances_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'describe_reserved_elasticsearch_instances') ) @@ -806,7 +806,7 @@ def test_get_compatible_elasticsearch_versions_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'get_compatible_elasticsearch_versions') ) @@ -856,7 +856,7 @@ def test_get_upgrade_history_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'get_upgrade_history') ) @@ -868,7 +868,8 @@ def test_get_upgrade_status_positive(self): ret_val = { 'UpgradeStep': 'PRE_UPGRADE_CHECK', 'StepStatus': 'IN_PROGRESS', - 'UpgradeName': 'string' + 'UpgradeName': 'string', + 'ResponseMetadata': None, } with patch.object(self.conn, 'get_upgrade_status', @@ -895,7 +896,7 @@ def test_get_upgrade_status_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'get_upgrade_status') ) @@ -932,7 +933,7 @@ def test_list_domain_names_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'list_domain_names') ) @@ -985,7 +986,7 @@ def test_list_elasticsearch_instance_types_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'list_elasticsearch_instance_types') ) @@ -1021,7 +1022,7 @@ def test_list_elasticsearch_versions_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'list_elasticsearch_versions') ) @@ -1061,7 +1062,7 @@ def test_purchase_reserved_elasticsearch_instance_offering_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'purchase_reserved_elasticsearch_instance_offering') ) @@ -1106,7 +1107,7 @@ def test_start_elasticsearch_service_software_update_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'start_elasticsearch_service_software_update') ) @@ -1147,6 +1148,6 @@ def test_upgrade_elasticsearch_domain_error(self): ) self.assertFalse(result['result']) self.assertEqual( - result.get('error', {}).get('message'), + result.get('error', ''), ERROR_MESSAGE.format(101, 'upgrade_elasticsearch_domain') ) From 633b140121dc11de8842e87d465b8d55244c3582 Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Mon, 8 Jul 2019 15:42:44 +0200 Subject: [PATCH 07/12] salt/states/boto3_elasticsearch.py: Fix pylint warnings. --- salt/states/boto3_elasticsearch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/states/boto3_elasticsearch.py b/salt/states/boto3_elasticsearch.py index f189e627d92c..0a3558ce0d51 100644 --- a/salt/states/boto3_elasticsearch.py +++ b/salt/states/boto3_elasticsearch.py @@ -468,7 +468,7 @@ def upgraded( ''.format(name, elasticsearch_version)) if isinstance(ret['result'], bool): return ret - log.debug(__name__ + ':upgraded: Check upgrade in progress') + log.debug('%s :upgraded: Check upgrade in progress', __name__) # Check if an upgrade is already in progress res = __salt__['boto3_elasticsearch.get_upgrade_status']( name, @@ -503,7 +503,7 @@ def upgraded( if isinstance(ret['result'], bool): return ret - log.debug(__name__ + ':upgraded: Check upgrade eligibility') + log.debug('%s :upgraded: Check upgrade eligibility', __name__) # Check if the domain is eligible for an upgrade res = __salt__['boto3_elasticsearch.check_upgrade_eligibility']( name, @@ -519,7 +519,7 @@ def upgraded( 'be upgraded to version {}.' ''.format(name, elasticsearch_version)) else: - log.debug(__name__ + ':upgraded: Start the upgrade') + log.debug('%s :upgraded: Start the upgrade', __name__) # Start the upgrade if __opts__['test']: ret['result'] = None From 958b3d10843673a197a9ccbbce886088224ef067 Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Mon, 8 Jul 2019 18:29:01 +0200 Subject: [PATCH 08/12] Added and fixed documentation. --- doc/ref/modules/all/index.rst | 1 + .../all/salt.modules.boto3_elasticsearch.rst | 5 + doc/ref/states/all/index.rst | 1 + .../all/salt.states.boto3_elasticsearch.rst | 6 + salt/modules/boto3_elasticsearch.py | 336 +++++++++++------- salt/states/boto3_elasticsearch.py | 119 ++++--- 6 files changed, 287 insertions(+), 181 deletions(-) create mode 100644 doc/ref/modules/all/salt.modules.boto3_elasticsearch.rst create mode 100644 doc/ref/states/all/salt.states.boto3_elasticsearch.rst diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index 339b3ebe4d76..ea61b2895c66 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -48,6 +48,7 @@ execution modules bigip bluez_bluetooth boto3_elasticache + boto3_elasticsearch boto3_route53 boto_apigateway boto_asg diff --git a/doc/ref/modules/all/salt.modules.boto3_elasticsearch.rst b/doc/ref/modules/all/salt.modules.boto3_elasticsearch.rst new file mode 100644 index 000000000000..07c5408aa145 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.boto3_elasticsearch.rst @@ -0,0 +1,5 @@ +salt.modules.boto3_elasticsearch module +======================================= + +.. automodule:: salt.modules.boto3_elasticsearch + :members: diff --git a/doc/ref/states/all/index.rst b/doc/ref/states/all/index.rst index 699895d8a226..680083e13370 100644 --- a/doc/ref/states/all/index.rst +++ b/doc/ref/states/all/index.rst @@ -31,6 +31,7 @@ state modules bigip blockdev boto3_elasticache + boto3_elasticsearch boto3_route53 boto_apigateway boto_asg diff --git a/doc/ref/states/all/salt.states.boto3_elasticsearch.rst b/doc/ref/states/all/salt.states.boto3_elasticsearch.rst new file mode 100644 index 000000000000..5c16b2acc0a5 --- /dev/null +++ b/doc/ref/states/all/salt.states.boto3_elasticsearch.rst @@ -0,0 +1,6 @@ +salt.states.boto3_elasticsearch module +====================================== + +.. automodule:: salt.states.boto3_elasticsearch + :members: + :undoc-members: diff --git a/salt/modules/boto3_elasticsearch.py b/salt/modules/boto3_elasticsearch.py index ce08c9741c18..980f42cd8ecd 100644 --- a/salt/modules/boto3_elasticsearch.py +++ b/salt/modules/boto3_elasticsearch.py @@ -107,7 +107,8 @@ def add_tags( Specifying this overrides ``domain_name``. :param dict tags: The dict of tags to add to the Elasticsearch domain. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon failure, also contains a key 'error' with the error message as value. As a special case, tags whose key starts with `__` are ignored. @@ -160,7 +161,8 @@ def cancel_elasticsearch_service_software_update( :param str domain_name: The name of the domain that you want to stop the latest service software update on. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with the current service software options. Upon failure, also contains a key 'error' with the error message as value. @@ -202,76 +204,94 @@ def create_elasticsearch_domain( the following characters: a-z (lowercase), 0-9, and - (hyphen). :param str elasticsearch_version: String of format X.Y to specify version for the Elasticsearch domain eg. "1.5" or "2.3". - :param dict elasticsearch_cluster_config: Dict specifying the configuration - options for an Elasticsearch domain. Sub-options contained here are: - :param str InstanceType: The instance type for an Elasticsearch cluster. - :param int InstanceCount: The instance type for an Elasticsearch cluster. - :param bool DedicatedMasterEnabled: Indicate whether a dedicated master - node is enabled. - :param bool ZoneAwarenessEnabled: Indicate whether zone awareness is enabled. - If this is not enabled, the Elasticsearch domain will only be in one - availability zone. - :param dict ZoneAwarenessConfig: Specifies the zone awareness configuration - for a domain when zone awareness is enabled. Sub-options contained - here are: - :param int AvailabilityZoneCount: An integer value to indicate the - number of availability zones for a domain when zone awareness is - enabled. This should be equal to number of subnets if VPC endpoints - is enabled. Allowed values: 2, 3 - :param str DedicatedMasterType: The instance type for a dedicated master node. - :param int DedicatedMasterCount: Total number of dedicated master nodes, - active and on standby, for the cluster. + :param dict elasticsearch_cluster_config: Dictionary specifying the configuration + options for an Elasticsearch domain. Keys (case sensitive) in here are: + + - InstanceType (str): The instance type for an Elasticsearch cluster. + - InstanceCount (int): The instance type for an Elasticsearch cluster. + - DedicatedMasterEnabled (bool): Indicate whether a dedicated master + node is enabled. + - ZoneAwarenessEnabled (bool): Indicate whether zone awareness is enabled. + If this is not enabled, the Elasticsearch domain will only be in one + availability zone. + - ZoneAwarenessConfig (dict): Specifies the zone awareness configuration + for a domain when zone awareness is enabled. + Keys (case sensitive) in here are: + + - AvailabilityZoneCount (int): An integer value to indicate the + number of availability zones for a domain when zone awareness is + enabled. This should be equal to number of subnets if VPC endpoints + is enabled. Allowed values: 2, 3 + + - DedicatedMasterType (str): The instance type for a dedicated master node. + - DedicatedMasterCount (int): Total number of dedicated master nodes, + active and on standby, for the cluster. :param dict ebs_options: Dict specifying the options to enable or disable and - specifying the type and size of EBS storage volumes. - Sub-options contained here are: - :param bool EBSEnabled: Specifies whether EBS-based storage is enabled. - :param str VolumeType: Specifies the volume type for EBS-based storage. - :param int VolumeSize: Integer to specify the size of an EBS volume. - :param int Iops: Specifies the IOPD for a Provisioned IOPS EBS volume (SSD). - :param str/dict access_policies: Dict or JSON string with the IAM access policy. + specifying the type and size of EBS storage volumes. + Keys (case sensitive) in here are: + + - EBSEnabled (bool): Specifies whether EBS-based storage is enabled. + - VolumeType (str): Specifies the volume type for EBS-based storage. + - VolumeSize (int): Integer to specify the size of an EBS volume. + - Iops (int): Specifies the IOPD for a Provisioned IOPS EBS volume (SSD). + :type access_policies: str or dict + :param access_policies: Dict or JSON string with the IAM access policy. :param dict snapshot_options: Dict specifying the snapshot options. - Sub-options contained here are: - :param int AutomatedSnapshotStartHour: Specifies the time, in UTC format, - when the service takes a daily automated snapshot of the specified - Elasticsearch domain. Default value is 0 hours. + Keys (case sensitive) in here are: + + - AutomatedSnapshotStartHour (int): Specifies the time, in UTC format, + when the service takes a daily automated snapshot of the specified + Elasticsearch domain. Default value is 0 hours. :param dict vpc_options: Dict with the options to specify the subnets and security - groups for the VPC endpoint. Sub-options contained here are: - :param list SubnetIds: The list of subnets for the VPC endpoint. - :param list SecurityGroupIds: The list of security groups for the VPC endpoint. + groups for the VPC endpoint. + Keys (case sensitive) in here are: + + - SubnetIds (list): The list of subnets for the VPC endpoint. + - SecurityGroupIds (list): The list of security groups for the VPC endpoint. :param dict cognito_options: Dict with options to specify the cognito user and - identity pools for Kibana authentication. Sub-options contained here are: - :param bool Enabled: Specifies the option to enable Cognito for Kibana authentication. - :param str UserPoolId: Specifies the Cognito user pool ID for Kibana authentication. - :param str IdentityPoolId: Specifies the Cognito identity pool ID for Kibana authentication. - :param str RoleArn: Specifies the role ARN that provides Elasticsearch permissions - for accessing Cognito resources. + identity pools for Kibana authentication. + Keys (case sensitive) in here are: + + - Enabled (bool): Specifies the option to enable Cognito for Kibana authentication. + - UserPoolId (str): Specifies the Cognito user pool ID for Kibana authentication. + - IdentityPoolId (str): Specifies the Cognito identity pool ID for Kibana authentication. + - RoleArn (str): Specifies the role ARN that provides Elasticsearch permissions + for accessing Cognito resources. :param dict encryption_at_rest_options: Dict specifying the encryption at rest - options. Sub-options contained here are: - :param bool Enabled: Specifies the option to enable Encryption At Rest. - :param str KmsKeyId: Specifies the KMS Key ID for Encryption At Rest options. + options. Keys (case sensitive) in here are: + + - Enabled (bool): Specifies the option to enable Encryption At Rest. + - KmsKeyId (str): Specifies the KMS Key ID for Encryption At Rest options. :param dict node_to_node_encryption_options: Dict specifying the node to node - encryption options. Sub-options contained here are: - :param bool Enabled: Specify True to enable node-to-node encryption. + encryption options. Keys (case sensitive) in here are: + + - Enabled (bool): Specify True to enable node-to-node encryption. :param dict advanced_options: Dict with option to allow references to indices in an HTTP request body. Must be False when configuring access to individual sub-resources. By default, the value is True. See http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide\ - /es-createupdatedomains.html#es-createdomain-configure-advanced-options + /es-createupdatedomains.html#es-createdomain-configure-advanced-options for more information. :param dict log_publishing_options: Dict with options for various type of logs. - The keys denote the type of log file and can be one of the following: - INDEX_SLOW_LOGS, SEARCH_SLOW_LOGS, ES_APPLICATION_LOGS. - The value assigned to each key is a dict with the following sub-options: - :param str CloudWatchLogsLogGroupArn: The ARN of the Cloudwatch log - group to which the log needs to be published. - :param bool Enabled: Specifies whether given log publishing option is enabled or not. + The keys denote the type of log file and can be one of the following: + + - INDEX_SLOW_LOGS + - SEARCH_SLOW_LOGS + - ES_APPLICATION_LOGS + + The value assigned to each key is a dict with the following case sensitive keys: + + - CloudWatchLogsLogGroupArn (str): The ARN of the Cloudwatch log + group to which the log needs to be published. + - Enabled (bool): Specifies whether given log publishing option is enabled or not. :param bool blocking: Whether or not to wait (block) until the Elasticsearch domain has been created. Note: Not all instance types allow enabling encryption at rest. See https://docs.aws.amazon.com\ /elasticsearch-service/latest/developerguide/aes-supported-instance-types.html - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with the domain status configuration. Upon failure, also contains a key 'error' with the error message as value. @@ -292,7 +312,7 @@ def create_elasticsearch_domain( "VolumeType": "gp2", \\ "VolumeSize": 10, \\ "Iops": 0}' \\ - access_policies='{ + access_policies='{ \\ "Version": "2012-10-17", \\ "Statement": [ \\ {"Effect": "Allow", \\ @@ -302,6 +322,7 @@ def create_elasticsearch_domain( "Condition": {"IpAddress": {"aws:SourceIp": ["127.0.0.1"]}}}]} \\ snapshot_options='{"AutomatedSnapshotStartHour": 0}' \\ advanced_options='{"rest.action.multi.allow_explicit_index": "true"}' + ''' boto_kwargs = salt.utils.data.filter_falsey({ 'DomainName': domain_name, @@ -346,7 +367,8 @@ def delete_elasticsearch_domain( :param bool blocking: Whether or not to wait (block) until the Elasticsearch domain has been deleted. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon failure, also contains a key 'error' with the error message as value. .. versionadded:: Natrium @@ -356,6 +378,7 @@ def delete_elasticsearch_domain( .. code-block:: bash salt myminion boto_elasticsearch_domain.delete mydomain + ''' ret = {'result': False} try: @@ -378,7 +401,8 @@ def delete_elasticsearch_service_role( maintain VPC domains. Role deletion will fail if any existing VPC domains use the role. You must delete any such Elasticsearch domains before deleting the role. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon failure, also contains a key 'error' with the error message as value. .. versionadded:: Natrium @@ -402,7 +426,8 @@ def describe_elasticsearch_domain( :param str domain_name: The name of the domain to get the status of. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary ith key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with the domain status information. Upon failure, also contains a key 'error' with the error message as value. @@ -436,7 +461,8 @@ def describe_elasticsearch_domain_config( :param str domain_name: The name of the domain to describe. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with the current configuration information. Upon failure, also contains a key 'error' with the error message as value. @@ -447,6 +473,7 @@ def describe_elasticsearch_domain_config( .. code-block:: bash salt myminion boto_elasticsearch_domain.describe mydomain + ''' ret = {'result': False} try: @@ -469,9 +496,13 @@ def describe_elasticsearch_domains( :param list domain_names: List of domain names to get information for. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with the list of domain status information. Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + ''' ret = {'result': False} try: @@ -504,7 +535,8 @@ def describe_elasticsearch_instance_type_limits( to modify. This should be present only if we are querying for Elasticsearch ``Limits`` for existing domain. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with the limits information. Upon failure, also contains a key 'error' with the error message as value. @@ -539,7 +571,8 @@ def describe_reserved_elasticsearch_instance_offerings( filter value. Use this parameter to show only the available offering that matches the specified reservation identifier. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with the list of offerings information. Upon failure, also contains a key 'error' with the error message as value. @@ -576,9 +609,10 @@ def describe_reserved_elasticsearch_instances( filter value. Use this parameter to show only the reservation that matches the specified reserved Elasticsearch instance ID. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with a list of information on - reserved instances. + reserved instances. Upon failure, also contains a key 'error' with the error message as value. Note: Version 1.9.174 of boto3 has a bug in that reserved_elasticsearch_instance_id @@ -617,7 +651,8 @@ def get_compatible_elasticsearch_versions( :param str domain_name: The name of an Elasticsearch domain. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with a list of compatible versions. Upon failure, also contains a key 'error' with the error message as value. @@ -651,7 +686,8 @@ def get_upgrade_history( names start with a letter or number and can contain the following characters: a-z (lowercase), 0-9, and - (hyphen). - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with a list of upgrade histories. Upon failure, also contains a key 'error' with the error message as value. @@ -686,7 +722,8 @@ def get_upgrade_status( names start with a letter or number and can contain the following characters: a-z (lowercase), 0-9, and - (hyphen). - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with upgrade status information. Upon failure, also contains a key 'error' with the error message as value. @@ -711,7 +748,8 @@ def list_domain_names( ''' Returns the name of all Elasticsearch domains owned by the current user's account. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with a list of domain names. Upon failure, also contains a key 'error' with the error message as value. @@ -744,7 +782,8 @@ def list_elasticsearch_instance_types( are trying to modify. This should be present only if we are querying for list of available Elasticsearch instance types when modifying existing domain. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with a list of Elasticsearch instance types. Upon failure, also contains a key 'error' with the error message as value. @@ -775,7 +814,8 @@ def list_elasticsearch_versions( ''' List all supported Elasticsearch versions. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with a list of Elasticsearch versions. Upon failure, also contains a key 'error' with the error message as value. @@ -803,7 +843,8 @@ def list_tags( ''' Returns all tags for the given Elasticsearch domain. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with a dict of tags. Upon failure, also contains a key 'error' with the error message as value. @@ -814,6 +855,7 @@ def list_tags( .. code-block:: bash salt myminion boto_elasticsearch.list_tags my_domain + ''' if not any((arn, domain_name)): raise SaltInvocationError('At least one of domain_name or arn must be specified.') @@ -853,7 +895,8 @@ def purchase_reserved_elasticsearch_instance_offering( :param str reservation_name: A customer-specified identifier to track this reservation. :param int instance_count: The number of Elasticsearch instances to reserve. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with purchase information. Upon failure, also contains a key 'error' with the error message as value. @@ -890,7 +933,8 @@ def remove_tags( :param str arn: The ARN of the Elasticsearch domain you want to remove tags from. Specifying this overrides ``domain_name``. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon failure, also contains a key 'error' with the error message as value. .. versionadded:: Natrium @@ -899,7 +943,8 @@ def remove_tags( .. code-block:: bash - salt myminion boto_cloudtrail.remove_tags my_trail tag_a=tag_value tag_b=tag_value + salt myminion boto_cloudtrail.remove_tags tag_keys='["foo", "bar"]' my_domain + ''' if not any((arn, domain_name)): raise SaltInvocationError('At least one of domain_name or arn must be specified.') @@ -935,7 +980,8 @@ def start_elasticsearch_service_software_update( :param str domain_name: The name of the domain that you want to update to the latest service software. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with service software information. Upon failure, also contains a key 'error' with the error message as value. @@ -971,68 +1017,82 @@ def update_elasticsearch_domain_config( Modifies the cluster configuration of the specified Elasticsearch domain, for example setting the instance type and the number of instances. - param str domain_name: The name of the Elasticsearch domain that you are creating. + :param str domain_name: The name of the Elasticsearch domain that you are creating. Domain names are unique across the domains owned by an account within an AWS region. Domain names must start with a letter or number and can contain the following characters: a-z (lowercase), 0-9, and - (hyphen). - param dict elasticsearch_cluster_config: Dict specifying the configuration - options for an Elasticsearch domain. Sub-options contained here are: - param str InstanceType: The instance type for an Elasticsearch cluster. - param int InstanceCount: The instance type for an Elasticsearch cluster. - param bool DedicatedMasterEnabled: Indicate whether a dedicated master - node is enabled. - param bool ZoneAwarenessEnabled: Indicate whether zone awareness is enabled. - param dict ZoneAwarenessConfig: Specifies the zone awareness configuration - for a domain when zone awareness is enabled. Sub-options contained - here are: - param int AvailabilityZoneCount: An integer value to indicate the - number of availability zones for a domain when zone awareness is - enabled. This should be equal to number of subnets if VPC endpoints - is enabled. - param str DedicatedMasterType: The instance type for a dedicated master node. - param int DedicatedMasterCount: Total number of dedicated master nodes, - active and on standby, for the cluster. - param dict ebs_options: Dict specifying the options to enable or disable and - specifying the type and size of EBS storage volumes. - Sub-options contained here are: - param bool EBSEnabled: Specifies whether EBS-based storage is enabled. - param str VolumeType: Specifies the volume type for EBS-based storage. - param int VolumeSize: Integer to specify the size of an EBS volume. - param int Iops: Specifies the IOPD for a Provisioned IOPS EBS volume (SSD). - param dict snapshot_options: Dict specifying the snapshot options. - Sub-options contained here are: - param int AutomatedSnapshotStartHour: Specifies the time, in UTC format, - when the service takes a daily automated snapshot of the specified - Elasticsearch domain. Default value is 0 hours. - param dict vpc_options: Dict with the options to specify the subnets and security - groups for the VPC endpoint. Sub-options contained here are: - param list SubnetIds: The list of subnets for the VPC endpoint. - param list SecurityGroupIds: The list of security groups for the VPC endpoint. - param dict cognito_options: Dict with options to specify the cognito user and - identity pools for Kibana authentication. Sub-options contained here are: - param bool Enabled: Specifies the option to enable Cognito for Kibana authentication. - param str UserPoolId: Specifies the Cognito user pool ID for Kibana authentication. - param str IdentityPoolId: Specifies the Cognito identity pool ID for Kibana authentication. - param str RoleArn: Specifies the role ARN that provides Elasticsearch permissions - for accessing Cognito resources. - param dict advanced_options: Dict with option to allow references to indices + :param dict elasticsearch_cluster_config: Dictionary specifying the configuration + options for an Elasticsearch domain. Keys (case sensitive) in here are: + + - InstanceType (str): The instance type for an Elasticsearch cluster. + - InstanceCount (int): The instance type for an Elasticsearch cluster. + - DedicatedMasterEnabled (bool): Indicate whether a dedicated master + node is enabled. + - ZoneAwarenessEnabled (bool): Indicate whether zone awareness is enabled. + - ZoneAwarenessConfig (dict): Specifies the zone awareness configuration + for a domain when zone awareness is enabled. + Keys (case sensitive) in here are: + + - AvailabilityZoneCount (int): An integer value to indicate the + number of availability zones for a domain when zone awareness is + enabled. This should be equal to number of subnets if VPC endpoints + is enabled. + + - DedicatedMasterType (str): The instance type for a dedicated master node. + - DedicatedMasterCount (int): Total number of dedicated master nodes, + active and on standby, for the cluster. + :param dict ebs_options: Dict specifying the options to enable or disable and + specifying the type and size of EBS storage volumes. + Keys (case sensitive) in here are: + + - EBSEnabled (bool): Specifies whether EBS-based storage is enabled. + - VolumeType (str): Specifies the volume type for EBS-based storage. + - VolumeSize (int): Integer to specify the size of an EBS volume. + - Iops (int): Specifies the IOPD for a Provisioned IOPS EBS volume (SSD). + :param dict snapshot_options: Dict specifying the snapshot options. + Keys (case sensitive) in here are: + + - AutomatedSnapshotStartHour (int): Specifies the time, in UTC format, + when the service takes a daily automated snapshot of the specified + Elasticsearch domain. Default value is 0 hours. + :param dict vpc_options: Dict with the options to specify the subnets and security + groups for the VPC endpoint. + Keys (case sensitive) in here are: + + - SubnetIds (list): The list of subnets for the VPC endpoint. + - SecurityGroupIds (list): The list of security groups for the VPC endpoint. + :param dict cognito_options: Dict with options to specify the cognito user and + identity pools for Kibana authentication. + Keys (case sensitive) in here are: + + - Enabled (bool): Specifies the option to enable Cognito for Kibana authentication. + - UserPoolId (str): Specifies the Cognito user pool ID for Kibana authentication. + - IdentityPoolId (str): Specifies the Cognito identity pool ID for Kibana authentication. + - RoleArn (str): Specifies the role ARN that provides Elasticsearch permissions + for accessing Cognito resources. + :param dict advanced_options: Dict with option to allow references to indices in an HTTP request body. Must be False when configuring access to individual sub-resources. By default, the value is True. See http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide\ - /es-createupdatedomains.html#es-createdomain-configure-advanced-options + /es-createupdatedomains.html#es-createdomain-configure-advanced-options for more information. - param str/dict access_policies: Dict or JSON string with the IAM access policy. - param dict log_publishing_options: Dict with options for various type of logs. - The keys denote the type of log file and can be one of the following: - INDEX_SLOW_LOGS, SEARCH_SLOW_LOGS, ES_APPLICATION_LOGS. - The value assigned to each key is a dict with the following sub-options: - param str CloudWatchLogsLogGroupArn: The ARN of the Cloudwatch log - group to which the log needs to be published. - param bool Enabled: Specifies whether given log publishing option is enabled or not. + :param str/dict access_policies: Dict or JSON string with the IAM access policy. + :param dict log_publishing_options: Dict with options for various type of logs. + The keys denote the type of log file and can be one of the following: + + INDEX_SLOW_LOGS, SEARCH_SLOW_LOGS, ES_APPLICATION_LOGS. + + The value assigned to each key is a dict with the following case sensitive keys: + + - CloudWatchLogsLogGroupArn (str): The ARN of the Cloudwatch log + group to which the log needs to be published. + - Enabled (bool): Specifies whether given log publishing option + is enabled or not. :param bool blocking: Whether or not to wait (block) until the Elasticsearch domain has been updated. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with the domain configuration. Upon failure, also contains a key 'error' with the error message as value. @@ -1108,7 +1168,8 @@ def upgrade_elasticsearch_domain( :param bool blocking: Whether or not to wait (block) until the Elasticsearch domain has been upgraded. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with the domain configuration. Upon failure, also contains a key 'error' with the error message as value. @@ -1143,7 +1204,8 @@ def exists( :param str domain_name: The name of the domain to check. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon failure, also contains a key 'error' with the error message as value. .. versionadded:: Natrium @@ -1153,6 +1215,7 @@ def exists( .. code-block:: bash salt myminion boto_elasticsearch_domain.exists mydomain + ''' ret = {'result': False} try: @@ -1173,7 +1236,12 @@ def wait_for_upgrade( :param str name: The name of the domain to wait for. - :return dict: + :rtype dict: + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + ''' ret = {'result': False} try: @@ -1197,12 +1265,14 @@ def check_upgrade_eligibility( This assumes that the Elasticsearch domain is at rest at the moment this function is called. I.e. The domain is not in the process of : + - being created. - being updated. - another upgrade running, or a check thereof. - being deleted. Behind the scenes, this does 3 things: + - Check if ``elasticsearch_version`` is among the compatible elasticsearch versions. - Perform a check if the Elasticsearch domain is eligible for the upgrade. - Check the result of the check and return the result as a boolean. @@ -1210,9 +1280,13 @@ def check_upgrade_eligibility( :param str name: The Elasticsearch domain name to check. :param str elasticsearch_version: The Elasticsearch version to upgrade to. - :return dict: With key 'result' and as value a boolean denoting success or failure. + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon success, also contains a key 'reponse' with boolean result of the check. Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + ''' ret = {'result': False} # Check if the desired version is in the list of compatible versions diff --git a/salt/states/boto3_elasticsearch.py b/salt/states/boto3_elasticsearch.py index 0a3558ce0d51..3e0915952f45 100644 --- a/salt/states/boto3_elasticsearch.py +++ b/salt/states/boto3_elasticsearch.py @@ -106,68 +106,87 @@ def present( :param str elasticsearch_version: String of format X.Y to specify version for the Elasticsearch domain eg. "1.5" or "2.3". :param dict elasticsearch_cluster_config: Dict specifying the configuration - options for an Elasticsearch domain. Sub-options contained here are: - :param str InstanceType: The instance type for an Elasticsearch cluster. - :param int InstanceCount: The instance type for an Elasticsearch cluster. - :param bool DedicatedMasterEnabled: Indicate whether a dedicated master - node is enabled. - :param bool ZoneAwarenessEnabled: Indicate whether zone awareness is enabled. - :param dict ZoneAwarenessConfig: Specifies the zone awareness configuration - for a domain when zone awareness is enabled. Sub-options contained - here are: - :param int AvailabilityZoneCount: An integer value to indicate the - number of availability zones for a domain when zone awareness is - enabled. This should be equal to number of subnets if VPC endpoints - is enabled. - :param str DedicatedMasterType: The instance type for a dedicated master node. - :param int DedicatedMasterCount: Total number of dedicated master nodes, - active and on standby, for the cluster. + options for an Elasticsearch domain. + Keys (case sensitive) in here are: + + - InstanceType (str): The instance type for an Elasticsearch cluster. + - InstanceCount (int): The instance type for an Elasticsearch cluster. + - DedicatedMasterEnabled (bool): Indicate whether a dedicated master + node is enabled. + - ZoneAwarenessEnabled (bool): Indicate whether zone awareness is enabled. + - ZoneAwarenessConfig (dict): Specifies the zone awareness configuration + for a domain when zone awareness is enabled. + Keys (case sensitive) in here are: + + - AvailabilityZoneCount (int): An integer value to indicate the + number of availability zones for a domain when zone awareness is + enabled. This should be equal to number of subnets if VPC endpoints + is enabled. + - DedicatedMasterType (str): The instance type for a dedicated master node. + - DedicatedMasterCount (int): Total number of dedicated master nodes, + active and on standby, for the cluster. :param dict ebs_options: Dict specifying the options to enable or disable and - specifying the type and size of EBS storage volumes. - Sub-options contained here are: - :param bool EBSEnabled: Specifies whether EBS-based storage is enabled. - :param str VolumeType: Specifies the volume type for EBS-based storage. - :param int VolumeSize: Integer to specify the size of an EBS volume. - :param int Iops: Specifies the IOPD for a Provisioned IOPS EBS volume (SSD). - :param str/dict access_policies: Dict or JSON string with the IAM access policy. + specifying the type and size of EBS storage volumes. + Keys (case sensitive) in here are: + + - EBSEnabled (bool): Specifies whether EBS-based storage is enabled. + - VolumeType (str): Specifies the volume type for EBS-based storage. + - VolumeSize (int): Integer to specify the size of an EBS volume. + - Iops (int): Specifies the IOPD for a Provisioned IOPS EBS volume (SSD). + :type access_policies: str or dict + :param access_policies: Dict or JSON string with the IAM access policy. :param dict snapshot_options: Dict specifying the snapshot options. - Sub-options contained here are: - :param int AutomatedSnapshotStartHour: Specifies the time, in UTC format, - when the service takes a daily automated snapshot of the specified - Elasticsearch domain. Default value is 0 hours. + Keys (case senstive) in here are: + + - AutomatedSnapshotStartHour (int): Specifies the time, in UTC format, + when the service takes a daily automated snapshot of the specified + Elasticsearch domain. Default value is 0 hours. :param dict vpc_options: Dict with the options to specify the subnets and security - groups for the VPC endpoint. Sub-options contained here are: - :param list SubnetIds: The list of subnets for the VPC endpoint. - :param list SecurityGroupIds: The list of security groups for the VPC endpoint. + groups for the VPC endpoint. + Keys (case sensitive) in here are: + + - SubnetIds (list): The list of subnets for the VPC endpoint. + - SecurityGroupIds (list): The list of security groups for the VPC endpoint. :param dict cognito_options: Dict with options to specify the cognito user and - identity pools for Kibana authentication. Sub-options contained here are: - :param bool Enabled: Specifies the option to enable Cognito for Kibana authentication. - :param str UserPoolId: Specifies the Cognito user pool ID for Kibana authentication. - :param str IdentityPoolId: Specifies the Cognito identity pool ID for Kibana authentication. - :param str RoleArn: Specifies the role ARN that provides Elasticsearch permissions - for accessing Cognito resources. + identity pools for Kibana authentication. + Keys (case senstive) in here are: + + - Enabled (bool): Specifies the option to enable Cognito for Kibana authentication. + - UserPoolId (str): Specifies the Cognito user pool ID for Kibana authentication. + - IdentityPoolId (str): Specifies the Cognito identity pool ID for Kibana authentication. + - RoleArn (str): Specifies the role ARN that provides Elasticsearch permissions + for accessing Cognito resources. :param dict encryption_at_rest_options: Dict specifying the encryption at rest - options. This option can only be used for the creation of a new Elasticsearch - domain. Sub-options contained here are: - :param bool Enabled: Specifies the option to enable Encryption At Rest. - :param str KmsKeyId: Specifies the KMS Key ID for Encryption At Rest options. + options. This option can only be used for the creation of a new Elasticsearch + domain. + Keys (case sensitive) in here are: + + - Enabled (bool): Specifies the option to enable Encryption At Rest. + - KmsKeyId (str): Specifies the KMS Key ID for Encryption At Rest options. :param dict node_to_node_encryption_options: Dict specifying the node to node - encryption options. This option can only be used for the creation of - a new Elasticsearch domain. Sub-options contained here are: - :param bool Enabled: Specify True to enable node-to-node encryption. + encryption options. This option can only be used for the creation of + a new Elasticsearch domain. + Keys (case sensitive) in here are: + + - Enabled (bool): Specify True to enable node-to-node encryption. :param dict advanced_options: Dict with option to allow references to indices in an HTTP request body. Must be False when configuring access to individual sub-resources. By default, the value is True. See http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide\ - /es-createupdatedomains.html#es-createdomain-configure-advanced-options + /es-createupdatedomains.html#es-createdomain-configure-advanced-options for more information. :param dict log_publishing_options: Dict with options for various type of logs. - The keys denote the type of log file and can be one of the following: - INDEX_SLOW_LOGS, SEARCH_SLOW_LOGS, ES_APPLICATION_LOGS. - The value assigned to each key is a dict with the following sub-options: - :param str CloudWatchLogsLogGroupArn: The ARN of the Cloudwatch log - group to which the log needs to be published. - :param bool Enabled: Specifies whether given log publishing option is enabled or not. + The keys denote the type of log file and can be one of the following: + + - INDEX_SLOW_LOGS + - SEARCH_SLOW_LOGS + - ES_APPLICATION_LOGS + + The value assigned to each key is a dict with the following case sensitive keys: + + - CloudWatchLogsLogGroupArn (str): The ARN of the Cloudwatch log + group to which the log needs to be published. + - Enabled (bool): Specifies whether given log publishing option is enabled or not. :param bool blocking: Whether or not the state should wait for all operations (create/update/upgrade) to be completed. Default: ``True`` :param dict tags: Dict of tags to ensure are present on the Elasticsearch domain. From 4499a081feb37086bf5707898bff6f99d79bf73d Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Tue, 9 Jul 2019 10:23:27 +0200 Subject: [PATCH 09/12] salt/modules/boto3_elasticsearch.py: Removed trivial CLI examples. Added nontrivial CLI examples. Fixed broken CLI example. --- salt/modules/boto3_elasticsearch.py | 136 +++++++++++++--------------- 1 file changed, 65 insertions(+), 71 deletions(-) diff --git a/salt/modules/boto3_elasticsearch.py b/salt/modules/boto3_elasticsearch.py index 980f42cd8ecd..98e8a070edc6 100644 --- a/salt/modules/boto3_elasticsearch.py +++ b/salt/modules/boto3_elasticsearch.py @@ -111,16 +111,13 @@ def add_tags( :return: Dictionary with key 'result' and as value a boolean denoting success or failure. Upon failure, also contains a key 'error' with the error message as value. - As a special case, tags whose key starts with `__` are ignored. - .. versionadded:: Natrium CLI Example: .. code-block:: bash - salt myminion boto_elasticsearch_domain.add_tags mydomain tags='{"foo": "bar", "baz": "qux"}' - + salt myminion boto3_elasticsearch.add_tags domain_name=mydomain tags='{"foo": "bar", "baz": "qux"}' ''' if not any((arn, domain_name)): raise SaltInvocationError('At least one of domain_name or arn must be specified.') @@ -301,28 +298,27 @@ def create_elasticsearch_domain( .. code-block:: bash - salt myminion boto_elasticsearch_domain.create mydomain \\ - elasticsearch_cluster_config='{ \\ - "InstanceType": "t2.micro.elasticsearch", \\ - "InstanceCount": 1, \\ - "DedicatedMasterEnabled": False, \\ - "ZoneAwarenessEnabled": False}' \\ - ebs_options='{ \\ - "EBSEnabled": True, \\ - "VolumeType": "gp2", \\ - "VolumeSize": 10, \\ - "Iops": 0}' \\ - access_policies='{ \\ - "Version": "2012-10-17", \\ - "Statement": [ \\ - {"Effect": "Allow", \\ - "Principal": {"AWS": "*"}, \\ - "Action": "es:*", \\ - "Resource": "arn:aws:es:us-east-1:111111111111:domain/mydomain/*", \\ - "Condition": {"IpAddress": {"aws:SourceIp": ["127.0.0.1"]}}}]} \\ - snapshot_options='{"AutomatedSnapshotStartHour": 0}' \\ - advanced_options='{"rest.action.multi.allow_explicit_index": "true"}' - + salt myminion boto3_elasticsearch.create_elasticsearch_domain mydomain \\ + elasticsearch_cluster_config='{ \\ + "InstanceType": "t2.micro.elasticsearch", \\ + "InstanceCount": 1, \\ + "DedicatedMasterEnabled": False, \\ + "ZoneAwarenessEnabled": False}' \\ + ebs_options='{ \\ + "EBSEnabled": True, \\ + "VolumeType": "gp2", \\ + "VolumeSize": 10, \\ + "Iops": 0}' \\ + access_policies='{ \\ + "Version": "2012-10-17", \\ + "Statement": [ \\ + {"Effect": "Allow", \\ + "Principal": {"AWS": "*"}, \\ + "Action": "es:*", \\ + "Resource": "arn:aws:es:us-east-1:111111111111:domain/mydomain/*", \\ + "Condition": {"IpAddress": {"aws:SourceIp": ["127.0.0.1"]}}}]}' \\ + snapshot_options='{"AutomatedSnapshotStartHour": 0}' \\ + advanced_options='{"rest.action.multi.allow_explicit_index": "true"}' ''' boto_kwargs = salt.utils.data.filter_falsey({ 'DomainName': domain_name, @@ -373,12 +369,6 @@ def delete_elasticsearch_domain( .. versionadded:: Natrium - CLI Example: - - .. code-block:: bash - - salt myminion boto_elasticsearch_domain.delete mydomain - ''' ret = {'result': False} try: @@ -433,12 +423,6 @@ def describe_elasticsearch_domain( .. versionadded:: Natrium - CLI Example: - - .. code-block:: bash - - salt myminion boto_elasticsearch_domain.status mydomain - ''' ret = {'result': False} try: @@ -468,12 +452,6 @@ def describe_elasticsearch_domain_config( .. versionadded:: Natrium - CLI Example: - - .. code-block:: bash - - salt myminion boto_elasticsearch_domain.describe mydomain - ''' ret = {'result': False} try: @@ -503,6 +481,11 @@ def describe_elasticsearch_domains( .. versionadded:: Natrium + CLI Example: + + .. code-block:: bash + + salt myminion boto3_elasticsearch.describe_elasticsearch_domains '["domain_a", "domain_b"]' ''' ret = {'result': False} try: @@ -542,6 +525,13 @@ def describe_elasticsearch_instance_type_limits( .. versionadded:: Natrium + CLI Example: + + .. code-block:: bash + + salt myminion boto3_elasticsearch.describe_elasticsearch_instance_type_limits \\ + instance_type=r3.8xlarge.elasticsearch \\ + elasticsearch_version='6.2' ''' ret = {'result': False} boto_params = salt.utils.data.filter_falsey({ @@ -615,7 +605,7 @@ def describe_reserved_elasticsearch_instances( reserved instances. Upon failure, also contains a key 'error' with the error message as value. - Note: Version 1.9.174 of boto3 has a bug in that reserved_elasticsearch_instance_id + :note: Version 1.9.174 of boto3 has a bug in that reserved_elasticsearch_instance_id is considered a required argument, even though the documentation says otherwise. .. versionadded:: Natrium @@ -850,12 +840,6 @@ def list_tags( .. versionadded:: Natrium - CLI Example: - - .. code-block:: bash - - salt myminion boto_elasticsearch.list_tags my_domain - ''' if not any((arn, domain_name)): raise SaltInvocationError('At least one of domain_name or arn must be specified.') @@ -943,8 +927,7 @@ def remove_tags( .. code-block:: bash - salt myminion boto_cloudtrail.remove_tags tag_keys='["foo", "bar"]' my_domain - + salt myminion boto3_elasticsearch.remove_tags '["foo", "bar"]' domain_name=my_domain ''' if not any((arn, domain_name)): raise SaltInvocationError('At least one of domain_name or arn must be specified.') @@ -1102,18 +1085,23 @@ def update_elasticsearch_domain_config( .. code-block:: bash - salt myminion boto_elasticsearch_domain.update mydomain \\ - {'InstanceType': 't2.micro.elasticsearch', 'InstanceCount': 1, \\ - 'DedicatedMasterEnabled': false, 'ZoneAwarenessEnabled': false} \\ - {'EBSEnabled': true, 'VolumeType': 'gp2', 'VolumeSize': 10, \\ - 'Iops': 0} \\ - {"Version": "2012-10-17", "Statement": [{"Effect": "Allow", \\ - "Principal": {"AWS": "*"}, "Action": "es:*", \\ - "Resource": "arn:aws:es:us-east-1:111111111111:domain/mydomain/*", \\ - "Condition": {"IpAddress": {"aws:SourceIp": ["127.0.0.1"]}}}]} \\ - {"AutomatedSnapshotStartHour": 0} \\ - {"rest.action.multi.allow_explicit_index": "true"} - + salt myminion boto3_elasticsearch.update_elasticsearch_domain mydomain \\ + elasticsearch_cluster_config='{\\ + "InstanceType": "t2.micro.elasticsearch", \\ + "InstanceCount": 1, \\ + "DedicatedMasterEnabled": false, + "ZoneAwarenessEnabled": false}' \\ + ebs_options='{\\ + "EBSEnabled": true, \\ + "VolumeType": "gp2", \\ + "VolumeSize": 10, \\ + "Iops": 0}' \\ + access_policies='{"Version": "2012-10-17", "Statement": [{\\ + "Effect": "Allow", "Principal": {"AWS": "*"}, "Action": "es:*", \\ + "Resource": "arn:aws:es:us-east-1:111111111111:domain/mydomain/*", \\ + "Condition": {"IpAddress": {"aws:SourceIp": ["127.0.0.1"]}}}]}' \\ + snapshot_options='{"AutomatedSnapshotStartHour": 0}' \\ + advanced_options='{"rest.action.multi.allow_explicit_index": "true"}' ''' ret = {'result': False} boto_kwargs = salt.utils.data.filter_falsey({ @@ -1175,6 +1163,13 @@ def upgrade_elasticsearch_domain( .. versionadded:: Natrium + CLI Example: + + .. code-block:: bash + + salt myminion boto3_elasticsearch.upgrade_elasticsearch_domain mydomain \\ + target_version='6.7' \\ + perform_check_only=True ''' ret = {'result': False} boto_params = salt.utils.data.filter_falsey({ @@ -1210,12 +1205,6 @@ def exists( .. versionadded:: Natrium - CLI Example: - - .. code-block:: bash - - salt myminion boto_elasticsearch_domain.exists mydomain - ''' ret = {'result': False} try: @@ -1287,6 +1276,11 @@ def check_upgrade_eligibility( .. versionadded:: Natrium + CLI Example: + + .. code-block:: bash + + salt myminion boto3_elasticsearch.check_upgrade_eligibility mydomain '6.7' ''' ret = {'result': False} # Check if the desired version is in the list of compatible versions From b04198d191b82155e361001ddc3ee40239440a0b Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Tue, 9 Jul 2019 10:53:10 +0200 Subject: [PATCH 10/12] salt/states/boto3_elasticsearch.py: Deduplicated duplicate code into _check_return_value --- salt/states/boto3_elasticsearch.py | 41 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/salt/states/boto3_elasticsearch.py b/salt/states/boto3_elasticsearch.py index 3e0915952f45..20c1a5e088d4 100644 --- a/salt/states/boto3_elasticsearch.py +++ b/salt/states/boto3_elasticsearch.py @@ -80,6 +80,21 @@ def __virtual__(): return __virtualname__ +def _check_return_value(ret): + ''' + Helper function to check if the 'result' key of the return value has been + properly set. This is to detect unexpected code-paths that would otherwise + return a 'success'-y value but not actually be succesful. + + :param dict ret: The returned value of a state function. + ''' + if ret['result'] == 'oops': + ret['result'] = False + ret['comment'].append('An internal error has occurred: The result value was ' + 'not properly changed.') + return ret + + def present( name, elasticsearch_version=None, @@ -363,11 +378,7 @@ def present( 'changes:new:tags', res['changes']['new'] ) - - if ret['result'] == 'oops': - ret['result'] = False - ret['comment'].append('An internal error has occurred: The result value was ' - 'not properly changed.') + ret = _check_return_value(ret) return ret @@ -425,10 +436,7 @@ def absent( ret['result'] = True ret['comment'].append('Elasticsearch domain "{}" is already absent.' ''.format(name)) - if ret['result'] == 'oops': - ret['result'] = False - ret['comment'].append('An internal error has occurred: The result value was ' - 'not properly changed.') + ret = _check_return_value(ret) return ret @@ -562,10 +570,7 @@ def upgraded( ''.format(name, elasticsearch_version)) ret['changes'] = {'old': current_domain['ElasticsearchVersion'], 'new': elasticsearch_version} - if ret['result'] == 'oops': - ret['result'] = False - ret['comment'].append('An internal error has occurred: The result value was ' - 'not properly changed.') + ret = _check_return_value(ret) return ret @@ -653,10 +658,7 @@ def latest( ret['comment'].append('Elasticsearch domain "{}" is already at its ' 'latest minor version {}.' ''.format(name, current_version)) - if ret['result'] == 'oops': - ret['result'] = False - ret['comment'].append('An internal error has occurred: The result value was ' - 'not properly changed.') + ret = _check_return_value(ret) if ret['result'] and ret['changes'] and not minor_only: # Try and see if we can upgrade again res = latest(name, minor_only=minor_only, region=region, keyid=keyid, key=key, profile=profile) @@ -748,8 +750,5 @@ def tagged( ret['result'] = True ret['comment'].append('Tags on Elasticsearch domain "{}" have been ' '{}ed.'.format(name, 'replac' if replace else 'add')) - if ret['result'] == 'oops': - ret['result'] = False - ret['comment'].append('An internal error has occurred: The result value was ' - 'not properly changed.') + ret = _check_return_value(ret) return ret From e7b61edf71eeee5668ce63b924856dc701c9758f Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Tue, 9 Jul 2019 11:12:34 +0200 Subject: [PATCH 11/12] salt/modules/boto3_elasticsearch.py: Fix incorrect CLI example. --- salt/modules/boto3_elasticsearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/boto3_elasticsearch.py b/salt/modules/boto3_elasticsearch.py index 98e8a070edc6..b50c128e834f 100644 --- a/salt/modules/boto3_elasticsearch.py +++ b/salt/modules/boto3_elasticsearch.py @@ -1085,7 +1085,7 @@ def update_elasticsearch_domain_config( .. code-block:: bash - salt myminion boto3_elasticsearch.update_elasticsearch_domain mydomain \\ + salt myminion boto3_elasticsearch.update_elasticsearch_domain_config mydomain \\ elasticsearch_cluster_config='{\\ "InstanceType": "t2.micro.elasticsearch", \\ "InstanceCount": 1, \\ From fff663ee9d7e7e35aa9c68a4accb0474b797769e Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Tue, 31 Dec 2019 10:28:48 +0100 Subject: [PATCH 12/12] Remove NO_MOCK and NO_MOCK_REASON --- tests/unit/modules/test_boto3_elasticsearch.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/modules/test_boto3_elasticsearch.py b/tests/unit/modules/test_boto3_elasticsearch.py index 563cee5a7e01..30fab8914629 100644 --- a/tests/unit/modules/test_boto3_elasticsearch.py +++ b/tests/unit/modules/test_boto3_elasticsearch.py @@ -13,7 +13,7 @@ # Import Salt Testing libs from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import skipIf, TestCase -from tests.support.mock import (NO_MOCK, NO_MOCK_REASON, MagicMock, patch) +from tests.support.mock import (MagicMock, patch) # Import Salt libs import salt.loader @@ -136,7 +136,6 @@ def __virtual__(): @skipIf(HAS_BOTO3 is False, 'The boto module must be installed.') @skipIf(LooseVersion(boto3.__version__) < LooseVersion(REQUIRED_BOTO3_VERSION), 'The boto3 module must be greater or equal to version {}'.format(REQUIRED_BOTO3_VERSION)) -@skipIf(NO_MOCK, NO_MOCK_REASON) class Boto3ElasticsearchTestCase(TestCase, LoaderModuleMockMixin): ''' TestCase for salt.modules.boto3_elasticsearch module