From daf338d4c7123d69c7c8466d46172f0bdc56c2b5 Mon Sep 17 00:00:00 2001 From: serosset Date: Wed, 12 Jan 2022 02:19:01 +0000 Subject: [PATCH 1/4] draft implementation of opensearch module --- plugins/modules/opensearch.py | 892 ++++++++++++++++++ tests/integration/targets/opensearch/aliases | 5 + .../targets/opensearch/defaults/main.yml | 2 + .../targets/opensearch/meta/main.yml | 3 + .../targets/opensearch/tasks/main.yml | 23 + 5 files changed, 925 insertions(+) create mode 100644 plugins/modules/opensearch.py create mode 100644 tests/integration/targets/opensearch/aliases create mode 100644 tests/integration/targets/opensearch/defaults/main.yml create mode 100644 tests/integration/targets/opensearch/meta/main.yml create mode 100644 tests/integration/targets/opensearch/tasks/main.yml diff --git a/plugins/modules/opensearch.py b/plugins/modules/opensearch.py new file mode 100644 index 00000000000..eab1ca93a68 --- /dev/null +++ b/plugins/modules/opensearch.py @@ -0,0 +1,892 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright: Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +DOCUMENTATION = """ +--- +module: opensearch +short_description: Creates OpenSearch or ElasticSearch domain. +description: + - Creates a new Amazon OpenSearch Service domain. +version_added: "2.1" +options: + domain_name: + description: + - The name of the Amazon OpenSearch/ElasticSearch Service domain. + Domain names are unique across the domains owned by an account within an AWS region. + required: true + type: str + engine_type: + description: + - The engine type to use. "ElasticSearch" | "OpenSearch" + required: false + type: str + choices: ['OpenSearch', 'Elasticsearch'] + default: OpenSearch + version: + description: + - The version of ElasticSearch or OpenSearch to deploy. + required: false + type: str + default: "1.1" + instance_type: + description: + - Type of the instances to use for the domain. + required: false + type: str + default: t2.small.search + instance_count: + description: + - Number of instances for the domain. + required: true + type: int + dedicated_master: + description: + - A boolean value to indicate whether a dedicated master node is enabled. + required: true + type: bool + zone_awareness: + description: + - A boolean value to indicate whether zone awareness is enabled. + required: true + type: bool + availability_zone_count: + description: An integer value to indicate the number of availability zones for a domain when zone awareness is enabled. + This should be equal to number of subnets if VPC endpoints is enabled. + required: false + type: int + ebs: + description: + - Specifies whether EBS-based storage is enabled. + required: true + type: bool + warm_enabled: + description: True to enable UltraWarm storage. + required: false + type: bool + default: false + warm_type: + description: The instance type for the OpenSearch domain's warm nodes. + required: false + type: str + warm_count: + description: The number of UltraWarm nodes in the domain. + required: false + type: int + cold_storage_enabled: + description: True to enable cold storage. Elasticsearch 7.9 or above. + required: false + type: bool + dedicated_master_instance_type: + description: + - The instance type for a dedicated master node. + required: false + type: str + dedicated_master_instance_count: + description: + - Total number of dedicated master nodes, active and on standby, for the domain. + required: false + type: int + volume_type: + description: + - Specifies the volume type for EBS-based storage. "standard"|"gp2"|"io1" + required: true + type: str + volume_size: + description: + - Integer to specify the size of an EBS volume. + required: true + type: int + iops: + description: + - The IOPD for a Provisioned IOPS EBS volume (SSD). + required: false + type: int + vpc_subnets: + description: + - Specifies the subnet ids for VPC endpoint. + required: false + type: list + vpc_security_groups: + description: + - Specifies the security group ids for VPC endpoint. + required: false + type: list + snapshot_hour: + description: + - Integer value from 0 to 23 specifying when the service takes a daily automated snapshot of the specified Elasticsearch domain. + required: false + type: int + default: 0 + access_policies: + description: + - IAM access policy as a JSON-formatted string. + required: true + type: dict + encryption_at_rest_enabled: + description: + - Should data be encrypted while at rest. + required: false + type: bool + encryption_at_rest_kms_key_id: + description: + - If encryption_at_rest_enabled is True, this identifies the encryption key to use + required: false + type: str + cognito_enabled: + description: + - The option to enable Cognito for OpenSearch Dashboards authentication. + required: false + type: bool + default: false + cognito_user_pool_id: + description: + - The Cognito user pool ID for OpenSearch Dashboards authentication. + required: false + type: str + cognito_identity_pool_id: + description: + - The Cognito identity pool ID for OpenSearch Dashboards authentication. + required: false + type: str + cognito_role_arn: + description: + - The role ARN that provides OpenSearch permissions for accessing Cognito resources. + required: false + type: str + domain_endpoint_options: + description: + - Options to specify configuration that will be applied to the domain endpoint. + suboptions: + enforce_https: + description: + - Whether only HTTPS endpoint should be enabled for the domain. + type: bool + tls_security_policy: + description: + - Specify the TLS security policy to apply to the HTTPS endpoint of the domain. + type: str + custom_endpoint_enabled: + description: + - Whether to enable a custom endpoint for the domain. + type: bool + custom_endpoint: + description: + - The fully qualified domain for your custom endpoint. + type: str + custom_endpoint_certificate_arn: + description: + - The ACM certificate ARN for your custom endpoint. + type: str + type: dict + advanced_security_options: + description: + - Specifies advanced security options. + suboptions: + enabled: + description: + - True if advanced security is enabled. + type: bool + internal_user_database_enabled: + description: + - True if the internal user database is enabled. + type: bool + master_user_options: + description: + - Credentials for the master user: username and password, ARN, or both. + suboptions: + master_user_arn: + description: + - ARN for the master user (if IAM is enabled). + type: str + master_user_name: + description: + - The master user's username, which is stored in the Amazon OpenSearch Service domain's internal database. + type: str + master_user_password: + description: + - The master user's password, which is stored in the Amazon OpenSearch Service domain's internal database. + type: str + type: dict + saml_options: + description: + - The SAML application configuration for the domain. + subptions: + enabled: + description: + - True if SAML is enabled. + type: bool + idp: + description: + - The SAML Identity Provider's information. + suboptions: + metadata_content: + description: + - The metadata of the SAML application in XML format. + type: str + entity_id: + description: + - The unique entity ID of the application in SAML identity provider. + type: str + type: dict + master_user_name: + description: + - The SAML master username, which is stored in the Amazon OpenSearch Service domain's internal database. + type: str + master_backend_role: + description: + - The backend role that the SAML master user is mapped to. + type: str + subject_key: + description: + - Element of the SAML assertion to use for username. Default is NameID. + type: str + roles_key: + description: + - Element of the SAML assertion to use for backend roles. Default is roles. + type: str + session_timeout_minutes: + description: + - The duration, in minutes, after which a user session becomes inactive. Acceptable values are between 1 and 1440, and the default value is 60. + type: int + type: dict + type: dict + auto_tune_options: + description: + - Specifies Auto-Tune options. + suboptions: + desired_state: + description: + - The Auto-Tune desired state. Valid values are ENABLED and DISABLED. + type: str + choices: ['ENABLED', 'DISABLED'] + maintenance_schedules: + description: + - A list of maintenance schedules. + suboptions: + start_at: + description: + - The timestamp at which the Auto-Tune maintenance schedule starts. + type: str + duration: + suboptions: + value: + description: + - Integer to specify the value of a maintenance schedule duration. + type: int + unit: + description: + - The unit of a maintenance schedule duration. Valid value is HOURS. + choices: ['HOURS'] + type: str + cron_expression_for_recurrence: + description: + - A cron expression for a recurring maintenance schedule. + type: str + type: list + elements: dict + default: [] + type: dict + wait: + description: + - Whether or not to wait for snapshot creation or deletion. + type: bool + default: 'no' + wait_timeout: + description: + - how long before wait gives up, in seconds. + default: 300 + type: int + tags: + description: + - tags dict to apply to a snapshot. + type: dict + purge_tags: + description: + - whether to remove tags not present in the C(tags) parameter. + default: True + type: bool +author: + - "Jose Armesto (@fiunchinho)" + - "Sebastien Rosset (@sebastien.rosset)" +extends_documentation_fragment: +- amazon.aws.aws +- amazon.aws.ec2 +""" + +EXAMPLES = """ + +- name: Create OpenSearch domain + community.aws.opensearch: + domain_name: "my-domain" + engine_type: OpenSearch + version: "1.1" + instance_type: "m3.medium.search" + instance_count: 2 + dedicated_master: True + zone_awareness: True + availability_zone_count: 2 + dedicated_master_instance_type: "t2.micro.search" + dedicated_master_instance_count: 2 + ebs: True + volume_type: "io1" + volume_size: 10 + iops: 1000 + warm_enabled: true + warm_type: "ultrawarm1.medium.search" + warm_count: 1 + cold_storage_enabled: false + vpc_subnets: + - "subnet-e537d64a" + - "subnet-e537d64b" + vpc_security_groups: + - "sg-dd2f13cb" + - "sg-dd2f13cc" + snapshot_hour: 13 + access_policies: "{{ lookup('file', 'policy.json') | from_json }}" + encryption_at_rest_enabled: false + auto_tune_options: + desired_state: "ENABLED" + maintenance_schedules: + - start_at: "2025-01-12" + duration: + value: 1 + unit: "HOURS" + cron_expression_for_recurrence: "cron(0 12 * * ? *)" + - start_at: "2032-01-12" + duration: + value: 2 + unit: "HOURS" + cron_expression_for_recurrence: "cron(0 12 * * ? *)" + tags: + Environment: Development + Application: foo + wait: true + +""" + +import json +import time + +try: + import botocore +except ImportError: + pass # handled by AnsibleAWSModule + +from ansible.module_utils.six import string_types + +# import module snippets +from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ( + camel_dict_to_snake_dict, + AWSRetry, + compare_aws_tags, +) +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ( + boto3_tag_list_to_ansible_dict, + ansible_dict_to_boto3_tag_list, +) + + +def get_domain(client, module, domain_name): + try: + response = client.describe_domain(DomainName=domain_name) + except client.exceptions.ResourceNotFoundException: + return None + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Couldn't get domain {0}".format(domain_name)) + return response["DomainStatus"] + + +def get_domain_config(client, module, domain_name): + try: + response = client.describe_domain_config(DomainName=domain_name) + except client.exceptions.ResourceNotFoundException: + return None + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Couldn't get domain {0}".format(domain_name)) + return response["DomainConfig"] + + +def domain_to_facts(client, module, domain): + try: + domain["Tags"] = boto3_tag_list_to_ansible_dict( + client.list_tags(ARN=domain["ARN"], aws_retry=True)["TagList"] + ) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws( + e, "Couldn't get tags for domain %s" % domain["domain_name"] + ) + except KeyError: + module.fail_json(msg=str(domain)) + + return camel_dict_to_snake_dict(domain, ignore_list=["Tags"]) + + +def wait_for_domain_status(client, module, domain_name, waiter_name): + if not module.params["wait"]: + return + timeout = module.params["wait_timeout"] + deadline = time.time() + timeout + while time.time() < deadline: + status = get_domain(client, module, domain_name) + if status is None: + if waiter_name == "domain_deleted": + return + else: + if ( + waiter_name == "domain_available" + and status["Created"] + and not status["Processing"] + and not status["UpgradeProcessing"] + ): + return + time.sleep(15) + + +def ensure_domain_absent(client, module): + domain_name = module.params.get("domain_name") + changed = False + + domain = get_domain(client, module, domain_name) + try: + client.delete_domain(DomainName=domain_name) + changed = True + except ( + botocore.exceptions.ClientError, + botocore.exceptions.BotoCoreError, + ) as e: + module.fail_json_aws(e, msg="trying to delete domain") + + # If we're not waiting for a delete to complete then we're all done + # so just return + if not domain or not module.params.get("wait"): + return dict(changed=changed) + try: + wait_for_domain_status(client, module, domain_name, "domain_deleted") + return dict(changed=changed) + except client.exceptions.ResourceNotFoundException: + return dict(changed=changed) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, "awaiting domain deletion") + + +def ensure_tags(client, module, resource_arn, existing_tags, tags, purge_tags): + if tags is None: + return False + tags_to_add, tags_to_remove = compare_aws_tags(existing_tags, tags, purge_tags) + changed = bool(tags_to_add or tags_to_remove) + if tags_to_add: + try: + client.add_tags( + ARN=resource_arn, + TagList=ansible_dict_to_boto3_tag_list(tags_to_add), + ) + except ( + botocore.exceptions.ClientError, + botocore.exceptions.BotoCoreError, + ) as e: + module.fail_json_aws( + e, "Couldn't add tags to domain {0}".format(resource_arn) + ) + if tags_to_remove: + try: + client.remove_tags(ARN=resource_arn, TagKeys=tags_to_remove) + except ( + botocore.exceptions.ClientError, + botocore.exceptions.BotoCoreError, + ) as e: + module.fail_json_aws( + e, "Couldn't remove tags from domain {0}".format(resource_arn) + ) + return changed + + +def upgrade_domain(client, module, current_version, target_engine_version): + domain_name = module.params.get("domain_name") + parameters = { + "DomainName": domain_name, + "TargetVersion": target_engine_version, + "PerformCheckOnly": False, + } + # If background tasks are in progress, wait until they complete. + # This can take several hours depending on the cluster size. + wait_for_domain_status(client, module, domain_name, "domain_available") + + try: + client.upgrade_domain(**parameters) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws( + e, + msg="Couldn't upgrade domain {0} from {1} to {2}".format( + domain_name, current_version, target_engine_version + ), + ) + + if module.params.get("wait"): + wait_for_domain_status(client, module, domain_name, "domain_available") + + +def ensure_domain_present(client, module): + domain_name = module.params.get("domain_name") + target_engine_version = "%s_%s" % ( + module.params.get("engine_type"), + module.params.get("version"), + ) + + domain_config = { + "InstanceType": module.params.get("instance_type"), + "InstanceCount": int(module.params.get("instance_count")), + "DedicatedMasterEnabled": module.params.get("dedicated_master"), + "ZoneAwarenessEnabled": module.params.get("zone_awareness"), + "WarmEnabled": module.params.get("warm_enabled"), + } + if module.params.get("cold_storage_enabled") is not None: + domain_config["ColdStorageOptions"] = { + "Enabled": module.params.get("cold_storage_enabled"), + } + + if module.params.get("zone_awareness"): + domain_config["ZoneAwarenessConfig"] = { + "AvailabilityZoneCount": module.params.get("availability_zone_count"), + } + + if module.params.get("warm_enabled"): + domain_config["WarmType"] = module.params.get("warm_type") + domain_config["WarmCount"] = module.params.get("warm_count") + + ebs_options = {"EBSEnabled": module.params.get("ebs")} + + encryption_at_rest_enabled = module.params.get("encryption_at_rest_enabled") == True + encryption_at_rest_options = {"Enabled": encryption_at_rest_enabled} + + if encryption_at_rest_enabled: + encryption_at_rest_options["KmsKeyId"] = module.params.get( + "encryption_at_rest_kms_key_id" + ) + + vpc_options = {} + vpc_subnets = module.params.get("vpc_subnets") + if vpc_subnets: + if isinstance(vpc_subnets, string_types): + vpc_subnets = [x.strip() for x in vpc_subnets.split(",")] + vpc_options["SubnetIds"] = vpc_subnets + + vpc_security_groups = module.params.get("vpc_security_groups") + if vpc_security_groups: + if isinstance(vpc_security_groups, string_types): + vpc_security_groups = [x.strip() for x in vpc_security_groups.split(",")] + vpc_options["SecurityGroupIds"] = vpc_security_groups + + if domain_config["DedicatedMasterEnabled"]: + domain_config["DedicatedMasterType"] = module.params.get( + "dedicated_master_instance_type" + ) + domain_config["DedicatedMasterCount"] = module.params.get( + "dedicated_master_instance_count" + ) + + if ebs_options["EBSEnabled"]: + ebs_options["VolumeType"] = module.params.get("volume_type") + ebs_options["VolumeSize"] = module.params.get("volume_size") + + if module.params.get("iops") is not None: + ebs_options["Iops"] = module.params.get("iops") + + snapshot_options = { + "AutomatedSnapshotStartHour": module.params.get("snapshot_hour") + } + + cognito_options = {} + if module.params.get("cognito_enabled"): + cognito_options["Enabled"] = True + cognito_options["UserPoolId"] = module.params.get("cognito_user_pool_id") + cognito_options["IdentityPoolId"] = module.params.get( + "cognito_identity_pool_id" + ) + cognito_options["RoleArn"] = module.params.get("cognito_role_arn") + else: + cognito_options["Enabled"] = False + + auto_tune_options = {} + opts = module.params.get("auto_tune_options") + if opts: + if opts.get("desired_state") is not None: + auto_tune_options["DesiredState"] = opts.get("desired_state") + schedules = opts.get("maintenance_schedules") + if schedules is not None: + auto_tune_options["MaintenanceSchedules"] = [] + for s in schedules: + auto_tune_options["MaintenanceSchedules"].append({ + "StartAt": s.get("start_at"), + "Duration": { + "Value": s.get("duration").get("value"), + "Unit": s.get("duration").get("unit"), + }, + "CronExpressionForRecurrence": s.get("cron_expression_for_recurrence") + }) + + changed = False + + try: + pdoc = json.dumps(module.params.get("access_policies")) + except Exception as e: + module.fail_json( + msg="Failed to convert the policy into valid JSON: %s" % str(e) + ) + + domain = get_domain(client, module, domain_name) + if domain is not None: + if target_engine_version != domain["EngineVersion"]: + changed = True + upgrade_domain( + client, module, domain["EngineVersion"], target_engine_version + ) + + # Modify the provided policy to provide reliable changed detection + policy_dict = module.params.get("access_policies") + for statement in policy_dict["Statement"]: + if "Resource" not in statement: + # The opensearch APIs will implicitly set this + statement["Resource"] = "%s/*" % domain["ARN"] + pdoc = json.dumps(policy_dict) + + if domain["ClusterConfig"] != domain_config: + changed = True + + if domain["EBSOptions"] != ebs_options: + changed = True + + if domain["CognitoOptions"] != cognito_options: + changed = True + + if domain["AutoTuneOptions"] != auto_tune_options: + changed = True + + if "VPCOptions" in domain: + if domain["VPCOptions"]["SubnetIds"] != vpc_options["SubnetIds"]: + changed = True + if ( + domain["VPCOptions"]["SecurityGroupIds"] + != vpc_options["SecurityGroupIds"] + ): + changed = True + + if domain["SnapshotOptions"] != snapshot_options: + changed = True + + current_policy_dict = json.loads(domain["AccessPolicies"]) + if current_policy_dict != policy_dict: + changed = True + + if changed: + parameters = { + "DomainName": module.params.get("domain_name"), + "EncryptionAtRestOptions": encryption_at_rest_options, + "ClusterConfig": domain_config, + "EBSOptions": ebs_options, + "SnapshotOptions": snapshot_options, + "AccessPolicies": pdoc, + "CognitoOptions": cognito_options, + "AutoTuneOptions": auto_tune_options, + } + + if vpc_options["SubnetIds"] or vpc_options["SecurityGroupIds"]: + parameters["VPCOptions"] = vpc_options + + try: + client.update_domain_config(**parameters) + except ( + botocore.exceptions.BotoCoreError, + botocore.exceptions.ClientError, + ) as e: + module.fail_json_aws( + e, msg="Couldn't update domain {0}".format(domain_name) + ) + + else: + changed = True + parameters = { + "DomainName": module.params.get("domain_name"), + "EngineVersion": target_engine_version, + "EncryptionAtRestOptions": encryption_at_rest_options, + "ClusterConfig": domain_config, + "EBSOptions": ebs_options, + "SnapshotOptions": snapshot_options, + "AccessPolicies": pdoc, + "CognitoOptions": cognito_options, + "AutoTuneOptions": auto_tune_options, + } + + if vpc_options["SubnetIds"] or vpc_options["SecurityGroupIds"]: + parameters["VPCOptions"] = vpc_options + + try: + client.create_domain(**parameters) + except ( + botocore.exceptions.BotoCoreError, + botocore.exceptions.ClientError, + ) as e: + module.fail_json_aws( + e, msg="Couldn't update domain {0}".format(domain_name) + ) + + if module.params.get("wait"): + wait_for_domain_status(client, module, domain_name, "domain_available") + + try: + existing_tags = boto3_tag_list_to_ansible_dict( + client.list_tags(ARN=domain["ARN"], aws_retry=True)["TagList"] + ) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, "Couldn't get tags for domain %s" % domain_name) + + desired_tags = module.params["tags"] + purge_tags = module.params["purge_tags"] + changed |= ensure_tags( + client, module, domain["ARN"], existing_tags, desired_tags, purge_tags + ) + + domain = get_domain(client, module, domain_name) + + return dict(changed=changed, **domain_to_facts(client, module, domain)) + + +def main(): + + domain_endpoint_options = dict( + enforce_https=dict(type="bool"), + tls_security_policy=dict(), + custom_endpoint_enabled=dict(type="bool"), + custom_endpoint=dict(), + custom_endpoint_certificate_arn=dict(), + ) + + master_user_options = dict( + master_user_arn=dict(), + master_user_name=dict(), + master_user_password=dict(), + ) + advanced_security_options = dict( + enabled=dict(type="bool"), + internal_user_database_enabled=dict(type="bool"), + master_user_options=dict(type="dict", options=master_user_options), + ) + + idp = dict( + metadata_content=dict(), + entity_id=dict(), + ) + saml_options = dict( + enabled=dict(type="bool"), + idp=dict(type="dict", options=idp), + master_user_name=dict(), + master_backend_role=dict(), + subject_key=dict(), + roles_key=dict(), + session_timeout_minutes=dict(type="int"), + ) + + auto_tune_duration = dict( + value=dict(type="int"), + unit=dict(choices=["HOURS"]), + ) + auto_tune_maintenance_schedules = dict( + start_at=dict(), + duration=dict(type="dict", options=auto_tune_duration), + cron_expression_for_recurrence=dict(), + ) + auto_tune_options = dict( + desired_state=dict(choices=["ENABLED", "DISABLED"]), + maintenance_schedules=dict( + type="list", + default=[], + elements="dict", + options=auto_tune_maintenance_schedules, + ), + ) + + module = AnsibleAWSModule( + argument_spec=dict( + state=dict(choices=["present", "absent"], default="present"), + domain_name=dict(required=True), + engine_type=dict( + default="OpenSearch", choices=["OpenSearch", "Elasticsearch"] + ), + version=dict(default="1.1"), + wait=dict(type="bool", default=False), + wait_timeout=dict(type="int", default=300), + instance_type=dict(default="t2.small.search"), + instance_count=dict(required=True, type="int"), + dedicated_master=dict(required=True, type="bool"), + zone_awareness=dict(required=True, type="bool"), + availability_zone_count=dict(required=False, type="int"), + dedicated_master_instance_type=dict(), + dedicated_master_instance_count=dict(type="int"), + ebs=dict(required=True, type="bool"), + volume_type=dict(required=True), + volume_size=dict(required=True, type="int"), + iops=dict(required=False, type="int"), + warm_enabled=dict(required=False, type="bool", default=False), + warm_type=dict(required=False), + warm_count=dict(required=False, type="int"), + cold_storage_enabled=dict(required=False, type="bool"), + access_policies=dict(required=True, type="dict"), + vpc_subnets=dict(type="list", elements="str", required=False), + vpc_security_groups=dict(type="list", elements="str", required=False), + snapshot_hour=dict(required=False, type="int", default=0), + encryption_at_rest_enabled=dict(type="bool", default=False), + encryption_at_rest_kms_key_id=dict(required=False), + cognito_enabled=dict(required=False, type="bool", default=False), + cognito_user_pool_id=dict(required=False), + cognito_identity_pool_id=dict(required=False), + cognito_role_arn=dict(required=False), + domain_endpoint_options=dict(type="dict", options=domain_endpoint_options), + advanced_security_options=dict( + type="dict", options=advanced_security_options + ), + saml_options=dict(type="dict", options=saml_options), + auto_tune_options=dict(type="dict", options=auto_tune_options), + tags=dict(type="dict"), + purge_tags=dict(type="bool", default=True), + ), + required_if=[ + ["warm_enabled", True, ["warm_type", "warm_count"]], + ["zone_awareness", True, ["availability_zone_count"]], + [ + "dedicated_master", + True, + ["dedicated_master_instance_type", "dedicated_master_instance_count"], + ], + ["ebs", True, ["volume_type", "volume_size"]], + [ + "cognito_enabled", + True, + [ + "cognito_user_pool_id", + "cognito_identity_pool_id", + "cognito_role_arn", + ], + ], + ["encryption_at_rest_enabled", True, ["encryption_at_rest_kms_key_id"]], + ], + ) + + try: + client = module.client( + "opensearch", retry_decorator=AWSRetry.jittered_backoff() + ) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Failed to connect to AWS") + + if module.params["state"] == "absent": + ret_dict = ensure_domain_absent(client, module) + else: + ret_dict = ensure_domain_present(client, module) + + module.exit_json(**ret_dict) + + +if __name__ == "__main__": + main() diff --git a/tests/integration/targets/opensearch/aliases b/tests/integration/targets/opensearch/aliases new file mode 100644 index 00000000000..22267bad72e --- /dev/null +++ b/tests/integration/targets/opensearch/aliases @@ -0,0 +1,5 @@ +# reason: slow +# Tests take around 30 minutes +#disabled + +cloud/aws diff --git a/tests/integration/targets/opensearch/defaults/main.yml b/tests/integration/targets/opensearch/defaults/main.yml new file mode 100644 index 00000000000..da6aef4bb20 --- /dev/null +++ b/tests/integration/targets/opensearch/defaults/main.yml @@ -0,0 +1,2 @@ +--- +# defaults file for opensearch tests diff --git a/tests/integration/targets/opensearch/meta/main.yml b/tests/integration/targets/opensearch/meta/main.yml new file mode 100644 index 00000000000..1f64f1169a9 --- /dev/null +++ b/tests/integration/targets/opensearch/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - prepare_tests + - setup_ec2 diff --git a/tests/integration/targets/opensearch/tasks/main.yml b/tests/integration/targets/opensearch/tasks/main.yml new file mode 100644 index 00000000000..be582776377 --- /dev/null +++ b/tests/integration/targets/opensearch/tasks/main.yml @@ -0,0 +1,23 @@ +--- +# tasks file for test_opensearch + + # ============================================================ + +- name: Test incomplete arguments with opensearch + + block: + + # ============================================================ + + - name: test without specifying required module options + ec2_asg: + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" + security_token: "{{ security_token | default(omit) }}" + ignore_errors: yes + register: result + + - name: assert name is a required module option + assert: + that: + - "result.msg == 'missing required arguments: name'" \ No newline at end of file From ea2963f96000041512a11873bc8db08290c56117 Mon Sep 17 00:00:00 2001 From: serosset Date: Wed, 12 Jan 2022 02:29:38 +0000 Subject: [PATCH 2/4] draft implementation of opensearch module get domain response on creation do not set cold storage option if version is less than 7.9 add skeleton integration test fix doc issues Add domain endpoint config options Add policy for test purpose Add integration tests, fix code such that tests can pass Add integration tests, fix code such that tests can pass Add integration tests, fix code such that tests can pass Add integration tests, fix code such that tests can pass Fix doc issues, address PR comments uncomment module defaults. add tasks in check mode Add more integration tests. Simplify argument spec. Fix linter issues. Add support for check_mode. Remove dead code. Add code comments. Uncomment module defaults add pylint comments Add more integration tests with check mode format code using black to fix pep8 issues fix integration tests fix linter issues add support for advanced security options. Fix linter issue add more examples wait until VPC is created add longer timeout for cluster upgrade test. Handle wait timeout error Customize wait timeout of integration tests, based on actual execution runtime fix pylint issues, autotune configuration add more info to wait timeout status, add unit tests upgrade boto3 for test purpose. Could be reverted later if needed add encryption at rest fix KMV key id parameter specify version of botocore compatible with boto3 Create/Delete KMS key for encryption at rest, change ES instance type to support encryption at rest add doc comment provide test execution timeline improve integration tests Add wait for deletion of Cluster provide test execution timeline split common functions to module_utils Specify minimum version of botocore Co-authored-by: Mark Chappell Revert changes for boto3 version Co-authored-by: Mark Chappell Specify virtual boto environment Co-authored-by: Mark Chappell Specify minimum version of botocore Co-authored-by: Mark Chappell Revert change for default boto3 version Co-authored-by: Mark Chappell Revert change for boto3 version Co-authored-by: Mark Chappell Specify minimum version of botocore Co-authored-by: Mark Chappell remove import which is no longer needed improve name of test tasks, add timing information fix assertion for auto-tune refactor to simplify code make auto-tune test more robust do not raise exception if trying to delete a ES cluster that does not exist handle deletion of resources in integration tests Handle case when ES cluster is attached to the Internet Handle case when ES cluster is attached to the Internet Handle case when ES cluster is attached to the Internet add code comments Handle case when ES cluster is attached to the Internet renamed ebs_options.enabled to ebs_enabled to make it consistent with the API improve code comments Add more test assertions reformat code reformat code fix syntax error in main.yml use key_arn instead of key_id format code custom_endpoint_enabled, fix assertion fix imports adding opensearch to runtime.yml, is it used for module defaults? split the integration test tasks in multiple files remove extra line Add route53 zone for custom opensearch endpoint Add timeout for HTTPs endpoint. Create Route53 CNAME record Delete Route53 zone Add blocks Increase timeout. Set custom endpoint fix doc issues move auto-tune at the end of the integration tests add unit tests. add function to compare versions. add support for upgrade through intermediate versions improve documentation don't wait for auto-tune background tasks to complete fix zone discovery issue when running outside us-east-1. fix name of es cluster, it must start with a letter fix test assertions fix code formatting fix linter issue move ensure_tags to module_utils use 'compare_policies' fix linter issue fix doc issues create single block for integration test add code comments, rename get_domain to get_domain_status add opensearch_info module improve integration tests fix deletion of VPC resources fix test assertion, add documentation fix indentation removed unused variables. Add assertions fix variable assignment add test fix camel_dict for 'Endpoints' attribute handle case when domain_status is None move test fix format issues fix doc issues add documentation move task to obtain endpoints property route53 is explicitly a global service mistakenly commented out 'always' block add 'allow_intermediate_upgrades' parameter cleanup route53 records fix validation issue with upgrade compatible versions fix doc issues fix validation issue with upgrade compatible versions add missing check mode in integration test add documentation for SAML improve integration tests update KMS policy remove trailing space reduce time to execute integration tests --- meta/runtime.yml | 2 + plugins/module_utils/opensearch.py | 280 +++ plugins/modules/opensearch.py | 1713 +++++++++++------ plugins/modules/opensearch_info.py | 530 +++++ plugins/modules/wafv2_web_acl_info.py | 4 +- .../integration/targets/lambda/tasks/main.yml | 2 +- .../targets/lambda_alias/tasks/main.yml | 2 +- .../targets/lambda_policy/tasks/main.yml | 4 +- tests/integration/targets/opensearch/aliases | 21 +- .../opensearch/files/opensearch_policy.json | 16 + .../targets/opensearch/meta/main.yml | 5 +- .../targets/opensearch/tasks/main.yml | 43 +- .../opensearch/tasks/test_create_cert.yml | 53 + .../tasks/test_delete_resources.yml | 61 + .../tasks/test_delete_vpc_resources.yml | 92 + .../opensearch/tasks/test_opensearch.yml | 1252 ++++++++++++ .../opensearch/tasks/test_vpc_setup.yml | 134 ++ .../opensearch/templates/kms_policy.j2 | 68 + tests/unit/plugins/modules/test_opensearch.py | 86 + 19 files changed, 3792 insertions(+), 576 deletions(-) create mode 100644 plugins/module_utils/opensearch.py create mode 100644 plugins/modules/opensearch_info.py create mode 100644 tests/integration/targets/opensearch/files/opensearch_policy.json create mode 100644 tests/integration/targets/opensearch/tasks/test_create_cert.yml create mode 100644 tests/integration/targets/opensearch/tasks/test_delete_resources.yml create mode 100644 tests/integration/targets/opensearch/tasks/test_delete_vpc_resources.yml create mode 100644 tests/integration/targets/opensearch/tasks/test_opensearch.yml create mode 100644 tests/integration/targets/opensearch/tasks/test_vpc_setup.yml create mode 100644 tests/integration/targets/opensearch/templates/kms_policy.j2 create mode 100644 tests/unit/plugins/modules/test_opensearch.py diff --git a/meta/runtime.yml b/meta/runtime.yml index 2d1710c06d6..df6c205015d 100644 --- a/meta/runtime.yml +++ b/meta/runtime.yml @@ -149,6 +149,8 @@ action_groups: - lightsail - networkfirewall_rule_group - networkfirewall_rule_group_info + - opensearch + - opensearch_info - rds_instance - rds_instance_info - rds_instance_snapshot diff --git a/plugins/module_utils/opensearch.py b/plugins/module_utils/opensearch.py new file mode 100644 index 00000000000..8189378e5c3 --- /dev/null +++ b/plugins/module_utils/opensearch.py @@ -0,0 +1,280 @@ +# This file is part of Ansible +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +from copy import deepcopy +import datetime +import functools +import time + +try: + import botocore +except ImportError: + pass # caught by AnsibleAWSModule + +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ( + ansible_dict_to_boto3_tag_list, + camel_dict_to_snake_dict, + compare_aws_tags, +) +from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code +from ansible_collections.amazon.aws.plugins.module_utils.tagging import ( + boto3_tag_list_to_ansible_dict, +) +from ansible.module_utils.six import string_types + + +def get_domain_status(client, module, domain_name): + """ + Get the status of an existing OpenSearch cluster. + """ + try: + response = client.describe_domain(DomainName=domain_name) + except is_boto3_error_code("ResourceNotFoundException"): + return None + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except + module.fail_json_aws(e, msg="Couldn't get domain {0}".format(domain_name)) + return response["DomainStatus"] + + +def get_domain_config(client, module, domain_name): + """ + Get the configuration of an existing OpenSearch cluster, convert the data + such that it can be used as input parameter to client.update_domain(). + The status info is removed. + The returned config includes the 'EngineVersion' property, it needs to be removed + from the dict before invoking client.update_domain(). + + Return (domain_config, domain_arn) or (None, None) if the domain does not exist. + """ + try: + response = client.describe_domain_config(DomainName=domain_name) + except is_boto3_error_code("ResourceNotFoundException"): + return (None, None) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except + module.fail_json_aws(e, msg="Couldn't get domain {0}".format(domain_name)) + domain_config = {} + arn = None + if response is not None: + for k in response["DomainConfig"]: + domain_config[k] = response["DomainConfig"][k]["Options"] + domain_config["DomainName"] = domain_name + # If ES cluster is attached to the Internet, the "VPCOptions" property is not present. + if "VPCOptions" in domain_config: + # The "VPCOptions" returned by the describe_domain_config API has + # additional attributes that would cause an error if sent in the HTTP POST body. + dc = {} + if "SubnetIds" in domain_config["VPCOptions"]: + dc["SubnetIds"] = deepcopy(domain_config["VPCOptions"]["SubnetIds"]) + if "SecurityGroupIds" in domain_config["VPCOptions"]: + dc["SecurityGroupIds"] = deepcopy(domain_config["VPCOptions"]["SecurityGroupIds"]) + domain_config["VPCOptions"] = dc + # The "StartAt" property is converted to datetime, but when doing comparisons it should + # be in the string format "YYYY-MM-DD". + for s in domain_config["AutoTuneOptions"]["MaintenanceSchedules"]: + if isinstance(s["StartAt"], datetime.datetime): + s["StartAt"] = s["StartAt"].strftime("%Y-%m-%d") + # Provisioning of "AdvancedOptions" is not supported by this module yet. + domain_config.pop("AdvancedOptions", None) + + # Get the ARN of the OpenSearch cluster. + domain = get_domain_status(client, module, domain_name) + if domain is not None: + arn = domain["ARN"] + return (domain_config, arn) + + +def normalize_opensearch(client, module, domain): + """ + Merge the input domain object with tags associated with the domain, + convert the attributes from camel case to snake case, and return the object. + """ + try: + domain["Tags"] = boto3_tag_list_to_ansible_dict( + client.list_tags(ARN=domain["ARN"], aws_retry=True)["TagList"] + ) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws( + e, "Couldn't get tags for domain %s" % domain["domain_name"] + ) + except KeyError: + module.fail_json(msg=str(domain)) + + return camel_dict_to_snake_dict(domain, ignore_list=["Tags"]) + + +def wait_for_domain_status(client, module, domain_name, waiter_name): + if not module.params["wait"]: + return + timeout = module.params["wait_timeout"] + deadline = time.time() + timeout + status_msg = "" + while time.time() < deadline: + status = get_domain_status(client, module, domain_name) + if status is None: + status_msg = "Not Found" + if waiter_name == "domain_deleted": + return + else: + status_msg = "Created: {0}. Processing: {1}. UpgradeProcessing: {2}".format( + status["Created"], + status["Processing"], + status["UpgradeProcessing"], + ) + if ( + waiter_name == "domain_available" + and status["Created"] + and not status["Processing"] + and not status["UpgradeProcessing"] + ): + return + time.sleep(15) + # Timeout occured. + module.fail_json( + msg=f"Timeout waiting for wait state '{waiter_name}'. {status_msg}" + ) + + +def parse_version(engine_version): + ''' + Parse the engine version, which should be Elasticsearch_X.Y or OpenSearch_X.Y + Return dict { 'engine_type': engine_type, 'major': major, 'minor': minor } + ''' + version = engine_version.split("_") + if len(version) != 2: + return None + semver = version[1].split(".") + if len(semver) != 2: + return None + engine_type = version[0] + if engine_type not in ['Elasticsearch', 'OpenSearch']: + return None + if not (semver[0].isdigit() and semver[1].isdigit()): + return None + major = int(semver[0]) + minor = int(semver[1]) + return {'engine_type': engine_type, 'major': major, 'minor': minor} + + +def compare_domain_versions(version1, version2): + supported_engines = { + 'Elasticsearch': 1, + 'OpenSearch': 2, + } + if isinstance(version1, string_types): + version1 = parse_version(version1) + if isinstance(version2, string_types): + version2 = parse_version(version2) + if version1 is None and version2 is not None: + return -1 + elif version1 is not None and version2 is None: + return 1 + elif version1 is None and version2 is None: + return 0 + e1 = supported_engines.get(version1.get('engine_type')) + e2 = supported_engines.get(version2.get('engine_type')) + if e1 < e2: + return -1 + elif e1 > e2: + return 1 + else: + if version1.get('major') < version2.get('major'): + return -1 + elif version1.get('major') > version2.get('major'): + return 1 + else: + if version1.get('minor') < version2.get('minor'): + return -1 + elif version1.get('minor') > version2.get('minor'): + return 1 + else: + return 0 + + +def get_target_increment_version(client, module, domain_name, target_version): + """ + Returns the highest compatible version which is less than or equal to target_version. + When upgrading a domain from version V1 to V2, it may not be possible to upgrade + directly from V1 to V2. The domain may have to be upgraded through intermediate versions. + Return None if there is no such version. + For example, it's not possible to upgrade directly from Elasticsearch 5.5 to 7.10. + """ + api_compatible_versions = None + try: + api_compatible_versions = client.get_compatible_versions(DomainName=domain_name) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws( + e, + msg="Couldn't get compatible versions for domain {0}".format( + domain_name), + ) + compat = api_compatible_versions.get('CompatibleVersions') + if compat is None: + module.fail_json( + "Unable to determine list of compatible versions", + compatible_versions=api_compatible_versions) + if len(compat) == 0: + module.fail_json( + "Unable to determine list of compatible versions", + compatible_versions=api_compatible_versions) + if compat[0].get("TargetVersions") is None: + module.fail_json( + "No compatible versions found", + compatible_versions=api_compatible_versions) + compatible_versions = [] + for v in compat[0].get("TargetVersions"): + if target_version == v: + # It's possible to upgrade directly to the target version. + return target_version + semver = parse_version(v) + if semver is not None: + compatible_versions.append(semver) + # No direct upgrade is possible. Upgrade to the highest version available. + compatible_versions = sorted(compatible_versions, key=functools.cmp_to_key(compare_domain_versions)) + # Return the highest compatible version which is lower than target_version + for v in reversed(compatible_versions): + if compare_domain_versions(v, target_version) <= 0: + return v + return None + + +def ensure_tags(client, module, resource_arn, existing_tags, tags, purge_tags): + if tags is None: + return False + tags_to_add, tags_to_remove = compare_aws_tags(existing_tags, tags, purge_tags) + changed = bool(tags_to_add or tags_to_remove) + if tags_to_add: + if module.check_mode: + module.exit_json( + changed=True, msg="Would have added tags to domain if not in check mode" + ) + try: + client.add_tags( + ARN=resource_arn, + TagList=ansible_dict_to_boto3_tag_list(tags_to_add), + ) + except ( + botocore.exceptions.ClientError, + botocore.exceptions.BotoCoreError, + ) as e: + module.fail_json_aws( + e, "Couldn't add tags to domain {0}".format(resource_arn) + ) + if tags_to_remove: + if module.check_mode: + module.exit_json( + changed=True, msg="Would have removed tags if not in check mode" + ) + try: + client.remove_tags(ARN=resource_arn, TagKeys=tags_to_remove) + except ( + botocore.exceptions.ClientError, + botocore.exceptions.BotoCoreError, + ) as e: + module.fail_json_aws( + e, "Couldn't remove tags from domain {0}".format(resource_arn) + ) + return changed diff --git a/plugins/modules/opensearch.py b/plugins/modules/opensearch.py index eab1ca93a68..422feb7d31a 100644 --- a/plugins/modules/opensearch.py +++ b/plugins/modules/opensearch.py @@ -3,161 +3,242 @@ # Copyright: Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + DOCUMENTATION = """ --- module: opensearch short_description: Creates OpenSearch or ElasticSearch domain. description: - - Creates a new Amazon OpenSearch Service domain. -version_added: "2.1" + - Creates or modify a Amazon OpenSearch Service domain. +version_added: 3.1.0 +author: "Sebastien Rosset (@sebastien-rosset)" options: - domain_name: - description: - - The name of the Amazon OpenSearch/ElasticSearch Service domain. - Domain names are unique across the domains owned by an account within an AWS region. - required: true - type: str - engine_type: - description: - - The engine type to use. "ElasticSearch" | "OpenSearch" - required: false - type: str - choices: ['OpenSearch', 'Elasticsearch'] - default: OpenSearch - version: - description: - - The version of ElasticSearch or OpenSearch to deploy. - required: false - type: str - default: "1.1" - instance_type: + state: description: - - Type of the instances to use for the domain. + - Creates or modifies an existing OpenSearch domain. + - Deletes an OpenSearch domain. required: false type: str - default: t2.small.search - instance_count: - description: - - Number of instances for the domain. - required: true - type: int - dedicated_master: - description: - - A boolean value to indicate whether a dedicated master node is enabled. - required: true - type: bool - zone_awareness: - description: - - A boolean value to indicate whether zone awareness is enabled. - required: true - type: bool - availability_zone_count: - description: An integer value to indicate the number of availability zones for a domain when zone awareness is enabled. - This should be equal to number of subnets if VPC endpoints is enabled. - required: false - type: int - ebs: + choices: ['present', 'absent'] + default: present + domain_name: description: - - Specifies whether EBS-based storage is enabled. + - The name of the Amazon OpenSearch/ElasticSearch Service domain. + - Domain names are unique across the domains owned by an account within an AWS region. required: true - type: bool - warm_enabled: - description: True to enable UltraWarm storage. - required: false - type: bool - default: false - warm_type: - description: The instance type for the OpenSearch domain's warm nodes. - required: false type: str - warm_count: - description: The number of UltraWarm nodes in the domain. - required: false - type: int - cold_storage_enabled: - description: True to enable cold storage. Elasticsearch 7.9 or above. - required: false - type: bool - dedicated_master_instance_type: + engine_version: description: - - The instance type for a dedicated master node. + -> + The engine version to use. For example, 'ElasticSearch_7.10' or 'OpenSearch_1.1'. + -> + If the currently running version is not equal to I(engine_version), + a cluster upgrade is triggered. + -> + It may not be possible to upgrade directly from the currently running version + to I(engine_version). In that case, the upgrade is performed incrementally by + upgrading to the highest compatible version, then repeat the operation until + the cluster is running at the target version. + -> + The upgrade operation fails if there is no path from current version to I(engine_version). + -> + See OpenSearch documentation for upgrade compatibility. required: false type: str - dedicated_master_instance_count: + allow_intermediate_upgrades: description: - - Total number of dedicated master nodes, active and on standby, for the domain. + - > + If true, allow OpenSearch domain to be upgraded through one or more intermediate versions. + - > + If false, do not allow OpenSearch domain to be upgraded through intermediate versions. + The upgrade operation fails if it's not possible to ugrade to I(engine_version) directly. required: false - type: int - volume_type: - description: - - Specifies the volume type for EBS-based storage. "standard"|"gp2"|"io1" - required: true - type: str - volume_size: - description: - - Integer to specify the size of an EBS volume. - required: true - type: int - iops: + type: bool + default: true + cluster_config: description: - - The IOPD for a Provisioned IOPS EBS volume (SSD). - required: false - type: int - vpc_subnets: + - Parameters for the cluster configuration of an OpenSearch Service domain. + type: dict + suboptions: + instance_type: + description: + - Type of the instances to use for the domain. + required: false + type: str + instance_count: + description: + - Number of instances for the domain. + required: false + type: int + zone_awareness: + description: + - A boolean value to indicate whether zone awareness is enabled. + required: false + type: bool + availability_zone_count: + description: + - > + An integer value to indicate the number of availability zones for a domain when zone awareness is enabled. + This should be equal to number of subnets if VPC endpoints is enabled. + required: false + type: int + dedicated_master: + description: + - A boolean value to indicate whether a dedicated master node is enabled. + required: false + type: bool + dedicated_master_instance_type: + description: + - The instance type for a dedicated master node. + required: false + type: str + dedicated_master_instance_count: + description: + - Total number of dedicated master nodes, active and on standby, for the domain. + required: false + type: int + warm_enabled: + description: + - True to enable UltraWarm storage. + required: false + type: bool + warm_type: + description: + - The instance type for the OpenSearch domain's warm nodes. + required: false + type: str + warm_count: + description: + - The number of UltraWarm nodes in the domain. + required: false + type: int + cold_storage_options: + description: + - Specifies the ColdStorageOptions config for a Domain. + type: dict + suboptions: + enabled: + description: + - True to enable cold storage. Supported on Elasticsearch 7.9 or above. + required: false + type: bool + ebs_options: description: - - Specifies the subnet ids for VPC endpoint. - required: false - type: list - vpc_security_groups: + - Parameters to configure EBS-based storage for an OpenSearch Service domain. + type: dict + suboptions: + ebs_enabled: + description: + - Specifies whether EBS-based storage is enabled. + required: false + type: bool + volume_type: + description: + - Specifies the volume type for EBS-based storage. "standard"|"gp2"|"io1" + required: false + type: str + volume_size: + description: + - Integer to specify the size of an EBS volume. + required: false + type: int + iops: + description: + - The IOPD for a Provisioned IOPS EBS volume (SSD). + required: false + type: int + vpc_options: description: - - Specifies the security group ids for VPC endpoint. - required: false - type: list - snapshot_hour: + - Options to specify the subnets and security groups for a VPC endpoint. + type: dict + suboptions: + subnets: + description: + - Specifies the subnet ids for VPC endpoint. + required: false + type: list + elements: str + security_groups: + description: + - Specifies the security group ids for VPC endpoint. + required: false + type: list + elements: str + snapshot_options: description: - - Integer value from 0 to 23 specifying when the service takes a daily automated snapshot of the specified Elasticsearch domain. - required: false - type: int - default: 0 + - Option to set time, in UTC format, of the daily automated snapshot. + type: dict + suboptions: + automated_snapshot_start_hour: + description: + - > + Integer value from 0 to 23 specifying when the service takes a daily automated snapshot + of the specified Elasticsearch domain. + required: false + type: int access_policies: description: - IAM access policy as a JSON-formatted string. - required: true - type: dict - encryption_at_rest_enabled: - description: - - Should data be encrypted while at rest. - required: false - type: bool - encryption_at_rest_kms_key_id: - description: - - If encryption_at_rest_enabled is True, this identifies the encryption key to use - required: false - type: str - cognito_enabled: - description: - - The option to enable Cognito for OpenSearch Dashboards authentication. required: false - type: bool - default: false - cognito_user_pool_id: + type: dict + encryption_at_rest_options: description: - - The Cognito user pool ID for OpenSearch Dashboards authentication. - required: false - type: str - cognito_identity_pool_id: + - Parameters to enable encryption at rest. + type: dict + suboptions: + enabled: + description: + - Should data be encrypted while at rest. + required: false + type: bool + kms_key_id: + description: + - If encryption at rest enabled, this identifies the encryption key to use. + - The value should be a KMS key ARN. It can also be the KMS key id. + required: false + type: str + node_to_node_encryption_options: description: - - The Cognito identity pool ID for OpenSearch Dashboards authentication. - required: false - type: str - cognito_role_arn: + - Node-to-node encryption options. + type: dict + suboptions: + enabled: + description: + - True to enable node-to-node encryption. + required: false + type: bool + cognito_options: description: - - The role ARN that provides OpenSearch permissions for accessing Cognito resources. - required: false - type: str + - Parameters to configure OpenSearch Service to use Amazon Cognito authentication for OpenSearch Dashboards. + type: dict + suboptions: + enabled: + description: + - The option to enable Cognito for OpenSearch Dashboards authentication. + required: false + type: bool + user_pool_id: + description: + - The Cognito user pool ID for OpenSearch Dashboards authentication. + required: false + type: str + identity_pool_id: + description: + - The Cognito identity pool ID for OpenSearch Dashboards authentication. + required: false + type: str + role_arn: + description: + - The role ARN that provides OpenSearch permissions for accessing Cognito resources. + required: false + type: str domain_endpoint_options: description: - Options to specify configuration that will be applied to the domain endpoint. + type: dict suboptions: enforce_https: description: @@ -179,14 +260,15 @@ description: - The ACM certificate ARN for your custom endpoint. type: str - type: dict advanced_security_options: description: - Specifies advanced security options. + type: dict suboptions: enabled: description: - True if advanced security is enabled. + - You must enable node-to-node encryption to use advanced security options. type: bool internal_user_database_enabled: description: @@ -194,7 +276,8 @@ type: bool master_user_options: description: - - Credentials for the master user: username and password, ARN, or both. + - Credentials for the master user, username and password, ARN, or both. + type: dict suboptions: master_user_arn: description: @@ -202,24 +285,31 @@ type: str master_user_name: description: - - The master user's username, which is stored in the Amazon OpenSearch Service domain's internal database. + - The username of the master user, which is stored in the Amazon OpenSearch Service domain internal database. type: str master_user_password: description: - - The master user's password, which is stored in the Amazon OpenSearch Service domain's internal database. + - The password of the master user, which is stored in the Amazon OpenSearch Service domain internal database. type: str - type: dict saml_options: description: - The SAML application configuration for the domain. - subptions: + type: dict + suboptions: enabled: description: - True if SAML is enabled. + - To use SAML authentication, you must enable fine-grained access control. + - You can only enable SAML authentication for OpenSearch Dashboards on existing domains, + not during the creation of new ones. + - Domains only support one Dashboards authentication method at a time. + If you have Amazon Cognito authentication for OpenSearch Dashboards enabled, + you must disable it before you can enable SAML. type: bool idp: description: - The SAML Identity Provider's information. + type: dict suboptions: metadata_content: description: @@ -229,10 +319,9 @@ description: - The unique entity ID of the application in SAML identity provider. type: str - type: dict master_user_name: description: - - The SAML master username, which is stored in the Amazon OpenSearch Service domain's internal database. + - The SAML master username, which is stored in the Amazon OpenSearch Service domain internal database. type: str master_backend_role: description: @@ -250,26 +339,30 @@ description: - The duration, in minutes, after which a user session becomes inactive. Acceptable values are between 1 and 1440, and the default value is 60. type: int - type: dict - type: dict auto_tune_options: description: - Specifies Auto-Tune options. + type: dict suboptions: desired_state: description: - The Auto-Tune desired state. Valid values are ENABLED and DISABLED. type: str - choices: ['ENABLED', 'DISABLED'] - maintenance_schedules: + choices: ['ENABLED', 'DISABLED'] + maintenance_schedules: description: - A list of maintenance schedules. + type: list + elements: dict suboptions: start_at: description: - The timestamp at which the Auto-Tune maintenance schedule starts. type: str duration: + description: + - Specifies maintenance schedule duration, duration value and duration unit. + type: dict suboptions: value: description: @@ -278,19 +371,15 @@ unit: description: - The unit of a maintenance schedule duration. Valid value is HOURS. - choices: ['HOURS'] + choices: ['HOURS'] type: str cron_expression_for_recurrence: description: - A cron expression for a recurring maintenance schedule. type: str - type: list - elements: dict - default: [] - type: dict wait: description: - - Whether or not to wait for snapshot creation or deletion. + - Whether or not to wait for completion of OpenSearch creation, modification or deletion. type: bool default: 'no' wait_timeout: @@ -300,16 +389,15 @@ type: int tags: description: - - tags dict to apply to a snapshot. + - tags dict to apply to an OpenSearch cluster. type: dict purge_tags: description: - whether to remove tags not present in the C(tags) parameter. default: True type: bool -author: - - "Jose Armesto (@fiunchinho)" - - "Sebastien Rosset (@sebastien.rosset)" +requirements: +- botocore >= 1.21.38 extends_documentation_fragment: - amazon.aws.aws - amazon.aws.ec2 @@ -317,37 +405,59 @@ EXAMPLES = """ -- name: Create OpenSearch domain +- name: Create OpenSearch domain for dev environment, no zone awareness, no dedicated masters + community.aws.opensearch: + domain_name: "dev-cluster" + engine_version: Elasticsearch_1.1 + cluster_config: + instance_type: "t2.small.search" + instance_count: 2 + zone_awareness: false + dedicated_master: false + ebs_options: + ebs_enabled: true + volume_type: "gp2" + volume_size: 10 + access_policies: "{{ lookup('file', 'policy.json') | from_json }}" + +- name: Create OpenSearch domain with dedicated masters community.aws.opensearch: domain_name: "my-domain" - engine_type: OpenSearch - version: "1.1" - instance_type: "m3.medium.search" - instance_count: 2 - dedicated_master: True - zone_awareness: True - availability_zone_count: 2 - dedicated_master_instance_type: "t2.micro.search" - dedicated_master_instance_count: 2 - ebs: True - volume_type: "io1" - volume_size: 10 - iops: 1000 - warm_enabled: true - warm_type: "ultrawarm1.medium.search" - warm_count: 1 - cold_storage_enabled: false - vpc_subnets: - - "subnet-e537d64a" - - "subnet-e537d64b" - vpc_security_groups: - - "sg-dd2f13cb" - - "sg-dd2f13cc" - snapshot_hour: 13 + engine_version: OpenSearch_1.1 + cluster_config: + instance_type: "t2.small.search" + instance_count: 12 + dedicated_master: true + zone_awareness: true + availability_zone_count: 2 + dedicated_master_instance_type: "t2.small.search" + dedicated_master_instance_count: 3 + warm_enabled: true + warm_type: "ultrawarm1.medium.search" + warm_count: 1 + cold_storage_options: + enabled: false + ebs_options: + ebs_enabled: true + volume_type: "io1" + volume_size: 10 + iops: 1000 + vpc_options: + subnets: + - "subnet-e537d64a" + - "subnet-e537d64b" + security_groups: + - "sg-dd2f13cb" + - "sg-dd2f13cc" + snapshot_options: + automated_snapshot_start_hour: 13 access_policies: "{{ lookup('file', 'policy.json') | from_json }}" - encryption_at_rest_enabled: false + encryption_at_rest_options: + enabled: false + node_to_node_encryption_options: + enabled: false auto_tune_options: - desired_state: "ENABLED" + enabled: true maintenance_schedules: - start_at: "2025-01-12" duration: @@ -361,13 +471,28 @@ cron_expression_for_recurrence: "cron(0 12 * * ? *)" tags: Environment: Development - Application: foo + Application: Search + wait: true + +- name: Increase size of EBS volumes for existing cluster + community.aws.opensearch: + domain_name: "my-domain" + ebs_options: + volume_size: 5 + wait: true + +- name: Increase instance count for existing cluster + community.aws.opensearch: + domain_name: "my-domain" + cluster_config: + instance_count: 40 wait: true """ +from copy import deepcopy +import datetime import json -import time try: import botocore @@ -377,86 +502,43 @@ from ansible.module_utils.six import string_types # import module snippets -from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ( - camel_dict_to_snake_dict, - AWSRetry, - compare_aws_tags, +from ansible_collections.amazon.aws.plugins.module_utils.core import ( + AnsibleAWSModule, + is_boto3_error_code, ) from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ( + AWSRetry, boto3_tag_list_to_ansible_dict, - ansible_dict_to_boto3_tag_list, + compare_policies, +) +from ansible_collections.community.aws.plugins.module_utils.opensearch import ( + compare_domain_versions, + ensure_tags, + get_domain_status, + get_domain_config, + get_target_increment_version, + normalize_opensearch, + parse_version, + wait_for_domain_status, ) - - -def get_domain(client, module, domain_name): - try: - response = client.describe_domain(DomainName=domain_name) - except client.exceptions.ResourceNotFoundException: - return None - except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: - module.fail_json_aws(e, msg="Couldn't get domain {0}".format(domain_name)) - return response["DomainStatus"] - - -def get_domain_config(client, module, domain_name): - try: - response = client.describe_domain_config(DomainName=domain_name) - except client.exceptions.ResourceNotFoundException: - return None - except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: - module.fail_json_aws(e, msg="Couldn't get domain {0}".format(domain_name)) - return response["DomainConfig"] - - -def domain_to_facts(client, module, domain): - try: - domain["Tags"] = boto3_tag_list_to_ansible_dict( - client.list_tags(ARN=domain["ARN"], aws_retry=True)["TagList"] - ) - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws( - e, "Couldn't get tags for domain %s" % domain["domain_name"] - ) - except KeyError: - module.fail_json(msg=str(domain)) - - return camel_dict_to_snake_dict(domain, ignore_list=["Tags"]) - - -def wait_for_domain_status(client, module, domain_name, waiter_name): - if not module.params["wait"]: - return - timeout = module.params["wait_timeout"] - deadline = time.time() + timeout - while time.time() < deadline: - status = get_domain(client, module, domain_name) - if status is None: - if waiter_name == "domain_deleted": - return - else: - if ( - waiter_name == "domain_available" - and status["Created"] - and not status["Processing"] - and not status["UpgradeProcessing"] - ): - return - time.sleep(15) def ensure_domain_absent(client, module): domain_name = module.params.get("domain_name") changed = False - domain = get_domain(client, module, domain_name) + domain = get_domain_status(client, module, domain_name) + if module.check_mode: + module.exit_json( + changed=True, msg="Would have deleted domain if not in check mode" + ) try: client.delete_domain(DomainName=domain_name) changed = True - except ( - botocore.exceptions.ClientError, - botocore.exceptions.BotoCoreError, - ) as e: + except is_boto3_error_code("ResourceNotFoundException"): + # The resource does not exist, or it has already been deleted + return dict(changed=False) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="trying to delete domain") # If we're not waiting for a delete to complete then we're all done @@ -466,237 +548,731 @@ def ensure_domain_absent(client, module): try: wait_for_domain_status(client, module, domain_name, "domain_deleted") return dict(changed=changed) - except client.exceptions.ResourceNotFoundException: + except is_boto3_error_code("ResourceNotFoundException"): return dict(changed=changed) - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, "awaiting domain deletion") -def ensure_tags(client, module, resource_arn, existing_tags, tags, purge_tags): - if tags is None: - return False - tags_to_add, tags_to_remove = compare_aws_tags(existing_tags, tags, purge_tags) - changed = bool(tags_to_add or tags_to_remove) - if tags_to_add: +def upgrade_domain(client, module, source_version, target_engine_version): + domain_name = module.params.get("domain_name") + # Determine if it's possible to upgrade directly from source version + # to target version, or if it's necessary to upgrade through intermediate major versions. + next_version = target_engine_version + # When perform_check_only is true, indicates that an upgrade eligibility check needs + # to be performed. Does not actually perform the upgrade. + perform_check_only = False + if module.check_mode: + perform_check_only = True + current_version = source_version + while current_version != target_engine_version: + v = get_target_increment_version(client, module, domain_name, target_engine_version) + if v is None: + # There is no compatible version, according to the get_compatible_versions() API. + # The upgrade should fail, but try anyway. + next_version = target_engine_version + if next_version != target_engine_version: + # It's not possible to upgrade directly to the target version. + # Check the module parameters to determine if this is allowed or not. + if not module.params.get("allow_intermediate_upgrades"): + module.fail_json(msg="Cannot upgrade from {0} to version {1}. The highest compatible version is {2}".format( + source_version, target_engine_version, next_version)) + + parameters = { + "DomainName": domain_name, + "TargetVersion": next_version, + "PerformCheckOnly": perform_check_only, + } + + if not module.check_mode: + # If background tasks are in progress, wait until they complete. + # This can take several hours depending on the cluster size and the type of background tasks + # (maybe an upgrade is already in progress). + # It's not possible to upgrade a domain that has background tasks are in progress, + # the call to client.upgrade_domain would fail. + wait_for_domain_status(client, module, domain_name, "domain_available") + try: - client.add_tags( - ARN=resource_arn, - TagList=ansible_dict_to_boto3_tag_list(tags_to_add), - ) - except ( - botocore.exceptions.ClientError, - botocore.exceptions.BotoCoreError, - ) as e: + client.upgrade_domain(**parameters) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + # In check mode (=> PerformCheckOnly==True), a ValidationException may be + # raised if it's not possible to upgrade to the target version. module.fail_json_aws( - e, "Couldn't add tags to domain {0}".format(resource_arn) + e, + msg="Couldn't upgrade domain {0} from {1} to {2}".format( + domain_name, current_version, next_version + ), ) - if tags_to_remove: - try: - client.remove_tags(ARN=resource_arn, TagKeys=tags_to_remove) - except ( - botocore.exceptions.ClientError, - botocore.exceptions.BotoCoreError, - ) as e: - module.fail_json_aws( - e, "Couldn't remove tags from domain {0}".format(resource_arn) + + if module.check_mode: + module.exit_json( + changed=True, + msg="Would have upgraded domain from {0} to {1} if not in check mode".format( + current_version, next_version + ), ) - return changed + current_version = next_version + if module.params.get("wait"): + wait_for_domain_status(client, module, domain_name, "domain_available") -def upgrade_domain(client, module, current_version, target_engine_version): - domain_name = module.params.get("domain_name") - parameters = { - "DomainName": domain_name, - "TargetVersion": target_engine_version, - "PerformCheckOnly": False, - } - # If background tasks are in progress, wait until they complete. - # This can take several hours depending on the cluster size. - wait_for_domain_status(client, module, domain_name, "domain_available") - try: - client.upgrade_domain(**parameters) - except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: - module.fail_json_aws( - e, - msg="Couldn't upgrade domain {0} from {1} to {2}".format( - domain_name, current_version, target_engine_version - ), - ) +def set_cluster_config( + module, current_domain_config, desired_domain_config, change_set +): + changed = False - if module.params.get("wait"): - wait_for_domain_status(client, module, domain_name, "domain_available") + cluster_config = desired_domain_config["ClusterConfig"] + cluster_opts = module.params.get("cluster_config") + if cluster_opts is not None: + if cluster_opts.get("instance_type") is not None: + cluster_config["InstanceType"] = cluster_opts.get("instance_type") + if cluster_opts.get("instance_count") is not None: + cluster_config["InstanceCount"] = cluster_opts.get("instance_count") + if cluster_opts.get("zone_awareness") is not None: + cluster_config["ZoneAwarenessEnabled"] = cluster_opts.get("zone_awareness") + if cluster_config["ZoneAwarenessEnabled"]: + if cluster_opts.get("availability_zone_count") is not None: + cluster_config["ZoneAwarenessConfig"] = { + "AvailabilityZoneCount": cluster_opts.get( + "availability_zone_count" + ), + } + + if cluster_opts.get("dedicated_master") is not None: + cluster_config["DedicatedMasterEnabled"] = cluster_opts.get( + "dedicated_master" + ) + if cluster_config["DedicatedMasterEnabled"]: + if cluster_opts.get("dedicated_master_instance_type") is not None: + cluster_config["DedicatedMasterType"] = cluster_opts.get( + "dedicated_master_instance_type" + ) + if cluster_opts.get("dedicated_master_instance_count") is not None: + cluster_config["DedicatedMasterCount"] = cluster_opts.get( + "dedicated_master_instance_count" + ) + if cluster_opts.get("warm_enabled") is not None: + cluster_config["WarmEnabled"] = cluster_opts.get("warm_enabled") + if cluster_config["WarmEnabled"]: + if cluster_opts.get("warm_type") is not None: + cluster_config["WarmType"] = cluster_opts.get("warm_type") + if cluster_opts.get("warm_count") is not None: + cluster_config["WarmCount"] = cluster_opts.get("warm_count") + + cold_storage_opts = None + if cluster_opts is not None: + cold_storage_opts = cluster_opts.get("cold_storage_options") + if compare_domain_versions(desired_domain_config["EngineVersion"], "Elasticsearch_7.9") < 0: + # If the engine version is ElasticSearch < 7.9, cold storage is not supported. + # When querying a domain < 7.9, the AWS API indicates cold storage is disabled (Enabled: False), + # which makes sense. However, trying to do HTTP POST with Enable: False causes an API error. + # The 'ColdStorageOptions' attribute should not be present in HTTP POST. + if cold_storage_opts is not None and cold_storage_opts.get("enabled"): + module.fail_json(msg="Cold Storage is not supported") + cluster_config.pop("ColdStorageOptions", None) + if ( + current_domain_config is not None + and "ClusterConfig" in current_domain_config + ): + # Remove 'ColdStorageOptions' from the current domain config, otherwise the actual vs desired diff + # will indicate a change must be done. + current_domain_config["ClusterConfig"].pop("ColdStorageOptions", None) + else: + # Elasticsearch 7.9 and above support ColdStorageOptions. + if ( + cold_storage_opts is not None + and cold_storage_opts.get("enabled") is not None + ): + cluster_config["ColdStorageOptions"] = { + "Enabled": cold_storage_opts.get("enabled"), + } -def ensure_domain_present(client, module): - domain_name = module.params.get("domain_name") - target_engine_version = "%s_%s" % ( - module.params.get("engine_type"), - module.params.get("version"), - ) + if ( + current_domain_config is not None + and current_domain_config["ClusterConfig"] != cluster_config + ): + change_set.append( + "ClusterConfig changed from {0} to {1}".format( + current_domain_config["ClusterConfig"], cluster_config + ) + ) + changed = True + return changed - domain_config = { - "InstanceType": module.params.get("instance_type"), - "InstanceCount": int(module.params.get("instance_count")), - "DedicatedMasterEnabled": module.params.get("dedicated_master"), - "ZoneAwarenessEnabled": module.params.get("zone_awareness"), - "WarmEnabled": module.params.get("warm_enabled"), - } - if module.params.get("cold_storage_enabled") is not None: - domain_config["ColdStorageOptions"] = { - "Enabled": module.params.get("cold_storage_enabled"), + +def set_ebs_options(module, current_domain_config, desired_domain_config, change_set): + changed = False + ebs_config = desired_domain_config["EBSOptions"] + ebs_opts = module.params.get("ebs_options") + if ebs_opts is None: + return changed + if ebs_opts.get("ebs_enabled") is not None: + ebs_config["EBSEnabled"] = ebs_opts.get("ebs_enabled") + + if not ebs_config["EBSEnabled"]: + desired_domain_config["EBSOptions"] = { + "EBSEnabled": False, } + else: + if ebs_opts.get("volume_type") is not None: + ebs_config["VolumeType"] = ebs_opts.get("volume_type") + if ebs_opts.get("volume_size") is not None: + ebs_config["VolumeSize"] = ebs_opts.get("volume_size") + if ebs_opts.get("iops") is not None: + ebs_config["Iops"] = ebs_opts.get("iops") + + if ( + current_domain_config is not None + and current_domain_config["EBSOptions"] != ebs_config + ): + change_set.append( + "EBSOptions changed from {0} to {1}".format( + current_domain_config["EBSOptions"], ebs_config + ) + ) + changed = True + return changed - if module.params.get("zone_awareness"): - domain_config["ZoneAwarenessConfig"] = { - "AvailabilityZoneCount": module.params.get("availability_zone_count"), + +def set_encryption_at_rest_options( + module, current_domain_config, desired_domain_config, change_set +): + changed = False + encryption_at_rest_config = desired_domain_config["EncryptionAtRestOptions"] + encryption_at_rest_opts = module.params.get("encryption_at_rest_options") + if encryption_at_rest_opts is None: + return False + if encryption_at_rest_opts.get("enabled") is not None: + encryption_at_rest_config["Enabled"] = encryption_at_rest_opts.get("enabled") + if not encryption_at_rest_config["Enabled"]: + desired_domain_config["EncryptionAtRestOptions"] = { + "Enabled": False, } + else: + if encryption_at_rest_opts.get("kms_key_id") is not None: + encryption_at_rest_config["KmsKeyId"] = encryption_at_rest_opts.get( + "kms_key_id" + ) - if module.params.get("warm_enabled"): - domain_config["WarmType"] = module.params.get("warm_type") - domain_config["WarmCount"] = module.params.get("warm_count") + if ( + current_domain_config is not None + and current_domain_config["EncryptionAtRestOptions"] + != encryption_at_rest_config + ): + change_set.append( + "EncryptionAtRestOptions changed from {0} to {1}".format( + current_domain_config["EncryptionAtRestOptions"], + encryption_at_rest_config, + ) + ) + changed = True + return changed - ebs_options = {"EBSEnabled": module.params.get("ebs")} - encryption_at_rest_enabled = module.params.get("encryption_at_rest_enabled") == True - encryption_at_rest_options = {"Enabled": encryption_at_rest_enabled} +def set_node_to_node_encryption_options( + module, current_domain_config, desired_domain_config, change_set +): + changed = False + node_to_node_encryption_config = desired_domain_config[ + "NodeToNodeEncryptionOptions" + ] + node_to_node_encryption_opts = module.params.get("node_to_node_encryption_options") + if node_to_node_encryption_opts is None: + return changed + if node_to_node_encryption_opts.get("enabled") is not None: + node_to_node_encryption_config["Enabled"] = node_to_node_encryption_opts.get( + "enabled" + ) - if encryption_at_rest_enabled: - encryption_at_rest_options["KmsKeyId"] = module.params.get( - "encryption_at_rest_kms_key_id" + if ( + current_domain_config is not None + and current_domain_config["NodeToNodeEncryptionOptions"] + != node_to_node_encryption_config + ): + change_set.append( + "NodeToNodeEncryptionOptions changed from {0} to {1}".format( + current_domain_config["NodeToNodeEncryptionOptions"], + node_to_node_encryption_config, + ) ) + changed = True + return changed - vpc_options = {} - vpc_subnets = module.params.get("vpc_subnets") - if vpc_subnets: + +def set_vpc_options(module, current_domain_config, desired_domain_config, change_set): + changed = False + vpc_config = None + if "VPCOptions" in desired_domain_config: + vpc_config = desired_domain_config["VPCOptions"] + vpc_opts = module.params.get("vpc_options") + if vpc_opts is None: + return changed + vpc_subnets = vpc_opts.get("subnets") + if vpc_subnets is not None: + if vpc_config is None: + vpc_config = {} + desired_domain_config["VPCOptions"] = vpc_config + # OpenSearch cluster is attached to VPC if isinstance(vpc_subnets, string_types): vpc_subnets = [x.strip() for x in vpc_subnets.split(",")] - vpc_options["SubnetIds"] = vpc_subnets + vpc_config["SubnetIds"] = vpc_subnets - vpc_security_groups = module.params.get("vpc_security_groups") - if vpc_security_groups: + vpc_security_groups = vpc_opts.get("security_groups") + if vpc_security_groups is not None: + if vpc_config is None: + vpc_config = {} + desired_domain_config["VPCOptions"] = vpc_config if isinstance(vpc_security_groups, string_types): vpc_security_groups = [x.strip() for x in vpc_security_groups.split(",")] - vpc_options["SecurityGroupIds"] = vpc_security_groups + vpc_config["SecurityGroupIds"] = vpc_security_groups + + if current_domain_config is not None: + # Modify existing cluster. + current_cluster_is_vpc = False + desired_cluster_is_vpc = False + if ( + "VPCOptions" in current_domain_config + and "SubnetIds" in current_domain_config["VPCOptions"] + and len(current_domain_config["VPCOptions"]["SubnetIds"]) > 0 + ): + current_cluster_is_vpc = True + if ( + "VPCOptions" in desired_domain_config + and "SubnetIds" in desired_domain_config["VPCOptions"] + and len(desired_domain_config["VPCOptions"]["SubnetIds"]) > 0 + ): + desired_cluster_is_vpc = True + if current_cluster_is_vpc != desired_cluster_is_vpc: + # AWS does not allow changing the type. Don't fail here so we return the AWS API error. + change_set.append("VPCOptions changed between Internet and VPC") + changed = True + elif desired_cluster_is_vpc is False: + # There are no VPCOptions to configure. + pass + else: + # Note the subnets may be the same but be listed in a different order. + if set(current_domain_config["VPCOptions"]["SubnetIds"]) != set( + vpc_config["SubnetIds"] + ): + change_set.append( + "SubnetIds changed from {0} to {1}".format( + current_domain_config["VPCOptions"]["SubnetIds"], + vpc_config["SubnetIds"], + ) + ) + changed = True + if set(current_domain_config["VPCOptions"]["SecurityGroupIds"]) != set( + vpc_config["SecurityGroupIds"] + ): + change_set.append( + "SecurityGroup changed from {0} to {1}".format( + current_domain_config["VPCOptions"]["SecurityGroupIds"], + vpc_config["SecurityGroupIds"], + ) + ) + changed = True + return changed + - if domain_config["DedicatedMasterEnabled"]: - domain_config["DedicatedMasterType"] = module.params.get( - "dedicated_master_instance_type" +def set_snapshot_options( + module, current_domain_config, desired_domain_config, change_set +): + changed = False + snapshot_config = desired_domain_config["SnapshotOptions"] + snapshot_opts = module.params.get("snapshot_options") + if snapshot_opts is None: + return changed + if snapshot_opts.get("automated_snapshot_start_hour") is not None: + snapshot_config["AutomatedSnapshotStartHour"] = snapshot_opts.get( + "automated_snapshot_start_hour" ) - domain_config["DedicatedMasterCount"] = module.params.get( - "dedicated_master_instance_count" + if ( + current_domain_config is not None + and current_domain_config["SnapshotOptions"] != snapshot_config + ): + change_set.append("SnapshotOptions changed") + changed = True + return changed + + +def set_cognito_options( + module, current_domain_config, desired_domain_config, change_set +): + changed = False + cognito_config = desired_domain_config["CognitoOptions"] + cognito_opts = module.params.get("cognito_options") + if cognito_opts is None: + return changed + if cognito_opts.get("enabled") is not None: + cognito_config["Enabled"] = cognito_opts.get("enabled") + if not cognito_config["Enabled"]: + desired_domain_config["CognitoOptions"] = { + "Enabled": False, + } + else: + if cognito_opts.get("cognito_user_pool_id") is not None: + cognito_config["UserPoolId"] = cognito_opts.get("cognito_user_pool_id") + if cognito_opts.get("cognito_identity_pool_id") is not None: + cognito_config["IdentityPoolId"] = cognito_opts.get( + "cognito_identity_pool_id" + ) + if cognito_opts.get("cognito_role_arn") is not None: + cognito_config["RoleArn"] = cognito_opts.get("cognito_role_arn") + + if ( + current_domain_config is not None + and current_domain_config["CognitoOptions"] != cognito_config + ): + change_set.append( + "CognitoOptions changed from {0} to {1}".format( + current_domain_config["CognitoOptions"], cognito_config + ) ) + changed = True + return changed - if ebs_options["EBSEnabled"]: - ebs_options["VolumeType"] = module.params.get("volume_type") - ebs_options["VolumeSize"] = module.params.get("volume_size") - if module.params.get("iops") is not None: - ebs_options["Iops"] = module.params.get("iops") +def set_advanced_security_options( + module, current_domain_config, desired_domain_config, change_set +): + changed = False + advanced_security_config = desired_domain_config["AdvancedSecurityOptions"] + advanced_security_opts = module.params.get("advanced_security_options") + if advanced_security_opts is None: + return changed + if advanced_security_opts.get("enabled") is not None: + advanced_security_config["Enabled"] = advanced_security_opts.get("enabled") + if not advanced_security_config["Enabled"]: + desired_domain_config["AdvancedSecurityOptions"] = { + "Enabled": False, + } + else: + if advanced_security_opts.get("internal_user_database_enabled") is not None: + advanced_security_config[ + "InternalUserDatabaseEnabled" + ] = advanced_security_opts.get("internal_user_database_enabled") + master_user_opts = advanced_security_opts.get("master_user_options") + if master_user_opts is not None: + if master_user_opts.get("master_user_arn") is not None: + advanced_security_config["MasterUserOptions"][ + "MasterUserARN" + ] = master_user_opts.get("master_user_arn") + if master_user_opts.get("master_user_name") is not None: + advanced_security_config["MasterUserOptions"][ + "MasterUserName" + ] = master_user_opts.get("master_user_name") + if master_user_opts.get("master_user_password") is not None: + advanced_security_config["MasterUserOptions"][ + "MasterUserPassword" + ] = master_user_opts.get("master_user_password") + saml_opts = advanced_security_opts.get("saml_options") + if saml_opts is not None: + if saml_opts.get("enabled") is not None: + advanced_security_config["SamlOptions"]["Enabled"] = saml_opts.get( + "enabled" + ) + idp_opts = saml_opts.get("idp") + if idp_opts is not None: + if idp_opts.get("metadata_content") is not None: + advanced_security_config["SamlOptions"]["Idp"][ + "MetadataContent" + ] = idp_opts.get("metadata_content") + if idp_opts.get("entity_id") is not None: + advanced_security_config["SamlOptions"]["Idp"][ + "EntityId" + ] = idp_opts.get("entity_id") + if saml_opts.get("master_user_name") is not None: + advanced_security_config["SamlOptions"][ + "MasterUserName" + ] = saml_opts.get("master_user_name") + if saml_opts.get("master_backend_role") is not None: + advanced_security_config["SamlOptions"][ + "MasterBackendRole" + ] = saml_opts.get("master_backend_role") + if saml_opts.get("subject_key") is not None: + advanced_security_config["SamlOptions"]["SubjectKey"] = saml_opts.get( + "subject_key" + ) + if saml_opts.get("roles_key") is not None: + advanced_security_config["SamlOptions"]["RolesKey"] = saml_opts.get( + "roles_key" + ) + if saml_opts.get("session_timeout_minutes") is not None: + advanced_security_config["SamlOptions"][ + "SessionTimeoutMinutes" + ] = saml_opts.get("session_timeout_minutes") + + if ( + current_domain_config is not None + and current_domain_config["AdvancedSecurityOptions"] != advanced_security_config + ): + change_set.append( + "AdvancedSecurityOptions changed from {0} to {1}".format( + current_domain_config["AdvancedSecurityOptions"], + advanced_security_config, + ) + ) + changed = True + return changed - snapshot_options = { - "AutomatedSnapshotStartHour": module.params.get("snapshot_hour") - } - cognito_options = {} - if module.params.get("cognito_enabled"): - cognito_options["Enabled"] = True - cognito_options["UserPoolId"] = module.params.get("cognito_user_pool_id") - cognito_options["IdentityPoolId"] = module.params.get( - "cognito_identity_pool_id" +def set_domain_endpoint_options( + module, current_domain_config, desired_domain_config, change_set +): + changed = False + domain_endpoint_config = desired_domain_config["DomainEndpointOptions"] + domain_endpoint_opts = module.params.get("domain_endpoint_options") + if domain_endpoint_opts is None: + return changed + if domain_endpoint_opts.get("enforce_https") is not None: + domain_endpoint_config["EnforceHTTPS"] = domain_endpoint_opts.get( + "enforce_https" ) - cognito_options["RoleArn"] = module.params.get("cognito_role_arn") - else: - cognito_options["Enabled"] = False - - auto_tune_options = {} - opts = module.params.get("auto_tune_options") - if opts: - if opts.get("desired_state") is not None: - auto_tune_options["DesiredState"] = opts.get("desired_state") - schedules = opts.get("maintenance_schedules") - if schedules is not None: - auto_tune_options["MaintenanceSchedules"] = [] - for s in schedules: - auto_tune_options["MaintenanceSchedules"].append({ - "StartAt": s.get("start_at"), - "Duration": { - "Value": s.get("duration").get("value"), - "Unit": s.get("duration").get("unit"), - }, - "CronExpressionForRecurrence": s.get("cron_expression_for_recurrence") - }) + if domain_endpoint_opts.get("tls_security_policy") is not None: + domain_endpoint_config["TLSSecurityPolicy"] = domain_endpoint_opts.get( + "tls_security_policy" + ) + if domain_endpoint_opts.get("custom_endpoint_enabled") is not None: + domain_endpoint_config["CustomEndpointEnabled"] = domain_endpoint_opts.get( + "custom_endpoint_enabled" + ) + if domain_endpoint_config["CustomEndpointEnabled"]: + if domain_endpoint_opts.get("custom_endpoint") is not None: + domain_endpoint_config["CustomEndpoint"] = domain_endpoint_opts.get( + "custom_endpoint" + ) + if domain_endpoint_opts.get("custom_endpoint_certificate_arn") is not None: + domain_endpoint_config[ + "CustomEndpointCertificateArn" + ] = domain_endpoint_opts.get("custom_endpoint_certificate_arn") + + if ( + current_domain_config is not None + and current_domain_config["DomainEndpointOptions"] != domain_endpoint_config + ): + change_set.append( + "DomainEndpointOptions changed from {0} to {1}".format( + current_domain_config["DomainEndpointOptions"], domain_endpoint_config + ) + ) + changed = True + return changed + +def set_auto_tune_options( + module, current_domain_config, desired_domain_config, change_set +): changed = False + auto_tune_config = desired_domain_config["AutoTuneOptions"] + auto_tune_opts = module.params.get("auto_tune_options") + if auto_tune_opts is None: + return changed + schedules = auto_tune_opts.get("maintenance_schedules") + if auto_tune_opts.get("desired_state") is not None: + auto_tune_config["DesiredState"] = auto_tune_opts.get("desired_state") + if auto_tune_config["DesiredState"] != "ENABLED": + desired_domain_config["AutoTuneOptions"] = { + "DesiredState": "DISABLED", + } + elif schedules is not None: + auto_tune_config["MaintenanceSchedules"] = [] + for s in schedules: + schedule_entry = {} + start_at = s.get("start_at") + if start_at is not None: + if isinstance(start_at, datetime.datetime): + # The property was parsed from yaml to datetime, but the AWS API wants a string + start_at = start_at.strftime("%Y-%m-%d") + schedule_entry["StartAt"] = start_at + duration_opt = s.get("duration") + if duration_opt is not None: + schedule_entry["Duration"] = {} + if duration_opt.get("value") is not None: + schedule_entry["Duration"]["Value"] = duration_opt.get("value") + if duration_opt.get("unit") is not None: + schedule_entry["Duration"]["Unit"] = duration_opt.get("unit") + if s.get("cron_expression_for_recurrence") is not None: + schedule_entry["CronExpressionForRecurrence"] = s.get( + "cron_expression_for_recurrence" + ) + auto_tune_config["MaintenanceSchedules"].append(schedule_entry) + if current_domain_config is not None: + if ( + current_domain_config["AutoTuneOptions"]["DesiredState"] + != auto_tune_config["DesiredState"] + ): + change_set.append( + "AutoTuneOptions.DesiredState changed from {0} to {1}".format( + current_domain_config["AutoTuneOptions"]["DesiredState"], + auto_tune_config["DesiredState"], + ) + ) + changed = True + if ( + auto_tune_config["MaintenanceSchedules"] + != current_domain_config["AutoTuneOptions"]["MaintenanceSchedules"] + ): + change_set.append( + "AutoTuneOptions.MaintenanceSchedules changed from {0} to {1}".format( + current_domain_config["AutoTuneOptions"]["MaintenanceSchedules"], + auto_tune_config["MaintenanceSchedules"], + ) + ) + changed = True + return changed + +def set_access_policy(module, current_domain_config, desired_domain_config, change_set): + access_policy_config = None + changed = False + access_policy_opt = module.params.get("access_policies") + if access_policy_opt is None: + return changed try: - pdoc = json.dumps(module.params.get("access_policies")) + access_policy_config = json.dumps(access_policy_opt) except Exception as e: module.fail_json( msg="Failed to convert the policy into valid JSON: %s" % str(e) ) - - domain = get_domain(client, module, domain_name) - if domain is not None: - if target_engine_version != domain["EngineVersion"]: - changed = True - upgrade_domain( - client, module, domain["EngineVersion"], target_engine_version + if current_domain_config is not None: + # Updating existing domain + current_access_policy = json.loads(current_domain_config["AccessPolicies"]) + if not compare_policies(current_access_policy, access_policy_opt): + change_set.append( + "AccessPolicy changed from {0} to {1}".format( + current_access_policy, access_policy_opt + ) ) - - # Modify the provided policy to provide reliable changed detection - policy_dict = module.params.get("access_policies") - for statement in policy_dict["Statement"]: - if "Resource" not in statement: - # The opensearch APIs will implicitly set this - statement["Resource"] = "%s/*" % domain["ARN"] - pdoc = json.dumps(policy_dict) - - if domain["ClusterConfig"] != domain_config: changed = True + desired_domain_config["AccessPolicies"] = access_policy_config + else: + # Creating new domain + desired_domain_config["AccessPolicies"] = access_policy_config + return changed - if domain["EBSOptions"] != ebs_options: - changed = True - if domain["CognitoOptions"] != cognito_options: - changed = True +def ensure_domain_present(client, module): + domain_name = module.params.get("domain_name") - if domain["AutoTuneOptions"] != auto_tune_options: - changed = True + # Create default if OpenSearch does not exist. If domain already exists, + # the data is populated by retrieving the current configuration from the API. + desired_domain_config = { + "DomainName": module.params.get("domain_name"), + "EngineVersion": "OpenSearch_1.1", + "ClusterConfig": { + "InstanceType": "t2.small.search", + "InstanceCount": 2, + "ZoneAwarenessEnabled": False, + "DedicatedMasterEnabled": False, + "WarmEnabled": False, + }, + # By default create ES attached to the Internet. + # If the "VPCOptions" property is specified, even if empty, the API server interprets + # as incomplete VPC configuration. + # "VPCOptions": {}, + "EBSOptions": { + "EBSEnabled": False, + }, + "EncryptionAtRestOptions": { + "Enabled": False, + }, + "NodeToNodeEncryptionOptions": { + "Enabled": False, + }, + "SnapshotOptions": { + "AutomatedSnapshotStartHour": 0, + }, + "CognitoOptions": { + "Enabled": False, + }, + "AdvancedSecurityOptions": { + "Enabled": False, + }, + "DomainEndpointOptions": { + "CustomEndpointEnabled": False, + }, + "AutoTuneOptions": { + "DesiredState": "DISABLED", + }, + } + # Determine if OpenSearch domain already exists. + # current_domain_config may be None if the domain does not exist. + (current_domain_config, domain_arn) = get_domain_config(client, module, domain_name) + if current_domain_config is not None: + desired_domain_config = deepcopy(current_domain_config) + + if module.params.get("engine_version") is not None: + # Validate the engine_version + v = parse_version(module.params.get("engine_version")) + if v is None: + module.fail_json( + "Invalid engine_version. Must be Elasticsearch_X.Y or OpenSearch_X.Y" + ) + desired_domain_config["EngineVersion"] = module.params.get("engine_version") - if "VPCOptions" in domain: - if domain["VPCOptions"]["SubnetIds"] != vpc_options["SubnetIds"]: - changed = True - if ( - domain["VPCOptions"]["SecurityGroupIds"] - != vpc_options["SecurityGroupIds"] - ): - changed = True + changed = False + change_set = [] # For check mode purpose - if domain["SnapshotOptions"] != snapshot_options: - changed = True + changed |= set_cluster_config( + module, current_domain_config, desired_domain_config, change_set + ) + changed |= set_ebs_options( + module, current_domain_config, desired_domain_config, change_set + ) + changed |= set_encryption_at_rest_options( + module, current_domain_config, desired_domain_config, change_set + ) + changed |= set_node_to_node_encryption_options( + module, current_domain_config, desired_domain_config, change_set + ) + changed |= set_vpc_options( + module, current_domain_config, desired_domain_config, change_set + ) + changed |= set_snapshot_options( + module, current_domain_config, desired_domain_config, change_set + ) + changed |= set_cognito_options( + module, current_domain_config, desired_domain_config, change_set + ) + changed |= set_advanced_security_options( + module, current_domain_config, desired_domain_config, change_set + ) + changed |= set_domain_endpoint_options( + module, current_domain_config, desired_domain_config, change_set + ) + changed |= set_auto_tune_options( + module, current_domain_config, desired_domain_config, change_set + ) + changed |= set_access_policy( + module, current_domain_config, desired_domain_config, change_set + ) - current_policy_dict = json.loads(domain["AccessPolicies"]) - if current_policy_dict != policy_dict: + if current_domain_config is not None: + if ( + desired_domain_config["EngineVersion"] + != current_domain_config["EngineVersion"] + ): changed = True + change_set.append("EngineVersion changed") + upgrade_domain( + client, + module, + current_domain_config["EngineVersion"], + desired_domain_config["EngineVersion"], + ) if changed: - parameters = { - "DomainName": module.params.get("domain_name"), - "EncryptionAtRestOptions": encryption_at_rest_options, - "ClusterConfig": domain_config, - "EBSOptions": ebs_options, - "SnapshotOptions": snapshot_options, - "AccessPolicies": pdoc, - "CognitoOptions": cognito_options, - "AutoTuneOptions": auto_tune_options, - } - - if vpc_options["SubnetIds"] or vpc_options["SecurityGroupIds"]: - parameters["VPCOptions"] = vpc_options - + if module.check_mode: + module.exit_json( + changed=True, + msg=f"Would have updated domain if not in check mode: {change_set}", + ) + # Remove the "EngineVersion" attribute, the AWS API does not accept this attribute. + desired_domain_config.pop("EngineVersion", None) try: - client.update_domain_config(**parameters) + client.update_domain_config(**desired_domain_config) except ( botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError, @@ -706,24 +1282,21 @@ def ensure_domain_present(client, module): ) else: - changed = True - parameters = { - "DomainName": module.params.get("domain_name"), - "EngineVersion": target_engine_version, - "EncryptionAtRestOptions": encryption_at_rest_options, - "ClusterConfig": domain_config, - "EBSOptions": ebs_options, - "SnapshotOptions": snapshot_options, - "AccessPolicies": pdoc, - "CognitoOptions": cognito_options, - "AutoTuneOptions": auto_tune_options, - } - - if vpc_options["SubnetIds"] or vpc_options["SecurityGroupIds"]: - parameters["VPCOptions"] = vpc_options + # Create new OpenSearch cluster + if module.params.get("access_policies") is None: + module.fail_json( + "state is present but the following is missing: access_policies" + ) + changed = True + if module.check_mode: + module.exit_json( + changed=True, msg="Would have created a domain if not in check mode" + ) try: - client.create_domain(**parameters) + response = client.create_domain(**desired_domain_config) + domain = response["DomainStatus"] + domain_arn = domain["ARN"] except ( botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError, @@ -732,12 +1305,9 @@ def ensure_domain_present(client, module): e, msg="Couldn't update domain {0}".format(domain_name) ) - if module.params.get("wait"): - wait_for_domain_status(client, module, domain_name, "domain_available") - try: existing_tags = boto3_tag_list_to_ansible_dict( - client.list_tags(ARN=domain["ARN"], aws_retry=True)["TagList"] + client.list_tags(ARN=domain_arn, aws_retry=True)["TagList"] ) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, "Couldn't get tags for domain %s" % domain_name) @@ -745,140 +1315,185 @@ def ensure_domain_present(client, module): desired_tags = module.params["tags"] purge_tags = module.params["purge_tags"] changed |= ensure_tags( - client, module, domain["ARN"], existing_tags, desired_tags, purge_tags + client, module, domain_arn, existing_tags, desired_tags, purge_tags ) - domain = get_domain(client, module, domain_name) - - return dict(changed=changed, **domain_to_facts(client, module, domain)) + if module.params.get("wait") and not module.check_mode: + wait_for_domain_status(client, module, domain_name, "domain_available") + domain = get_domain_status(client, module, domain_name) -def main(): + return dict(changed=changed, **normalize_opensearch(client, module, domain)) - domain_endpoint_options = dict( - enforce_https=dict(type="bool"), - tls_security_policy=dict(), - custom_endpoint_enabled=dict(type="bool"), - custom_endpoint=dict(), - custom_endpoint_certificate_arn=dict(), - ) - master_user_options = dict( - master_user_arn=dict(), - master_user_name=dict(), - master_user_password=dict(), - ) - advanced_security_options = dict( - enabled=dict(type="bool"), - internal_user_database_enabled=dict(type="bool"), - master_user_options=dict(type="dict", options=master_user_options), - ) - - idp = dict( - metadata_content=dict(), - entity_id=dict(), - ) - saml_options = dict( - enabled=dict(type="bool"), - idp=dict(type="dict", options=idp), - master_user_name=dict(), - master_backend_role=dict(), - subject_key=dict(), - roles_key=dict(), - session_timeout_minutes=dict(type="int"), - ) - - auto_tune_duration = dict( - value=dict(type="int"), - unit=dict(choices=["HOURS"]), - ) - auto_tune_maintenance_schedules = dict( - start_at=dict(), - duration=dict(type="dict", options=auto_tune_duration), - cron_expression_for_recurrence=dict(), - ) - auto_tune_options = dict( - desired_state=dict(choices=["ENABLED", "DISABLED"]), - maintenance_schedules=dict( - type="list", - default=[], - elements="dict", - options=auto_tune_maintenance_schedules, - ), - ) +def main(): module = AnsibleAWSModule( argument_spec=dict( state=dict(choices=["present", "absent"], default="present"), domain_name=dict(required=True), - engine_type=dict( - default="OpenSearch", choices=["OpenSearch", "Elasticsearch"] + engine_version=dict(), + allow_intermediate_upgrades=dict(required=False, type="bool", default=True), + access_policies=dict(required=False, type="dict"), + cluster_config=dict( + type="dict", + default=None, + options=dict( + instance_type=dict(), + instance_count=dict(required=False, type="int"), + zone_awareness=dict(required=False, type="bool"), + availability_zone_count=dict(required=False, type="int"), + dedicated_master=dict(required=False, type="bool"), + dedicated_master_instance_type=dict(), + dedicated_master_instance_count=dict(type="int"), + warm_enabled=dict(required=False, type="bool"), + warm_type=dict(required=False), + warm_count=dict(required=False, type="int"), + cold_storage_options=dict( + type="dict", + default=None, + options=dict( + enabled=dict(required=False, type="bool"), + ), + ), + ), + ), + snapshot_options=dict( + type="dict", + default=None, + options=dict( + automated_snapshot_start_hour=dict(required=False, type="int"), + ), + ), + ebs_options=dict( + type="dict", + default=None, + options=dict( + ebs_enabled=dict(required=False, type="bool"), + volume_type=dict(required=False), + volume_size=dict(required=False, type="int"), + iops=dict(required=False, type="int"), + ), + ), + vpc_options=dict( + type="dict", + default=None, + options=dict( + subnets=dict(type="list", elements="str", required=False), + security_groups=dict(type="list", elements="str", required=False), + ), + ), + cognito_options=dict( + type="dict", + default=None, + options=dict( + enabled=dict(required=False, type="bool"), + user_pool_id=dict(required=False), + identity_pool_id=dict(required=False), + role_arn=dict(required=False, no_log=False), + ), + ), + encryption_at_rest_options=dict( + type="dict", + default=None, + options=dict( + enabled=dict(type="bool"), + kms_key_id=dict(required=False), + ), + ), + node_to_node_encryption_options=dict( + type="dict", + default=None, + options=dict( + enabled=dict(type="bool"), + ), + ), + domain_endpoint_options=dict( + type="dict", + default=None, + options=dict( + enforce_https=dict(type="bool"), + tls_security_policy=dict(), + custom_endpoint_enabled=dict(type="bool"), + custom_endpoint=dict(), + custom_endpoint_certificate_arn=dict(), + ), ), - version=dict(default="1.1"), - wait=dict(type="bool", default=False), - wait_timeout=dict(type="int", default=300), - instance_type=dict(default="t2.small.search"), - instance_count=dict(required=True, type="int"), - dedicated_master=dict(required=True, type="bool"), - zone_awareness=dict(required=True, type="bool"), - availability_zone_count=dict(required=False, type="int"), - dedicated_master_instance_type=dict(), - dedicated_master_instance_count=dict(type="int"), - ebs=dict(required=True, type="bool"), - volume_type=dict(required=True), - volume_size=dict(required=True, type="int"), - iops=dict(required=False, type="int"), - warm_enabled=dict(required=False, type="bool", default=False), - warm_type=dict(required=False), - warm_count=dict(required=False, type="int"), - cold_storage_enabled=dict(required=False, type="bool"), - access_policies=dict(required=True, type="dict"), - vpc_subnets=dict(type="list", elements="str", required=False), - vpc_security_groups=dict(type="list", elements="str", required=False), - snapshot_hour=dict(required=False, type="int", default=0), - encryption_at_rest_enabled=dict(type="bool", default=False), - encryption_at_rest_kms_key_id=dict(required=False), - cognito_enabled=dict(required=False, type="bool", default=False), - cognito_user_pool_id=dict(required=False), - cognito_identity_pool_id=dict(required=False), - cognito_role_arn=dict(required=False), - domain_endpoint_options=dict(type="dict", options=domain_endpoint_options), advanced_security_options=dict( - type="dict", options=advanced_security_options + type="dict", + default=None, + options=dict( + enabled=dict(type="bool"), + internal_user_database_enabled=dict(type="bool"), + master_user_options=dict( + type="dict", + default=None, + options=dict( + master_user_arn=dict(), + master_user_name=dict(), + master_user_password=dict(no_log=True), + ), + ), + saml_options=dict( + type="dict", + default=None, + options=dict( + enabled=dict(type="bool"), + idp=dict( + type="dict", + default=None, + options=dict( + metadata_content=dict(), + entity_id=dict(), + ), + ), + master_user_name=dict(), + master_backend_role=dict(), + subject_key=dict(no_log=False), + roles_key=dict(no_log=False), + session_timeout_minutes=dict(type="int"), + ), + ), + ), + ), + auto_tune_options=dict( + type="dict", + default=None, + options=dict( + desired_state=dict(choices=["ENABLED", "DISABLED"]), + maintenance_schedules=dict( + type="list", + elements="dict", + default=None, + options=dict( + start_at=dict(), + duration=dict( + type="dict", + default=None, + options=dict( + value=dict(type="int"), + unit=dict(choices=["HOURS"]), + ), + ), + cron_expression_for_recurrence=dict(), + ), + ), + ), ), - saml_options=dict(type="dict", options=saml_options), - auto_tune_options=dict(type="dict", options=auto_tune_options), tags=dict(type="dict"), purge_tags=dict(type="bool", default=True), + wait=dict(type="bool", default=False), + wait_timeout=dict(type="int", default=300), ), - required_if=[ - ["warm_enabled", True, ["warm_type", "warm_count"]], - ["zone_awareness", True, ["availability_zone_count"]], - [ - "dedicated_master", - True, - ["dedicated_master_instance_type", "dedicated_master_instance_count"], - ], - ["ebs", True, ["volume_type", "volume_size"]], - [ - "cognito_enabled", - True, - [ - "cognito_user_pool_id", - "cognito_identity_pool_id", - "cognito_role_arn", - ], - ], - ["encryption_at_rest_enabled", True, ["encryption_at_rest_kms_key_id"]], - ], + supports_check_mode=True, ) + module.require_botocore_at_least("1.21.38") + try: - client = module.client( - "opensearch", retry_decorator=AWSRetry.jittered_backoff() - ) + client = module.client("opensearch", retry_decorator=AWSRetry.jittered_backoff()) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws(e, msg="Failed to connect to AWS") + module.fail_json_aws(e, msg="Failed to connect to AWS opensearch service") if module.params["state"] == "absent": ret_dict = ensure_domain_absent(client, module) diff --git a/plugins/modules/opensearch_info.py b/plugins/modules/opensearch_info.py new file mode 100644 index 00000000000..6a884fdb076 --- /dev/null +++ b/plugins/modules/opensearch_info.py @@ -0,0 +1,530 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright: Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +DOCUMENTATION = """ +--- +module: opensearch_info +short_description: obtain information about one or more OpenSearch or ElasticSearch domain. +description: + - obtain information about one Amazon OpenSearch Service domain. +version_added: 3.1.0 +author: "Sebastien Rosset (@sebastien-rosset)" +options: + domain_name: + description: + - The name of the Amazon OpenSearch/ElasticSearch Service domain. + required: false + type: str + tags: + description: + - > + A dict of tags that are used to filter OpenSearch domains that match + all tag key, value pairs. + required: false + type: dict +requirements: +- botocore >= 1.21.38 +extends_documentation_fragment: +- amazon.aws.aws +- amazon.aws.ec2 +""" + +EXAMPLES = ''' +- name: Get information about an OpenSearch domain instance + community.aws.opensearch_info: + domain-name: my-search-cluster + register: new_cluster_info + +- name: Get all OpenSearch instances + community.aws.opensearch_info: + +- name: Get all OpenSearch instances that have the specified Key, Value tags + community.aws.opensearch_info: + tags: + Applications: search + Environment: Development +''' + +RETURN = ''' +instances: + description: List of OpenSearch domain instances + returned: always + type: complex + contains: + domain_status: + description: The current status of the OpenSearch domain. + returned: always + type: complex + contains: + arn: + description: The ARN of the OpenSearch domain. + returned: always + type: str + domain_id: + description: The unique identifier for the OpenSearch domain. + returned: always + type: str + domain_name: + description: The name of the OpenSearch domain. + returned: always + type: str + created: + description: + - > + The domain creation status. True if the creation of a domain is complete. + False if domain creation is still in progress. + returned: always + type: bool + deleted: + description: + - > + The domain deletion status. + True if a delete request has been received for the domain but resource cleanup is still in progress. + False if the domain has not been deleted. + Once domain deletion is complete, the status of the domain is no longer returned. + returned: always + type: bool + endpoint: + description: The domain endpoint that you use to submit index and search requests. + returned: always + type: str + endpoints: + description: + - > + Map containing the domain endpoints used to submit index and search requests. + - > + When you create a domain attached to a VPC domain, this propery contains + the DNS endpoint to which service requests are submitted. + - > + If you query the opensearch_info immediately after creating the OpenSearch cluster, + the VPC endpoint may not be returned. It may take several minutes until the + endpoints is available. + type: dict + processing: + description: + - > + The status of the domain configuration. + True if Amazon OpenSearch Service is processing configuration changes. + False if the configuration is active. + returned: always + type: bool + upgrade_processing: + description: true if a domain upgrade operation is in progress. + returned: always + type: bool + engine_version: + description: The version of the OpenSearch domain. + returned: always + type: str + sample: OpenSearch_1.1 + cluster_config: + description: + - Parameters for the cluster configuration of an OpenSearch Service domain. + type: complex + contains: + instance_type: + description: + - Type of the instances to use for the domain. + type: str + instance_count: + description: + - Number of instances for the domain. + type: int + zone_awareness: + description: + - A boolean value to indicate whether zone awareness is enabled. + type: bool + availability_zone_count: + description: + - > + An integer value to indicate the number of availability zones for a domain when zone awareness is enabled. + This should be equal to number of subnets if VPC endpoints is enabled. + type: int + dedicated_master_enabled: + description: + - A boolean value to indicate whether a dedicated master node is enabled. + type: bool + zone_awareness_enabled: + description: + - A boolean value to indicate whether zone awareness is enabled. + type: bool + zone_awareness_config: + description: + - The zone awareness configuration for a domain when zone awareness is enabled. + type: complex + contains: + availability_zone_count: + description: + - An integer value to indicate the number of availability zones for a domain when zone awareness is enabled. + type: int + dedicated_master_type: + description: + - The instance type for a dedicated master node. + type: str + dedicated_master_count: + description: + - Total number of dedicated master nodes, active and on standby, for the domain. + type: int + warm_enabled: + description: + - True to enable UltraWarm storage. + type: bool + warm_type: + description: + - The instance type for the OpenSearch domain's warm nodes. + type: str + warm_count: + description: + - The number of UltraWarm nodes in the domain. + type: int + cold_storage_options: + description: + - Specifies the ColdStorageOptions config for a Domain. + type: complex + contains: + enabled: + description: + - True to enable cold storage. Supported on Elasticsearch 7.9 or above. + type: bool + ebs_options: + description: + - Parameters to configure EBS-based storage for an OpenSearch Service domain. + type: complex + contains: + ebs_enabled: + description: + - Specifies whether EBS-based storage is enabled. + type: bool + volume_type: + description: + - Specifies the volume type for EBS-based storage. "standard"|"gp2"|"io1" + type: str + volume_size: + description: + - Integer to specify the size of an EBS volume. + type: int + iops: + description: + - The IOPD for a Provisioned IOPS EBS volume (SSD). + type: int + vpc_options: + description: + - Options to specify the subnets and security groups for a VPC endpoint. + type: complex + contains: + vpc_id: + description: The VPC ID for the domain. + type: str + subnet_ids: + description: + - Specifies the subnet ids for VPC endpoint. + type: list + elements: str + security_group_ids: + description: + - Specifies the security group ids for VPC endpoint. + type: list + elements: str + availability_zones: + description: + - The Availability Zones for the domain.. + type: list + elements: str + snapshot_options: + description: + - Option to set time, in UTC format, of the daily automated snapshot. + type: complex + contains: + automated_snapshot_start_hour: + description: + - > + Integer value from 0 to 23 specifying when the service takes a daily automated snapshot + of the specified Elasticsearch domain. + type: int + access_policies: + description: + - IAM access policy as a JSON-formatted string. + type: complex + encryption_at_rest_options: + description: + - Parameters to enable encryption at rest. + type: complex + contains: + enabled: + description: + - Should data be encrypted while at rest. + type: bool + kms_key_id: + description: + - If encryption at rest enabled, this identifies the encryption key to use. + - The value should be a KMS key ARN. It can also be the KMS key id. + type: str + node_to_node_encryption_options: + description: + - Node-to-node encryption options. + type: complex + contains: + enabled: + description: + - True to enable node-to-node encryption. + type: bool + cognito_options: + description: + - Parameters to configure OpenSearch Service to use Amazon Cognito authentication for OpenSearch Dashboards. + type: complex + contains: + enabled: + description: + - The option to enable Cognito for OpenSearch Dashboards authentication. + type: bool + user_pool_id: + description: + - The Cognito user pool ID for OpenSearch Dashboards authentication. + type: str + identity_pool_id: + description: + - The Cognito identity pool ID for OpenSearch Dashboards authentication. + type: str + role_arn: + description: + - The role ARN that provides OpenSearch permissions for accessing Cognito resources. + type: str + domain_endpoint_options: + description: + - Options to specify configuration that will be applied to the domain endpoint. + type: complex + contains: + enforce_https: + description: + - Whether only HTTPS endpoint should be enabled for the domain. + type: bool + tls_security_policy: + description: + - Specify the TLS security policy to apply to the HTTPS endpoint of the domain. + type: str + custom_endpoint_enabled: + description: + - Whether to enable a custom endpoint for the domain. + type: bool + custom_endpoint: + description: + - The fully qualified domain for your custom endpoint. + type: str + custom_endpoint_certificate_arn: + description: + - The ACM certificate ARN for your custom endpoint. + type: str + advanced_security_options: + description: + - Specifies advanced security options. + type: complex + contains: + enabled: + description: + - True if advanced security is enabled. + - You must enable node-to-node encryption to use advanced security options. + type: bool + internal_user_database_enabled: + description: + - True if the internal user database is enabled. + type: bool + master_user_options: + description: + - Credentials for the master user, username and password, ARN, or both. + type: complex + contains: + master_user_arn: + description: + - ARN for the master user (if IAM is enabled). + type: str + master_user_name: + description: + - The username of the master user, which is stored in the Amazon OpenSearch Service domain internal database. + type: str + master_user_password: + description: + - The password of the master user, which is stored in the Amazon OpenSearch Service domain internal database. + type: str + saml_options: + description: + - The SAML application configuration for the domain. + type: complex + contains: + enabled: + description: + - True if SAML is enabled. + type: bool + idp: + description: + - The SAML Identity Provider's information. + type: complex + contains: + metadata_content: + description: + - The metadata of the SAML application in XML format. + type: str + entity_id: + description: + - The unique entity ID of the application in SAML identity provider. + type: str + master_user_name: + description: + - The SAML master username, which is stored in the Amazon OpenSearch Service domain internal database. + type: str + master_backend_role: + description: + - The backend role that the SAML master user is mapped to. + type: str + subject_key: + description: + - Element of the SAML assertion to use for username. Default is NameID. + type: str + roles_key: + description: + - Element of the SAML assertion to use for backend roles. Default is roles. + type: str + session_timeout_minutes: + description: + - > + The duration, in minutes, after which a user session becomes inactive. + Acceptable values are between 1 and 1440, and the default value is 60. + type: int + auto_tune_options: + description: + - Specifies Auto-Tune options. + type: complex + contains: + desired_state: + description: + - The Auto-Tune desired state. Valid values are ENABLED and DISABLED. + type: str + maintenance_schedules: + description: + - A list of maintenance schedules. + type: list + elements: dict + contains: + start_at: + description: + - The timestamp at which the Auto-Tune maintenance schedule starts. + type: str + duration: + description: + - Specifies maintenance schedule duration, duration value and duration unit. + type: complex + contains: + value: + description: + - Integer to specify the value of a maintenance schedule duration. + type: int + unit: + description: + - The unit of a maintenance schedule duration. Valid value is HOURS. + type: str + cron_expression_for_recurrence: + description: + - A cron expression for a recurring maintenance schedule. + type: str + domain_config: + description: The OpenSearch domain configuration + returned: always + type: complex + contains: + domain_name: + description: The name of the OpenSearch domain. + returned: always + type: str +''' + + +try: + import botocore +except ImportError: + pass # handled by AnsibleAWSModule + +from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ( + AWSRetry, + boto3_tag_list_to_ansible_dict, + camel_dict_to_snake_dict, +) +from ansible_collections.community.aws.plugins.module_utils.opensearch import ( + get_domain_config, + get_domain_status, +) + + +def domain_info(client, module): + domain_name = module.params.get('domain_name') + filter_tags = module.params.get('tags') + + domain_list = [] + if domain_name: + domain_status = get_domain_status(client, module, domain_name) + if domain_status: + domain_list.append({'DomainStatus': domain_status}) + else: + domain_summary_list = client.list_domain_names()['DomainNames'] + for d in domain_summary_list: + domain_status = get_domain_status(client, module, d['DomainName']) + if domain_status: + domain_list.append({'DomainStatus': domain_status}) + + # Get the domain tags + for domain in domain_list: + current_domain_tags = None + domain_arn = domain['DomainStatus']['ARN'] + try: + current_domain_tags = client.list_tags(ARN=domain_arn, aws_retry=True)["TagList"] + domain['Tags'] = boto3_tag_list_to_ansible_dict(current_domain_tags) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + # This could potentially happen if a domain is deleted between the time + # its domain status was queried and the tags were queried. + domain['Tags'] = {} + + # Filter by tags + if filter_tags: + for tag_key in filter_tags: + try: + domain_list = [c for c in domain_list if ('Tags' in c) and (tag_key in c['Tags']) and (c['Tags'][tag_key] == filter_tags[tag_key])] + except (TypeError, AttributeError) as e: + module.fail_json(msg="OpenSearch tag filtering error", exception=e) + + # Get the domain config + for idx, domain in enumerate(domain_list): + domain_name = domain['DomainStatus']['DomainName'] + (domain_config, arn) = get_domain_config(client, module, domain_name) + if domain_config: + domain['DomainConfig'] = domain_config + domain_list[idx] = camel_dict_to_snake_dict(domain, + ignore_list=['AdvancedOptions', 'Endpoints', 'Tags']) + + return dict(changed=False, domains=domain_list) + + +def main(): + module = AnsibleAWSModule( + argument_spec=dict( + domain_name=dict(required=False), + tags=dict(type='dict', required=False), + ), + supports_check_mode=True, + ) + module.require_botocore_at_least("1.21.38") + + try: + client = module.client("opensearch", retry_decorator=AWSRetry.jittered_backoff()) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Failed to connect to AWS opensearch service") + + module.exit_json(**domain_info(client, module)) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/wafv2_web_acl_info.py b/plugins/modules/wafv2_web_acl_info.py index 3fb91fbd802..d8f4fdf091b 100644 --- a/plugins/modules/wafv2_web_acl_info.py +++ b/plugins/modules/wafv2_web_acl_info.py @@ -28,8 +28,8 @@ type: str extends_documentation_fragment: - - amazon.aws.aws - - amazon.aws.ec2 +- amazon.aws.aws +- amazon.aws.ec2 ''' diff --git a/tests/integration/targets/lambda/tasks/main.yml b/tests/integration/targets/lambda/tasks/main.yml index a7a1680d9e6..c851bfef705 100644 --- a/tests/integration/targets/lambda/tasks/main.yml +++ b/tests/integration/targets/lambda/tasks/main.yml @@ -12,7 +12,7 @@ - name: create minimal lambda role iam_role: name: '{{ lambda_role_name }}' - assume_role_policy_document: '{{ lookup("file", "minimal_trust_policy.json") }}' + assume_role_policy_document: '{{ lookup("file", "opensearch_policy.json") }}' create_instance_profile: false managed_policies: - 'arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess' diff --git a/tests/integration/targets/lambda_alias/tasks/main.yml b/tests/integration/targets/lambda_alias/tasks/main.yml index 9b264f50c0a..608e2175d70 100644 --- a/tests/integration/targets/lambda_alias/tasks/main.yml +++ b/tests/integration/targets/lambda_alias/tasks/main.yml @@ -13,7 +13,7 @@ - name: create minimal lambda role iam_role: name: '{{ lambda_role_name }}' - assume_role_policy_document: '{{ lookup("file", "minimal_trust_policy.json") }}' + assume_role_policy_document: '{{ lookup("file", "opensearch_policy.json") }}' create_instance_profile: false managed_policies: - 'arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess' diff --git a/tests/integration/targets/lambda_policy/tasks/main.yml b/tests/integration/targets/lambda_policy/tasks/main.yml index e0b514bde43..c62cca32c1e 100644 --- a/tests/integration/targets/lambda_policy/tasks/main.yml +++ b/tests/integration/targets/lambda_policy/tasks/main.yml @@ -13,7 +13,7 @@ - name: create minimal lambda role iam_role: name: '{{ lambda_role_name }}' - assume_role_policy_document: '{{ lookup("file", "minimal_trust_policy.json") }}' + assume_role_policy_document: '{{ lookup("file", "opensearch_policy.json") }}' create_instance_profile: false managed_policies: - 'arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess' @@ -51,7 +51,7 @@ - name: create minimal lambda role iam_role: name: ansible_lambda_role - assume_role_policy_document: '{{ lookup(''file'', ''minimal_trust_policy.json'', convert_data=False) }}' + assume_role_policy_document: '{{ lookup(''file'', ''opensearch_policy.json'', convert_data=False) }}' create_instance_profile: false register: iam_role - name: wait 10 seconds for role to become available diff --git a/tests/integration/targets/opensearch/aliases b/tests/integration/targets/opensearch/aliases index 22267bad72e..80e3d995cb7 100644 --- a/tests/integration/targets/opensearch/aliases +++ b/tests/integration/targets/opensearch/aliases @@ -1,5 +1,24 @@ # reason: slow -# Tests take around 30 minutes +# +# The duration of the tasks below was obtained from a single test job. +# There may be significant variations across runs. +# +# OpenSearch Cluster attached to Internet: +# 13 minutes to create ES cluster. Task is not waiting for cluster to become available. +# OpenSearch Cluster attached to VPC: +# 17 minutes to create ES cluster with 2 nodes and dedicated masters. +# 12 minutes to increase size of EBS volumes. +# 12 minutes to increase the node count from 2 to 4. +# 7 minutes to reduce the node count from 4 to 2. +# 35 minutes to upgrade cluster from 7.1 to 7.10. +# 23 minutes to enable node-to-node encryption. +# 36 minutes to enable encryption at rest. +# 30 minutes to enable warm storage. +# 30 minutes to enable cold storage. +# 30 minutes to enable and enforce HTTPS on the domain endpoint. +# 3 minutes to enable auto-tune option. +# 45 minutes to delete cluster. + #disabled cloud/aws diff --git a/tests/integration/targets/opensearch/files/opensearch_policy.json b/tests/integration/targets/opensearch/files/opensearch_policy.json new file mode 100644 index 00000000000..eb8b4bc291b --- /dev/null +++ b/tests/integration/targets/opensearch/files/opensearch_policy.json @@ -0,0 +1,16 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "opensearch.amazonaws.com" + }, + "Action": [ + "es:*", + "kms:List*", + "kms:Describe*" + ] + } + ] +} diff --git a/tests/integration/targets/opensearch/meta/main.yml b/tests/integration/targets/opensearch/meta/main.yml index 1f64f1169a9..13d6ecd9144 100644 --- a/tests/integration/targets/opensearch/meta/main.yml +++ b/tests/integration/targets/opensearch/meta/main.yml @@ -1,3 +1,4 @@ dependencies: - - prepare_tests - - setup_ec2 + - role: setup_botocore_pip + vars: + botocore_version: "1.21.38" diff --git a/tests/integration/targets/opensearch/tasks/main.yml b/tests/integration/targets/opensearch/tasks/main.yml index be582776377..6d3b47cad6f 100644 --- a/tests/integration/targets/opensearch/tasks/main.yml +++ b/tests/integration/targets/opensearch/tasks/main.yml @@ -1,23 +1,30 @@ --- # tasks file for test_opensearch +- name: Run opensearch integration tests. - # ============================================================ - -- name: Test incomplete arguments with opensearch + module_defaults: + group/aws: + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" + security_token: "{{ security_token | default(omit) }}" + region: "{{ aws_region }}" + route53: + # Route53 is explicitly a global service + region: null + collections: + - amazon.aws + vars: + ansible_python_interpreter: "{{ botocore_virtualenv_interpreter }}" block: - - # ============================================================ - - - name: test without specifying required module options - ec2_asg: - aws_access_key: "{{ aws_access_key }}" - aws_secret_key: "{{ aws_secret_key }}" - security_token: "{{ security_token | default(omit) }}" - ignore_errors: yes - register: result - - - name: assert name is a required module option - assert: - that: - - "result.msg == 'missing required arguments: name'" \ No newline at end of file + # Get some information about who we are before starting our tests + # we'll need this as soon as we start working on the policies + - name: get ARN of calling user + aws_caller_info: + register: aws_caller_info + - include_tasks: test_delete_resources.yml + - include_tasks: test_create_cert.yml + - include_tasks: test_vpc_setup.yml + - include_tasks: test_opensearch.yml + always: + - include_tasks: test_delete_resources.yml diff --git a/tests/integration/targets/opensearch/tasks/test_create_cert.yml b/tests/integration/targets/opensearch/tasks/test_create_cert.yml new file mode 100644 index 00000000000..533e75e9653 --- /dev/null +++ b/tests/integration/targets/opensearch/tasks/test_create_cert.yml @@ -0,0 +1,53 @@ +- pip: + name: + # The 'cryptography' module is required by community.crypto.openssl_privatekey + - 'cryptography' + virtualenv: "{{ botocore_virtualenv }}" + virtualenv_command: "{{ botocore_virtualenv_command }}" + virtualenv_site_packages: no +- name: Create temporary directory + ansible.builtin.tempfile: + state: directory + suffix: build + register: tempdir_1 +- name: Generate private key + community.crypto.openssl_privatekey: + path: '{{ tempdir_1.path }}/rsa-private-key.pem' + type: RSA + size: 2048 +- name: Generate an OpenSSL Certificate Signing Request for own certs + community.crypto.openssl_csr: + path: '{{ tempdir_1.path }}/rsa-csr.pem' + privatekey_path: '{{ tempdir_1.path }}/rsa-private-key.pem' + common_name: 'opensearch.ansible-integ-test.com' +- name: Generate a Self Signed certificate + community.crypto.x509_certificate: + provider: selfsigned + path: '{{ tempdir_1.path }}/rsa-certificate.pem' + csr_path: '{{ tempdir_1.path }}/rsa-csr.pem' + privatekey_path: '{{ tempdir_1.path }}/rsa-private-key.pem' + selfsigned_digest: sha256 +- name: import certificate to ACM + aws_acm: + name_tag: 'opensearch.ansible-integ-test.com' + domain_name: 'opensearch.ansible-integ-test.com' + certificate: "{{ lookup('file', tempdir_1.path + '/rsa-certificate.pem') }}" + private_key: "{{ lookup('file', tempdir_1.path + '/rsa-private-key.pem') }}" + state: present + # tags: + # Application: search + # Environment: development + # purge_tags: false + register: upload_cert +- assert: + that: + - upload_cert.certificate.arn is defined + - upload_cert.certificate.domain_name == 'opensearch.ansible-integ-test.com' + - upload_cert.changed + +- set_fact: + opensearch_certificate_arn: "{{ upload_cert.certificate.arn }}" +- name: Delete temporary directory + ansible.builtin.file: + state: absent + path: "{{ tempdir_1.path }}" \ No newline at end of file diff --git a/tests/integration/targets/opensearch/tasks/test_delete_resources.yml b/tests/integration/targets/opensearch/tasks/test_delete_resources.yml new file mode 100644 index 00000000000..d9ddfc91347 --- /dev/null +++ b/tests/integration/targets/opensearch/tasks/test_delete_resources.yml @@ -0,0 +1,61 @@ +- name: Delete all resources that were created in the integration tests + block: + - name: Get list of OpenSearch domains + opensearch_info: + tags: + Environment: "Testing" + Application: "Search" + AnsibleTest: "AnsibleTestOpenSearchCluster" + register: opensearch_domains + + - name: Initiate deletion of all test-related OpenSearch clusters + opensearch: + state: absent + domain_name: "{{ domain_name }}" + with_items: "{{ opensearch_domains.domains }}" + vars: + domain_name: "{{ item.domain_status.domain_name }}" + + # We have to wait until the cluster is deleted before proceeding to delete + # security group, VPC, KMS. Otherwise there will be active references and + # deletion of the security group will fail. + - name: Delete OpenSearch clusters, wait until deletion is complete + opensearch: + state: absent + domain_name: "{{ domain_name }}" + wait: true + wait_timeout: "{{ 60 * 60 }}" + with_items: "{{ opensearch_domains.domains }}" + vars: + domain_name: "{{ item.domain_status.domain_name }}" + + - name: Get VPC info + ec2_vpc_net_info: + filters: + "tag:AnsibleTest": "AnsibleTestVpc" + register: vpc_info + + - name: delete VPC resources + include_tasks: test_delete_vpc_resources.yml + with_items: "{{ vpc_info.vpcs }}" + vars: + vpc_id: "{{ item.vpc_id }}" + vpc_name: "{{ item.tags['Name'] }}" + + - name: collect info about KMS keys used for test purpose + aws_kms_info: + filters: + "tag:AnsibleTest": "AnsibleTestVpc" + register: kms_info + - name: Delete KMS keys that were created for test purpose + aws_kms: + key_id: "{{ kms_arn }}" + state: absent + with_items: "{{ kms_info.kms_keys }}" + vars: + kms_arn: "{{ item.key_arn }}" + + - name: delete certificate from ACM + aws_acm: + name_tag: 'opensearch.ansible-integ-test.com' + state: absent diff --git a/tests/integration/targets/opensearch/tasks/test_delete_vpc_resources.yml b/tests/integration/targets/opensearch/tasks/test_delete_vpc_resources.yml new file mode 100644 index 00000000000..5fb803c9095 --- /dev/null +++ b/tests/integration/targets/opensearch/tasks/test_delete_vpc_resources.yml @@ -0,0 +1,92 @@ +- debug: + msg: "Deleting resources in VPC name: {{ vpc_name }}, id: {{ vpc_id }}" + +- name: Get the hosted Route53 zones + route53_info: + query: hosted_zone + hosted_zone_method: list + register: route53_zone_info + +- name: Get Route53 zone id + set_fact: + route53_zone_ids: "{{ route53_zone_info.HostedZones | selectattr('Name', 'equalto', 'ansible-integ-test.com.') | map(attribute='Id') | list }}" + +- name: Delete Route53 record + route53: + record: "opensearch.ansible-integ-test.com" + hosted_zone_id: "{{ route53_zone_ids[0] }}" + private_zone: true + type: CNAME + state: absent + vpc_id: '{{ vpc_id }}' + when: route53_zone_ids | length > 0 + +- name: Delete private Route53 zone for the VPC + route53_zone: + zone: "ansible-integ-test.com" + hosted_zone_id: "{{ route53_zone_ids[0] }}" + state: absent + vpc_id: '{{ vpc_id }}' + when: route53_zone_ids | length > 0 + +- name: Get security groups that have been created for test purpose in the VPC + ec2_group_info: + filters: + vpc-id: "{{ vpc_id }}" + register: sg_info + +- name: Delete security groups + ec2_group: + group_id: "{{ sg_id }}" + state: absent + loop_control: + loop_var: sg_item + with_items: "{{ sg_info.security_groups }}" + vars: + sg_id: "{{ sg_item.group_id }}" + sg_name: "{{ sg_item.group_name }}" + when: sg_name != 'default' + +- name: Delete internet gateway + ec2_vpc_igw: + vpc_id: "{{ vpc_id }}" + state: absent + +- name: Delete subnet_1 + ec2_vpc_subnet: + state: absent + vpc_id: "{{ vpc_id }}" + cidr: 10.55.77.0/24 + +- name: Delete subnet_2 + ec2_vpc_subnet: + state: absent + vpc_id: "{{ vpc_id }}" + cidr: 10.55.78.0/24 + +- name: Collect info about routing tables + ec2_vpc_route_table_info: + filters: + vpc-id: "{{ vpc_id }}" + # Exclude the main route table, which should not be deleted explicitly. + # It will be deleted automatically when the VPC is deleted + "tag:AnsibleTest": "AnsibleTestVpc" + register: routing_table_info + +- name: Delete routing tables + ec2_vpc_route_table: + state: absent + lookup: id + route_table_id: "{{ route_table_id }}" + loop_control: + loop_var: route_item + with_items: "{{ routing_table_info.route_tables }}" + vars: + route_table_id: "{{ route_item.id }}" + +- name: Delete VPC for use in testing + ec2_vpc_net: + name: "{{ vpc_name }}" + cidr_block: 10.55.0.0/16 + purge_cidrs: true + state: absent diff --git a/tests/integration/targets/opensearch/tasks/test_opensearch.yml b/tests/integration/targets/opensearch/tasks/test_opensearch.yml new file mode 100644 index 00000000000..1418e6d8016 --- /dev/null +++ b/tests/integration/targets/opensearch/tasks/test_opensearch.yml @@ -0,0 +1,1252 @@ +- name: Create OpenSearch clusters + block: + - name: test without specifying required module options + opensearch: + engine_version: "Elasticsearch_7.1" + ignore_errors: yes + register: result + + - name: assert domain_name is a required module option + assert: + that: + - "result.msg == 'missing required arguments: domain_name'" + + - name: Create public-facing OpenSearch cluster in check mode + opensearch: + state: present + domain_name: "es-{{ tiny_prefix }}-public" + engine_version: "OpenSearch_1.1" + cluster_config: + instance_type: "t2.small.search" + instance_count: 2 + dedicated_master: false + access_policies: "{{ lookup('file', 'opensearch_policy.json') | from_json }}" + ebs_options: + ebs_enabled: true + volume_size: 10 + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + check_mode: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain is changed" + + - name: Create public-facing OpenSearch cluster, changes expected + # This could take 30 minutes + opensearch: + state: present + # Note domain_name must be less than 28 characters and satisfy regex [a-z][a-z0-9\\-]+ + domain_name: "es-{{ tiny_prefix }}-public" + engine_version: "OpenSearch_1.1" + cluster_config: + instance_type: "t2.small.search" + instance_count: 2 + dedicated_master: false + access_policies: "{{ lookup('file', 'opensearch_policy.json') | from_json }}" + ebs_options: + # EBS must be enabled for "t2.small.search" + ebs_enabled: true + volume_size: 10 + tags: + # Note: The domain name must start with a letter, but the tiny prefix may start with + # a digit. + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + # Intentionally not waiting for cluster to be available. + # All other integration tests are performed with the + # OpenSearch cluster attached to the VPC. + wait: false + register: opensearch_domain + - assert: + that: + - "opensearch_domain.tags | length == 4" + - "opensearch_domain.engine_version == 'OpenSearch_1.1'" + - "opensearch_domain.cluster_config.instance_count == 2" + - "opensearch_domain.cluster_config.instance_type == 't2.small.search'" + - "opensearch_domain.cluster_config.dedicated_master_enabled == false" + - "opensearch_domain.cluster_config.warm_enabled == false" + - "opensearch_domain.ebs_options.ebs_enabled == true" + - "opensearch_domain.ebs_options.volume_size == 10" + # Below should be default settings when not specified as input arguments + - "opensearch_domain.ebs_options.volume_type == 'gp2'" + - "opensearch_domain.advanced_security_options.enabled == false" + - "opensearch_domain.cognito_options.enabled == false" + - "opensearch_domain.domain_endpoint_options.custom_endpoint_enabled == false" + - "opensearch_domain.encryption_at_rest_options.enabled == false" + - "opensearch_domain.node_to_node_encryption_options.enabled == false" + - "opensearch_domain.snapshot_options.automated_snapshot_start_hour == 0" + # Assert task has changed/not changed OpenSearch domain + - "opensearch_domain is changed" + + - name: Create public-facing OpenSearch cluster in check mode again + opensearch: + state: present + domain_name: "es-{{ tiny_prefix }}-public" + engine_version: "OpenSearch_1.1" + cluster_config: + instance_type: "t2.small.search" + instance_count: 2 + dedicated_master: false + access_policies: "{{ lookup('file', 'opensearch_policy.json') | from_json }}" + ebs_options: + ebs_enabled: true + volume_size: 10 + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Create public-facing OpenSearch cluster, no change expected + opensearch: + state: present + domain_name: "es-{{ tiny_prefix }}-public" + engine_version: "OpenSearch_1.1" + cluster_config: + instance_type: "t2.small.search" + instance_count: 2 + dedicated_master: false + access_policies: "{{ lookup('file', 'opensearch_policy.json') | from_json }}" + ebs_options: + ebs_enabled: true + volume_size: 10 + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Create VPC-attached OpenSearch cluster, check mode + opensearch: + state: present + domain_name: "es-{{ tiny_prefix }}-vpc" + engine_version: "Elasticsearch_7.1" + cluster_config: + instance_type: "m5.large.search" + instance_count: 2 + zone_awareness: true + availability_zone_count: 2 + dedicated_master: true + dedicated_master_instance_type: "m5.large.search" + dedicated_master_instance_count: 3 + ebs_options: + ebs_enabled: true + volume_type: "gp2" + volume_size: 10 + snapshot_options: + automated_snapshot_start_hour: 12 + access_policies: "{{ lookup('file', 'opensearch_policy.json') | from_json }}" + vpc_options: + subnets: + - "{{ testing_subnet_1.subnet.id }}" + - "{{ testing_subnet_2.subnet.id }}" + security_groups: + - "{{ sg.group_id }}" + encryption_at_rest_options: + enabled: false + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain is changed" + + - name: Create VPC-attached OpenSearch cluster, change expected + # Considerations for selecting node instances for test purpose: + # 1) Low cost + # 2) Supports encryption at rest, advanced security options, node-to-node encryption. + # See https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html + opensearch: + state: present + domain_name: "es-{{ tiny_prefix }}-vpc" + engine_version: "Elasticsearch_7.1" + cluster_config: + instance_type: "m5.large.search" + instance_count: 2 + zone_awareness: true + availability_zone_count: 2 + dedicated_master: true + dedicated_master_instance_type: "m5.large.search" + dedicated_master_instance_count: 3 + ebs_options: + ebs_enabled: true + volume_type: "gp2" + volume_size: 10 + snapshot_options: + automated_snapshot_start_hour: 12 + access_policies: "{{ lookup('file', 'opensearch_policy.json') | from_json }}" + vpc_options: + subnets: + - "{{ testing_subnet_1.subnet.id }}" + - "{{ testing_subnet_2.subnet.id }}" + security_groups: + - "{{ sg.group_id }}" + encryption_at_rest_options: + enabled: false + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + wait: true + # It could take about 45 minutes for the cluster to be created and available. + wait_timeout: "{{ 45 * 60 }}" + register: opensearch_domain + - assert: + that: + - "opensearch_domain.tags | length == 4" + - "opensearch_domain.engine_version == 'Elasticsearch_7.1'" + - "opensearch_domain.cluster_config.instance_count == 2" + - "opensearch_domain.cluster_config.instance_type == 'm5.large.search'" + - "opensearch_domain.cluster_config.dedicated_master_enabled == true" + - "opensearch_domain.cluster_config.dedicated_master_type == 'm5.large.search'" + - "opensearch_domain.cluster_config.dedicated_master_count == 3" + - "opensearch_domain.cluster_config.warm_enabled == false" + - "opensearch_domain.cluster_config.zone_awareness_enabled == true" + - "opensearch_domain.ebs_options.ebs_enabled == true" + - "opensearch_domain.ebs_options.volume_size == 10" + - "opensearch_domain.ebs_options.volume_type == 'gp2'" + - "opensearch_domain.snapshot_options.automated_snapshot_start_hour == 12" + - "opensearch_domain.vpc_options is defined" + - "opensearch_domain.vpc_options.vpc_id is defined" + - "opensearch_domain.vpc_options.vpc_id == testing_vpc.vpc.id" + - "opensearch_domain.vpc_options.subnet_ids is defined" + - "opensearch_domain.vpc_options.subnet_ids | length == 2" + - "opensearch_domain.vpc_options.security_group_ids is defined" + - "opensearch_domain.vpc_options.security_group_ids | length == 1" + # Assert task has changed/not changed OpenSearch domain + - "opensearch_domain is changed" + + - name: Create VPC-attached OpenSearch cluster, check mode again + opensearch: + state: present + domain_name: "es-{{ tiny_prefix }}-vpc" + engine_version: "Elasticsearch_7.1" + cluster_config: + instance_type: "m5.large.search" + instance_count: 2 + zone_awareness: true + availability_zone_count: 2 + dedicated_master: true + dedicated_master_instance_type: "m5.large.search" + dedicated_master_instance_count: 3 + ebs_options: + ebs_enabled: true + volume_type: "gp2" + volume_size: 10 + snapshot_options: + automated_snapshot_start_hour: 12 + access_policies: "{{ lookup('file', 'opensearch_policy.json') | from_json }}" + vpc_options: + subnets: + - "{{ testing_subnet_1.subnet.id }}" + - "{{ testing_subnet_2.subnet.id }}" + security_groups: + - "{{ sg.group_id }}" + encryption_at_rest_options: + enabled: false + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Create VPC-attached OpenSearch cluster, no change expected + opensearch: + state: present + domain_name: "es-{{ tiny_prefix }}-vpc" + engine_version: "Elasticsearch_7.1" + cluster_config: + instance_type: "m5.large.search" + instance_count: 2 + zone_awareness: true + availability_zone_count: 2 + dedicated_master: true + dedicated_master_instance_type: "m5.large.search" + dedicated_master_instance_count: 3 + ebs_options: + ebs_enabled: true + volume_type: "gp2" + volume_size: 10 + snapshot_options: + automated_snapshot_start_hour: 12 + access_policies: "{{ lookup('file', 'opensearch_policy.json') | from_json }}" + vpc_options: + subnets: + - "{{ testing_subnet_1.subnet.id }}" + - "{{ testing_subnet_2.subnet.id }}" + security_groups: + - "{{ sg.group_id }}" + encryption_at_rest_options: + enabled: false + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + wait: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Get list of OpenSearch domains + opensearch_info: + tags: + Environment: "Testing" + Application: "Search" + AnsibleTest: "AnsibleTestOpenSearchCluster" + register: opensearch_domains_info + - assert: + that: + - opensearch_domains_info.domains is defined + - opensearch_domains_info.domains | length == 2 + + - name: Get OpenSearch domains matching domain name + opensearch_info: + domain_name: "es-{{ tiny_prefix }}-vpc" + register: opensearch_domains_info + - assert: + that: + - opensearch_domains_info is not changed + - opensearch_domains_info.domains is defined + - opensearch_domains_info.domains | length == 1 + - opensearch_domains_info.domains[0].domain_config is defined + - opensearch_domains_info.domains[0].domain_status is defined + # Even after waiting for the OpenSearch cluster to complete installation, + # it may take a few additional minutes for the 'Endpoints' property + # to be present in the AWS API. + # See further down below. The same `opensearch_info` task is invoked + # after running a task that takes time, and that time there is + # an assert on the 'endpoint' property. + #- opensearch_domains_info.domains[0].domain_status.endpoints is defined + #- opensearch_domains_info.domains[0].domain_status.endpoints.vpc is defined + + - name: Tag Opensearch domain, check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + tag_a: 'value 1' + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Tag Opensearch domain, expect changes + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + tag_a: 'value 1' + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + wait: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain.tags | length == 5" + - opensearch_domain is changed + + - name: Tag Opensearch domain, check mode again + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + tag_a: 'value 1' + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Tag Opensearch domain, expect no change + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + tag_a: 'value 1' + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + wait: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain.tags | length == 5" + - opensearch_domain is not changed + + - name: Add tag_b to Opensearch domain without purging, check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + tags: + tag_b: 'value 2' + purge_tags: false + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Add tag_b to Opensearch domain without purging + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + tags: + tag_b: 'value 2' + purge_tags: false + wait: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain.tags | length == 6" + - opensearch_domain is changed + + - name: Add tag_b to Opensearch domain without purging, check mode again + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + tags: + tag_b: 'value 2' + purge_tags: false + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Add tag_b to Opensearch domain without purging, no change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + tags: + tag_b: 'value 2' + purge_tags: false + wait: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain.tags | length == 6" + - opensearch_domain is not changed + + - name: Set tags to Opensearch domain and purge tags, check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + purge_tags: true + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Set tags to Opensearch domain and purge tags + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + purge_tags: true + wait: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain.tags | length == 4" + - opensearch_domain is changed + + - name: Set tags to Opensearch domain and purge tags, check mode again + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + purge_tags: true + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Set tags to Opensearch domain and purge tags, no change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + tags: + Name: "es-{{ tiny_prefix }}-public" + Environment: Testing + Application: Search + AnsibleTest: "AnsibleTestOpenSearchCluster" + purge_tags: true + wait: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain.tags | length == 4" + - opensearch_domain is not changed + +- name: Change EBS storage configuration, snapshot hour, instance count + block: + - name: Increase size of EBS volumes, check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + ebs_options: + volume_size: 12 + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Increase size of EBS volumes, expect changes + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + ebs_options: + volume_size: 12 + wait: true + # It could take 20 minutes to increase the size of the EBS volumes + wait_timeout: "{{ 30 * 60 }}" + register: opensearch_domain + - assert: + that: + - "opensearch_domain.ebs_options.volume_size == 12" + - opensearch_domain is changed + + - name: Increase size of EBS volumes, check mode again + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + ebs_options: + volume_size: 12 + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Increase size of EBS volumes, expect no change + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + ebs_options: + volume_size: 12 + wait: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain.ebs_options.volume_size == 12" + - opensearch_domain is not changed + + - name: Get OpenSearch domains matching domain name + opensearch_info: + domain_name: "es-{{ tiny_prefix }}-vpc" + register: opensearch_domains_info + - assert: + that: + - opensearch_domains_info is not changed + - opensearch_domains_info.domains is defined + - opensearch_domains_info.domains | length == 1 + - opensearch_domains_info.domains[0].domain_config is defined + - opensearch_domains_info.domains[0].domain_status is defined + # Even after waiting for the OpenSearch cluster to complete installation, + # it may take a few additional minutes for the 'Endpoints' property + # to be present in the AWS API. + - opensearch_domains_info.domains[0].domain_status.endpoints is defined + - opensearch_domains_info.domains[0].domain_status.endpoints.vpc is defined + + - name: Set fact for OpenSearch endpoint FQDN in the VPC + set_fact: + opensearch_vpc_fqdn: "{{ opensearch_domains_info.domains[0].domain_status.endpoints.vpc }}" + + # Create a Route53 CNAME record. + # This CNAME record will be used when configuring a custom endpoint + # for the OpenSearch cluster. + - name: Create CNAME record for the OpenSearch custom endpoint + route53: + state: present + hosted_zone_id: "{{ route53_zone_id }}" + record: "opensearch.ansible-integ-test.com" + private_zone: true + type: CNAME + value: "{{ opensearch_vpc_fqdn }}" + + - name: Change snapshot start hour, check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + snapshot_options: + automated_snapshot_start_hour: 3 + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Change snapshot start hour + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + snapshot_options: + automated_snapshot_start_hour: 3 + wait: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain.snapshot_options.automated_snapshot_start_hour == 3" + - opensearch_domain is changed + + - name: Change snapshot start hour, check mode again + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + snapshot_options: + automated_snapshot_start_hour: 3 + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Change snapshot start hour, no change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + snapshot_options: + automated_snapshot_start_hour: 3 + wait: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Set instance count to 4 in the OpenSearch cluster, check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + # It's a 2-AZ deployment, therefore the node count must be even. + instance_count: 4 + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Set instance count to 4 in the OpenSearch cluster, changes expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + # It's a 2-AZ deployment, therefore the node count must be even. + instance_count: 4 + wait: true + # It could take 20 minutes to increase the number of nodes in the cluster + wait_timeout: "{{ 30 * 60 }}" + register: opensearch_domain + - assert: + that: + - "opensearch_domain.cluster_config.instance_count == 4" + - opensearch_domain is changed + + - name: Set instance count to 4 in the OpenSearch cluster, check mode again + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + # It's a 2-AZ deployment, therefore the node count must be even. + instance_count: 4 + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Set instance count to 4 in the OpenSearch cluster, no change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + # It's a 2-AZ deployment, therefore the node count must be even. + instance_count: 4 + wait: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Set instance count to 2 in the OpenSearch cluster, check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + instance_count: 2 + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Set instance count to 2 in the OpenSearch cluster, changes expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + instance_count: 2 + wait: true + # It could take 20 minutes to decrease the number of nodes in the cluster + wait_timeout: "{{ 30 * 60 }}" + register: opensearch_domain + - assert: + that: + - "opensearch_domain.cluster_config.instance_count == 2" + - opensearch_domain is changed + + - name: Set instance count to 2 in the OpenSearch cluster, check mode again + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + instance_count: 2 + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Set instance count to 2 in the OpenSearch cluster, no change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + instance_count: 2 + wait: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain.cluster_config.instance_count == 2" + - opensearch_domain is not changed + +- name: Upgrade OpenSearch cluster + block: + - name: Upgrade OpenSearch cluster, check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + engine_version: "Elasticsearch_7.10" + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Upgrade OpenSearch cluster, change expected + # Upgrade from version 7.1 to version 7.10. + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + engine_version: "Elasticsearch_7.10" + wait: true + # It could take 40 minutes to upgrade the cluster + wait_timeout: "{{ 50 * 60 }}" + register: opensearch_domain + - assert: + that: + - "opensearch_domain.engine_version == 'Elasticsearch_7.10'" + - "opensearch_domain.cluster_config.instance_count == 2" + - opensearch_domain is changed + + - name: Upgrade OpenSearch cluster, check mode again + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + engine_version: "Elasticsearch_7.10" + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Upgrade OpenSearch cluster, no change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + engine_version: "Elasticsearch_7.10" + register: opensearch_domain + - assert: + that: + - "opensearch_domain.engine_version == 'Elasticsearch_7.10'" + - "opensearch_domain.cluster_config.instance_count == 2" + - opensearch_domain is not changed + +- name: Configure encryption + when: false + block: + - name: Enable node to node encryption, check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + node_to_node_encryption_options: + enabled: true + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Enable node to node encryption, change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + node_to_node_encryption_options: + enabled: true + wait: true + # This may take a long time. + wait_timeout: "{{ 60 * 60 }}" + register: opensearch_domain + - assert: + that: + - "opensearch_domain.node_to_node_encryption_options.enabled == True" + - opensearch_domain is changed + + - name: Enable node to node encryption, check mode again + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + node_to_node_encryption_options: + enabled: true + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Enable node to node encryption, no change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + node_to_node_encryption_options: + enabled: true + wait: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain.node_to_node_encryption_options.enabled == True" + - opensearch_domain is not changed + + - name: Enable encryption at rest, check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + encryption_at_rest_options: + enabled: true + #kms_key_id: "{{ kms_test_key.key_id }}" + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Enable encryption at rest, change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + encryption_at_rest_options: + enabled: true + # key ARN has the format: + # 'arn:aws:kms:{{region}}:{{account}}:key/{{key_id}}' + # The API docs indicate the value is the KMS key id, which works + # However, the KMS key ARN seems to be a better option because + # the AWS API actually returns the KMS key ARN. + + # Do not set 'kms_key_id' to let AWS manage the key. + #kms_key_id: "{{ kms_test_key.key_arn }}" + wait: true + # It may take a long time for the task to complete. + wait_timeout: "{{ 60 * 60 }}" + register: opensearch_domain + - assert: + that: + - "opensearch_domain.encryption_at_rest_options.enabled == True" + - opensearch_domain is changed + + - name: Enable encryption at rest, check mode again + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + encryption_at_rest_options: + enabled: true + #kms_key_id: "{{ kms_test_key.key_arn }}" + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Enable encryption at rest, no change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + encryption_at_rest_options: + enabled: true + #kms_key_id: "{{ kms_test_key.key_arn }}" + wait: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + +- name: Configure HTTPs endpoint + when: false + block: + - name: Enforce HTTPS for OpenSearch endpoint, check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + domain_endpoint_options: + enforce_https: true + tls_security_policy: "Policy-Min-TLS-1-0-2019-07" + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Enforce HTTPS for OpenSearch endpoint, changes expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + domain_endpoint_options: + enforce_https: true + tls_security_policy: "Policy-Min-TLS-1-0-2019-07" + # Refer to CNAME that was defined in the previous tasks. + custom_endpoint_enabled: true + custom_endpoint: "opensearch.ansible-integ-test.com" + # TODO: create a certificate. There is a dependency on the aws_acm module + # which does not support certificates issued by AWS. + # The OpenSearch endpoint should have a certificate issued by a trusted CA + # otherwise clients to the OpenSearch cluster will not be able to validate + # the x.509 certificate of the OpenSearch endpoint. + custom_endpoint_certificate_arn: "{{ opensearch_certificate_arn }}" + wait: true + wait_timeout: "{{ 60 * 60 }}" + register: opensearch_domain + until: opensearch_domain is not failed + ignore_errors: yes + retries: 10 + # After enabling at rest encryption, there is a period during which the API fails, so retry. + delay: 30 + - assert: + that: + - "opensearch_domain.domain_endpoint_options.enforce_https == True" + - "opensearch_domain.domain_endpoint_options.tls_security_policy == 'Policy-Min-TLS-1-0-2019-07'" + #- "opensearch_domain.domain_endpoint_options.custom_endpoint_enabled == True" + - opensearch_domain is changed + + - name: Change TLS policy, check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + domain_endpoint_options: + tls_security_policy: "Policy-Min-TLS-1-2-2019-07" + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Change TLS policy, change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + domain_endpoint_options: + tls_security_policy: "Policy-Min-TLS-1-2-2019-07" + wait: true + wait_timeout: "{{ 60 * 60 }}" + register: opensearch_domain + - assert: + that: + - "opensearch_domain.domain_endpoint_options.enforce_https == True" + - "opensearch_domain.domain_endpoint_options.tls_security_policy == 'Policy-Min-TLS-1-2-2019-07'" + - opensearch_domain is changed + +- name: Configure advanced security + block: + - name: Enable advanced security, check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + advanced_security_options: + enabled: true + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + # Pre-requisites before enabling advanced security options: + # 1) node-to-node encryption must be enabled. + # 2) encryption at rest to must be enabled. + # 3) Enforce HTTPS in the domain endpoint options. + - name: Enable advanced security, change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + advanced_security_options: + enabled: true + wait: true + wait_timeout: "{{ 60 * 60 }}" + register: opensearch_domain + - assert: + that: + - "opensearch_domain.advanced_security_options.enabled == True" + - opensearch_domain is changed + + - name: Enable advanced security, check mode again + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + advanced_security_options: + enabled: true + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Enable advanced security, no change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + advanced_security_options: + enabled: true + wait: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain.advanced_security_options.enabled == True" + - opensearch_domain is not changed + +- name: Configure warm and cold storage + block: + - name: Enable warm storage in check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + warm_enabled: true + warm_type: "ultrawarm1.medium.search" + warm_count: 2 + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Enable warm storage, change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + warm_enabled: true + warm_type: "ultrawarm1.medium.search" + warm_count: 2 + wait: true + # Adding warm storage may take a long time. + wait_timeout: "{{ 45 * 60 }}" + register: opensearch_domain + - assert: + that: + - "opensearch_domain.cluster_config.warm_enabled == True" + - "opensearch_domain.cluster_config.warm_count == 2" + - "opensearch_domain.cluster_config.warm_type == 'ultrawarm1.medium.search'" + - opensearch_domain is changed + + - name: Enable warm storage in check mode again + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + warm_enabled: true + warm_type: "ultrawarm1.medium.search" + warm_count: 2 + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Enable warm storage, no change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + warm_enabled: true + warm_type: "ultrawarm1.medium.search" + warm_count: 2 + wait: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain.cluster_config.warm_enabled == True" + - "opensearch_domain.cluster_config.warm_count == 2" + - opensearch_domain is not changed + + - name: Enable cold storage in check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + cold_storage_options: + enabled: true + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Enable cold storage, change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + cold_storage_options: + enabled: true + wait: true + # Adding cold storage may take a long time. + wait_timeout: "{{ 45 * 60 }}" + register: opensearch_domain + - assert: + that: + - "opensearch_domain.cluster_config.cold_storage_options.enabled == True" + - opensearch_domain is changed + + - name: Enable cold storage in check mode again + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + cold_storage_options: + enabled: true + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Enable cold storage, no change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + cluster_config: + cold_storage_options: + enabled: true + wait: true + register: opensearch_domain + - assert: + that: + - "opensearch_domain.cluster_config.cold_storage_options.enabled == True" + - opensearch_domain is not changed + +- name: Configure auto-tune options + block: + - name: Enable auto-tune options in check mode + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + auto_tune_options: + desired_state: "ENABLED" + maintenance_schedules: + - start_at: "{{ ansible_date_time.year|int + 1 }}-01-01" + duration: + value: 10 + unit: "HOURS" + cron_expression_for_recurrence: "cron(0 12 * * ? *)" + wait: true + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is changed + + - name: Enable auto-tune options, changes expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + auto_tune_options: + desired_state: "ENABLED" + maintenance_schedules: + # Create a schedule one year from the date we start the test. + # If we hard code the date, we have to pick a date, but the API will fail + # if start_at is in the past. + - start_at: "{{ ansible_date_time.year|int + 1 }}-01-01" + duration: + value: 10 + unit: "HOURS" + cron_expression_for_recurrence: "cron(0 12 * * ? *)" + # I have observed that at least during one test run, + # the cluster status was stuck in 'processing=true' for over 24 hours + # after enabling auto-tune options. + # This does not seem right. From a test perspective, that means we cannot + # execute any change that needs to wait until the cluster is not processing + # background tasks. + wait: false + register: opensearch_domain + - assert: + that: + - "opensearch_domain.auto_tune_options.state == 'ENABLED'" + - opensearch_domain is changed + + - name: Enable auto-tune options in check mode again + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + auto_tune_options: + desired_state: "ENABLED" + maintenance_schedules: + - start_at: "{{ ansible_date_time.year|int + 1 }}-01-01" + duration: + value: 10 + unit: "HOURS" + cron_expression_for_recurrence: "cron(0 12 * * ? *)" + wait: false + check_mode: true + register: opensearch_domain + - assert: + that: + - opensearch_domain is not changed + + - name: Enable auto-tune options, no change expected + opensearch: + domain_name: "es-{{ tiny_prefix }}-vpc" + auto_tune_options: + desired_state: "ENABLED" + maintenance_schedules: + - start_at: "{{ ansible_date_time.year|int + 1 }}-01-01" + duration: + value: 10 + unit: "HOURS" + cron_expression_for_recurrence: "cron(0 12 * * ? *)" + wait: false + register: opensearch_domain + - assert: + that: + - "opensearch_domain.auto_tune_options.state == 'ENABLED'" + - opensearch_domain is not changed diff --git a/tests/integration/targets/opensearch/tasks/test_vpc_setup.yml b/tests/integration/targets/opensearch/tasks/test_vpc_setup.yml new file mode 100644 index 00000000000..90aeb50bbd6 --- /dev/null +++ b/tests/integration/targets/opensearch/tasks/test_vpc_setup.yml @@ -0,0 +1,134 @@ +- name: Configure pre-requisites for test, VPC and KMS resources + block: + - name: Create VPC for use in testing + ec2_vpc_net: + name: "{{ tiny_prefix }}-vpc" + state: present + cidr_block: 10.55.0.0/16 + tenancy: default + tags: + AnsibleEIPTest: "{{ tiny_prefix }}-vpc" + AnsibleTest: AnsibleTestVpc + register: testing_vpc + + - name: Wait until VPC is created. + ec2_vpc_net_info: + filters: + tag:AnsibleEIPTest: + - "{{ tiny_prefix }}-vpc" + register: vpc_info + retries: 120 + delay: 5 + until: + - vpc_info.vpcs | length > 0 + - vpc_info.vpcs[0].state == 'available' + + - name: Create internet gateway for use in testing + ec2_vpc_igw: + vpc_id: "{{ testing_vpc.vpc.id }}" + state: present + resource_tags: + Name: "{{ tiny_prefix }}-igw" + AnsibleTest: AnsibleTestVpc + register: igw + + # The list of AZs varies across regions and accounts. + # Gather info and pick two AZs for test purpose. + - name: gather AZ info in VPC for test purpose + amazon.aws.aws_az_info: + region: "{{ aws_region }}" + register: az_info + - assert: + that: + # We cannot run the test if this region does not have at least 2 AZs + - "az_info.availability_zones | length >= 2" + - set_fact: + test_az_zone1: "{{ az_info.availability_zones[0].zone_name }}" + test_az_zone2: "{{ az_info.availability_zones[1].zone_name }}" + - name: Create subnet_1 for use in testing + ec2_vpc_subnet: + state: present + vpc_id: "{{ testing_vpc.vpc.id }}" + cidr: 10.55.77.0/24 + az: "{{ test_az_zone1 }}" + resource_tags: + Name: "{{ tiny_prefix }}-subnet_1" + AnsibleTest: AnsibleTestVpc + wait: true + register: testing_subnet_1 + + - name: Create subnet_2 for use in testing + ec2_vpc_subnet: + state: present + vpc_id: "{{ testing_vpc.vpc.id }}" + cidr: 10.55.78.0/24 + az: "{{ test_az_zone2 }}" + resource_tags: + Name: "{{ tiny_prefix }}-subnet_2" + AnsibleTest: AnsibleTestVpc + wait: true + register: testing_subnet_2 + + - name: Create routing rules + ec2_vpc_route_table: + vpc_id: "{{ testing_vpc.vpc.id }}" + routes: + - dest: 0.0.0.0/0 + gateway_id: "{{ igw.gateway_id }}" + subnets: + - "{{ testing_subnet_1.subnet.id }}" + - "{{ testing_subnet_2.subnet.id }}" + tags: + Name: "{{ tiny_prefix }}-route" + AnsibleTest: AnsibleTestVpc + + - name: Create security group for use in testing + ec2_group: + name: "{{ tiny_prefix }}-sg" + description: a security group for ansible tests + vpc_id: "{{ testing_vpc.vpc.id }}" + tags: + Name: "{{ tiny_prefix }}-sg" + AnsibleTest: AnsibleTestVpc + rules: + - proto: tcp + from_port: 22 + to_port: 22 + cidr_ip: 0.0.0.0/0 + - proto: tcp + from_port: 80 + to_port: 80 + cidr_ip: 0.0.0.0/0 + register: sg + + # A private route53 zone is needed to configure a custom endpoint in the OpenSearch cluster. + # See https://docs.aws.amazon.com/opensearch-service/latest/developerguide/customendpoint.html + - name: Create private Route53 zone for the VPC + register: route53_zone + route53_zone: + comment: "zone for ansible integration tests" + zone: "ansible-integ-test.com" + state: present + vpc_id: '{{ testing_vpc.vpc.id }}' + tags: + Name: "{{ tiny_prefix }}-zone" + AnsibleTest: AnsibleTestVpc + + - name: Set fact for route53 zone id + set_fact: + route53_zone_id: "{{ route53_zone.result.zone_id }}" + + - name: Create KMS key for test purpose + # The key is needed for OpenSearch encryption at rest. + aws_kms: + alias: "{{ tiny_prefix }}-kms" + description: a key used for encryption at rest in test OpenSearch cluster + state: present + enabled: yes + key_spec: SYMMETRIC_DEFAULT + key_usage: ENCRYPT_DECRYPT + policy: "{{ lookup('template', 'kms_policy.j2') }}" + tags: + Name: "{{ tiny_prefix }}-kms" + AnsibleTest: AnsibleTestVpc + register: kms_test_key diff --git a/tests/integration/targets/opensearch/templates/kms_policy.j2 b/tests/integration/targets/opensearch/templates/kms_policy.j2 new file mode 100644 index 00000000000..e9ddc247298 --- /dev/null +++ b/tests/integration/targets/opensearch/templates/kms_policy.j2 @@ -0,0 +1,68 @@ +{ + "Id": "key-ansible-test-policy-123", + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Allow access for root user", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::{{ aws_caller_info.account }}:root" + }, + "Action": "kms:*", + "Resource": "*" + }, + { + "Sid": "AnsibleTestManage", + "Effect": "Allow", + "Principal": { + "AWS": "{{ aws_caller_info.arn }}" + }, + "Action": "kms:*", + "Resource": "*" + }, + { + "Sid": "Allow access for Key Administrators", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::{{ aws_caller_info.account }}:role/admin" + }, + "Action": "kms:*", + "Resource": "*" + }, + { + "Sid": "Allow access through OpenSearch Service for all principals in the account that are authorized to use OpenSearch Service", + "Effect": "Allow", + "Principal": { + "AWS": "*" + }, + "Action": [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:CreateGrant", + "kms:DescribeKey" + ], + "Resource": "*", + "Condition": { + "StringEquals": { + "kms:CallerAccount": "{{ aws_caller_info.account }}", + "kms:ViaService": "es.{{ aws_region }}.amazonaws.com" + } + } + }, + { + "Sid": "Allow OpenSearch service principals to describe the key directly", + "Effect": "Allow", + "Principal": { + "Service": "es.amazonaws.com" + }, + "Action": [ + "kms:Describe*", + "kms:Get*", + "kms:List*" + ], + "Resource": "*" + } + ] +} diff --git a/tests/unit/plugins/modules/test_opensearch.py b/tests/unit/plugins/modules/test_opensearch.py new file mode 100644 index 00000000000..836e2cf0788 --- /dev/null +++ b/tests/unit/plugins/modules/test_opensearch.py @@ -0,0 +1,86 @@ +# Copyright: Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import functools +from ansible_collections.community.aws.plugins.module_utils.opensearch import ( + compare_domain_versions, + parse_version, +) + + +def test_parse_version(): + test_versions = [ + ['Elasticsearch_5.5', {'engine_type': 'Elasticsearch', 'major': 5, 'minor': 5}], + ['Elasticsearch_7.1', {'engine_type': 'Elasticsearch', 'major': 7, 'minor': 1}], + ['Elasticsearch_7.10', {'engine_type': 'Elasticsearch', 'major': 7, 'minor': 10}], + ['OpenSearch_1.0', {'engine_type': 'OpenSearch', 'major': 1, 'minor': 0}], + ['OpenSearch_1.1', {'engine_type': 'OpenSearch', 'major': 1, 'minor': 1}], + ['OpenSearch_a.b', None], + ['OpenSearch_1.b', None], + ['OpenSearch_1-1', None], + ['OpenSearch_1.1.2', None], + ['OpenSearch_foo_1.1', None], + ['OpenSearch_1', None], + ['OpenSearch-1.0', None], + ['Foo_1.0', None], + ] + for expected in test_versions: + ret = parse_version(expected[0]) + if ret != expected[1]: + raise AssertionError( + f"parse_version({expected[0]} returned {ret}, expected {expected[1]}") + + +def test_version_compare(): + test_versions = [ + ['Elasticsearch_5.5', 'Elasticsearch_5.5', 0], + ['Elasticsearch_5.5', 'Elasticsearch_7.1', -1], + ['Elasticsearch_7.1', 'Elasticsearch_7.1', 0], + ['Elasticsearch_7.1', 'Elasticsearch_7.2', -1], + ['Elasticsearch_7.1', 'Elasticsearch_7.10', -1], + ['Elasticsearch_7.2', 'Elasticsearch_7.10', -1], + ['Elasticsearch_7.10', 'Elasticsearch_7.2', 1], + ['Elasticsearch_7.2', 'Elasticsearch_5.5', 1], + ['Elasticsearch_7.2', 'OpenSearch_1.0', -1], + ['Elasticsearch_7.2', 'OpenSearch_1.1', -1], + ['OpenSearch_1.1', 'OpenSearch_1.1', 0], + ['OpenSearch_1.0', 'OpenSearch_1.1', -1], + ['OpenSearch_1.1', 'OpenSearch_1.0', 1], + ['foo_1.1', 'OpenSearch_1.0', -1], + ['Elasticsearch_5.5', 'foo_1.0', 1], + ] + for v in test_versions: + ret = compare_domain_versions(v[0], v[1]) + if ret != v[2]: + raise AssertionError( + f"compare({v[0]}, {v[1]} returned {ret}, expected {v[2]}") + + +def test_sort_versions(): + input_versions = [ + 'Elasticsearch_5.6', + 'Elasticsearch_5.5', + 'Elasticsearch_7.10', + 'Elasticsearch_7.2', + 'foo_10.5', + 'OpenSearch_1.1', + 'OpenSearch_1.0', + 'Elasticsearch_7.3', + ] + expected_versions = [ + 'foo_10.5', + 'Elasticsearch_5.5', + 'Elasticsearch_5.6', + 'Elasticsearch_7.2', + 'Elasticsearch_7.3', + 'Elasticsearch_7.10', + 'OpenSearch_1.0', + 'OpenSearch_1.1', + ] + input_versions = sorted(input_versions, key=functools.cmp_to_key(compare_domain_versions)) + if input_versions != expected_versions: + raise AssertionError( + f"Expected {expected_versions}, got {input_versions}") From 2dcef5020ca7eedddcd40affff89dac39a4f7ae8 Mon Sep 17 00:00:00 2001 From: serosset Date: Tue, 8 Feb 2022 15:38:17 +0000 Subject: [PATCH 3/4] revert unwanted changes --- plugins/modules/wafv2_web_acl_info.py | 4 ++-- tests/integration/targets/lambda/tasks/main.yml | 2 +- tests/integration/targets/lambda_alias/tasks/main.yml | 2 +- tests/integration/targets/lambda_policy/tasks/main.yml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/modules/wafv2_web_acl_info.py b/plugins/modules/wafv2_web_acl_info.py index d8f4fdf091b..3fb91fbd802 100644 --- a/plugins/modules/wafv2_web_acl_info.py +++ b/plugins/modules/wafv2_web_acl_info.py @@ -28,8 +28,8 @@ type: str extends_documentation_fragment: -- amazon.aws.aws -- amazon.aws.ec2 + - amazon.aws.aws + - amazon.aws.ec2 ''' diff --git a/tests/integration/targets/lambda/tasks/main.yml b/tests/integration/targets/lambda/tasks/main.yml index c851bfef705..a7a1680d9e6 100644 --- a/tests/integration/targets/lambda/tasks/main.yml +++ b/tests/integration/targets/lambda/tasks/main.yml @@ -12,7 +12,7 @@ - name: create minimal lambda role iam_role: name: '{{ lambda_role_name }}' - assume_role_policy_document: '{{ lookup("file", "opensearch_policy.json") }}' + assume_role_policy_document: '{{ lookup("file", "minimal_trust_policy.json") }}' create_instance_profile: false managed_policies: - 'arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess' diff --git a/tests/integration/targets/lambda_alias/tasks/main.yml b/tests/integration/targets/lambda_alias/tasks/main.yml index 608e2175d70..9b264f50c0a 100644 --- a/tests/integration/targets/lambda_alias/tasks/main.yml +++ b/tests/integration/targets/lambda_alias/tasks/main.yml @@ -13,7 +13,7 @@ - name: create minimal lambda role iam_role: name: '{{ lambda_role_name }}' - assume_role_policy_document: '{{ lookup("file", "opensearch_policy.json") }}' + assume_role_policy_document: '{{ lookup("file", "minimal_trust_policy.json") }}' create_instance_profile: false managed_policies: - 'arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess' diff --git a/tests/integration/targets/lambda_policy/tasks/main.yml b/tests/integration/targets/lambda_policy/tasks/main.yml index c62cca32c1e..e0b514bde43 100644 --- a/tests/integration/targets/lambda_policy/tasks/main.yml +++ b/tests/integration/targets/lambda_policy/tasks/main.yml @@ -13,7 +13,7 @@ - name: create minimal lambda role iam_role: name: '{{ lambda_role_name }}' - assume_role_policy_document: '{{ lookup("file", "opensearch_policy.json") }}' + assume_role_policy_document: '{{ lookup("file", "minimal_trust_policy.json") }}' create_instance_profile: false managed_policies: - 'arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess' @@ -51,7 +51,7 @@ - name: create minimal lambda role iam_role: name: ansible_lambda_role - assume_role_policy_document: '{{ lookup(''file'', ''opensearch_policy.json'', convert_data=False) }}' + assume_role_policy_document: '{{ lookup(''file'', ''minimal_trust_policy.json'', convert_data=False) }}' create_instance_profile: false register: iam_role - name: wait 10 seconds for role to become available From 3a1e66d1d8a1691b465f9801e0f634272e5261ab Mon Sep 17 00:00:00 2001 From: Sebastien Rosset Date: Wed, 23 Feb 2022 12:49:28 -0800 Subject: [PATCH 4/4] Disable CI test by default --- tests/integration/targets/opensearch/aliases | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/targets/opensearch/aliases b/tests/integration/targets/opensearch/aliases index 80e3d995cb7..643f94c94f9 100644 --- a/tests/integration/targets/opensearch/aliases +++ b/tests/integration/targets/opensearch/aliases @@ -19,6 +19,6 @@ # 3 minutes to enable auto-tune option. # 45 minutes to delete cluster. -#disabled +disabled cloud/aws