diff --git a/.gitignore b/.gitignore index 0e9ac3c..66226d3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .cache/ venv/ +.venv/ test_inventory* diff --git a/roles/rke2/molecule/README.md b/roles/rke2/molecule/README.md new file mode 100644 index 0000000..f4c7f60 --- /dev/null +++ b/roles/rke2/molecule/README.md @@ -0,0 +1,30 @@ +# Molecule Scenarios +The molecule test scenarios are based on the cookie cutter ec2 instance and require the molecule plugin here: [molecule-plugin](https://github.com/ansible-community/molecule-plugins), the pip3 `requirements.txt` can be found in this directory while the ansible specfic requirements will be installed automatically when running molecule as a part of the `requirements` stage. +As this is an ec2 based scenario an AWS account is needed, you will need to define the following variables either as environment variables or in your aws cli config file (`~/.aws/config`) + +``` +export AWS_ACCESS_KEY_ID="" +export AWS_SECRET_ACCESS_KEY="" +``` + +or +``` +[default] +aws_access_key_id= +aws_secret_access_key= +``` + +It is worth noting that the EC2 driver does not provide a way to login to EC2 instances, this needs to be done manually, your ssh key can be found in `~/.cache/molecule/rke2/default/id_rsa` and the default user is `ansible`, you will be able to login like so: +`ssh ansible@000.000.000.000 -i ~/.cache/molecule/rke2/default/id_rsa` note that the keys location is dependant on the scenario name. + +# Available Scenarios +## default +The default scenario is the simplest possible scenario, with a single Ubuntu 20.04 master node and a single Ubuntu 20.04 worker node. + +# To Do + - Add tests + - Ensure node labels are applied + - Ensure setting CIS profile works as expected + - Add scenrios for all supported platforms + - Rocky + - SLES \ No newline at end of file diff --git a/roles/rke2/molecule/default/converge.yml b/roles/rke2/molecule/default/converge.yml new file mode 100644 index 0000000..1966131 --- /dev/null +++ b/roles/rke2/molecule/default/converge.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + gather_facts: true + pre_tasks: + - name: Set api_server_host + ansible.builtin.set_fact: + rke2_kubernetes_api_server_host: "{{ hostvars[groups['rke2_servers'][0]].ansible_host }}" + roles: + - role: rke2 + become: true \ No newline at end of file diff --git a/roles/rke2/molecule/default/create.yml b/roles/rke2/molecule/default/create.yml new file mode 100644 index 0000000..c128de6 --- /dev/null +++ b/roles/rke2/molecule/default/create.yml @@ -0,0 +1,333 @@ +--- +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: '{{ default_run_config | combine(run_config_from_file) }}' + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t2.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: '' + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + - proto: tcp + from_port: 9345 + to_port: 9345 + cidr_ip: "0.0.0.0/0" + - proto: tcp + from_port: 6443 + to_port: 6443 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: '{{ platforms }}' + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + backend: cryptography + private_key_format: pkcs1 + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + name: "{{ item.name }}" + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + molecule-run-id: "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" \ No newline at end of file diff --git a/roles/rke2/molecule/default/destroy.yml b/roles/rke2/molecule/default/destroy.yml new file mode 100644 index 0000000..54ca53b --- /dev/null +++ b/roles/rke2/molecule/default/destroy.yml @@ -0,0 +1,143 @@ +--- +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: '{{ default_run_config | combine(run_config_from_file) }}' + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: '{{ platforms }}' + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" \ No newline at end of file diff --git a/roles/rke2/molecule/default/molecule.yml b/roles/rke2/molecule/default/molecule.yml new file mode 100644 index 0000000..f6834ea --- /dev/null +++ b/roles/rke2/molecule/default/molecule.yml @@ -0,0 +1,61 @@ +--- +driver: + name: ec2 + +platforms: + - name: master-01 + image: ami-0862be96e41dcbf74 + instance_type: t2.medium + region: us-east-2 + assign_public_ip: true + vpc_subnet_id: subnet-095d88c4efe5abf6a + tags: + deployed-with: "molecule" + molecule-scenario: "default" + groups: + - rke2_servers + - name: worker-01 + image: ami-0862be96e41dcbf74 + instance_type: t2.medium + region: us-east-2 + assign_public_ip: true + vpc_subnet_id: subnet-095d88c4efe5abf6a + tags: + deployed-with: "molecule" + molecule-scenario: "default" + groups: + - rke2_agents + +provisioner: + name: ansible + playbooks: + converge: converge.yml + inventory: + hosts: + rke2_cluster: + vars: + rke2_install_version: v1.27.15+rke2r1 + children: + rke2_servers: + vars: + group_rke2_config: + node-label: + - serverGroupLabel=true + hosts: + master-01: + host_rke2_config: + node-label: + - host0Label=true + rke2_agents: + vars: + group_rke2_config: + node-label: + - agentGroupLabel=true + hosts: + worker-01: + host_rke2_config: + node-label: + - host1Label=true + +verifier: + name: ansible \ No newline at end of file diff --git a/roles/rke2/molecule/default/requirements.yml b/roles/rke2/molecule/default/requirements.yml new file mode 100644 index 0000000..4ece6bc --- /dev/null +++ b/roles/rke2/molecule/default/requirements.yml @@ -0,0 +1,5 @@ +--- +collections: + - name: ansible.utils + - name: amazon.aws + - name: community.crypto \ No newline at end of file diff --git a/roles/rke2/molecule/requirements.txt b/roles/rke2/molecule/requirements.txt new file mode 100644 index 0000000..60a8885 --- /dev/null +++ b/roles/rke2/molecule/requirements.txt @@ -0,0 +1,28 @@ +ansible-compat==24.7.0 +ansible-core==2.17.2 +attrs==23.2.0 +bracex==2.4 +cffi==1.16.0 +click==8.1.7 +click-help-colors==0.9.4 +cryptography==42.0.8 +enrich==1.2.7 +Jinja2==3.1.4 +jsonschema==4.23.0 +jsonschema-specifications==2023.12.1 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +mdurl==0.1.2 +molecule==24.7.0 +molecule-plugins==23.5.3 +packaging==24.1 +pluggy==1.5.0 +pycparser==2.22 +Pygments==2.18.0 +PyYAML==6.0.1 +referencing==0.35.1 +resolvelib==1.0.1 +rich==13.7.1 +rpds-py==0.19.0 +subprocess-tee==0.4.2 +wcmatch==8.5.2