From 5ea5b9e6e871445e23a3206c65bc5df1b940bf09 Mon Sep 17 00:00:00 2001 From: George Adams Date: Sun, 1 Jul 2018 14:38:07 +0100 Subject: [PATCH 1/3] WIP: ansible: add ansible tower playbook This playbook will create the basic AWX tower machine. --- ansible/inventory.yml | 1 + ansible/playbooks/create-ansible-tower.yml | 182 ++++++++++++++++++ ansible/plugins/inventory/nodejs_yaml.py | 4 +- .../bootstrap/tasks/partials/ubuntu1804.yml | 19 ++ 4 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 ansible/playbooks/create-ansible-tower.yml create mode 100644 ansible/roles/bootstrap/tasks/partials/ubuntu1804.yml diff --git a/ansible/inventory.yml b/ansible/inventory.yml index 28936255a..c5c5d2f97 100644 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -25,6 +25,7 @@ hosts: debian8-x64-1: {ip: 184.172.29.199, alias: registry-mirror} ubuntu1404-x64-1: {ip: 169.44.16.104, alias: ci-release} ubuntu1404-x64-2: {ip: 50.23.85.254} + ubuntu1804-x64-2: {ip: 169.62.77.235, alias: ansible} - release: diff --git a/ansible/playbooks/create-ansible-tower.yml b/ansible/playbooks/create-ansible-tower.yml new file mode 100644 index 000000000..b58adf0b0 --- /dev/null +++ b/ansible/playbooks/create-ansible-tower.yml @@ -0,0 +1,182 @@ +--- + +# +# sets up the host that runs ansible.nodejs.org +# + +- hosts: infra-softlayer-ubuntu1804-x64-2 + gather_facts: yes + roles: + - bootstrap + - package-upgrade + + pre_tasks: + - name: check if secrets are properly set + fail: + failed_when: not {{ secret }} + loop_control: + loop_var: secret + with_items: + - awx_password + - awx_org + + tasks: + + - name: set hostname + hostname: name="ansible.nodejs.org" + + - name: update /etc/hosts file - 127.0.0.1 + lineinfile: + dest: /etc/hosts + regexp: "^(.*)127.0.0.1 ansible.nodejs.org(.*)$" + line: "127.0.0.1 ansible.nodejs.org ansible-tower" + state: present + backup: yes + + - name: import docker gpg key + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + state: present + + - name: import docker apt repostory + apt_repository: + repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable + state: present + + - name: import ansible apt repostory + apt_repository: + repo: ppa:ansible/ansible + state: present + + - name: install apt dependencies + apt: + name: "{{ packages }}" + update_cache: yes + vars: + packages: + - ansible + - apt-transport-https + - binutils + - ca-certificates + - curl + - dkms + - docker-ce + - gcc + - git + - make + - patch + - python-pip + - python3-pip + - software-properties-common + - vim + + - name: install pip dependencies + pip: + name: "{{ packages }}" + vars: + packages: + - ansible-tower-cli + - django + - docker + - pexpect + + - name: start the docker service + service: + name: docker + state: started + + - name: create /opt/awx_install_files + file: path=/opt/awx_install_files state=directory + + - name: clone awx github repository + git: + repo: https://github.com/ansible/awx.git + dest: /opt/awx_install_files/awx + force: yes + + - name: run awx install files, this can take ~20 mins + command: chdir=/opt/awx_install_files/awx/installer/ ansible-playbook -i inventory install.yml + + - name: change awx admin password + expect: + command: docker exec -it awx_task awx-manage changepassword admin + responses: + (?i)password: "{{ awx_password }}" + + #TODO: change this to be https://ansible.nodejs.org and remove verify_ssl + - name: create /root/.tower_cli.cfg + blockinfile: + path: /root/.tower_cli.cfg + block: | + host: 169.62.77.235 + username: admin + password: {{ awx_password }} + verify_ssl: False + create: yes + mode: 0700 + + - name: delete Default organization created at install + command: tower-cli organization delete Default + + - name: create new organization (set in hosts file) + command: tower-cli organization create --name {{ awx_org }} + + - name: create new team belonging to organization + command: tower-cli team create --name {{ awx_org }} --organization {{ awx_org }} + + - name: add nodejs/build repo to AWX + command: tower-cli project create --organization {{ awx_org }} --name nodejs/build --scm-type git --scm-url https://github.com/nodejs/build --scm-branch master --scm-update-on-launch true + + - name: create inventory + command: tower-cli inventory create --organization {{ awx_org }} --name nodejs_inventory_github + + - name: create automate fetching inventory from inventory.yml + command: tower-cli inventory_source create --name github --inventory nodejs_inventory_github --source scm --source-project nodejs/build --source-path ansible/plugins/inventory/nodejs_yaml.py --overwrite true --overwrite-vars true --update-on-launch true + + - name: prevent ubuntu deleting /tmp folder on reboot + command: echo > /etc/init/mounted-tmp.conf + + - name: patch ubuntu weekly at 5 Sundays + cron: + name: "apt-get update" + minute: "0" + hour: "5" + weekday: "sun" + job: "apt-get update" + + - name: upgrade ubuntu weekly at 5:15 Sundays + cron: + name: "apt-get -y upgrade" + minute: "15" + hour: "5" + weekday: "sun" + job: "apt-get -y upgrade" + + - name: create /backup + file: path=/backup state=directory + + - name: setup Backups for AWX docker containers + blockinfile: + path: /backup/backup_docker_AWX.sh + block: | + DATE=`date +%m-%d-%Y` + + # Create Tar backups from the containers + docker save -o /backup/awx_test_backup_$DATE.tar ansible/awx_task + docker save -o /backup/awx_web_backup_$DATE.tar ansible/awx_web + docker save -o /backup/rabbitmq_backup_$DATE.tar rabbitmq + docker save -o /backup/postgres_backup_$DATE.tar postgres + docker save -o /backup/memcached_backup_$DATE.tar memcached + tar -cvf /backup/postgres_tmp_backup_$DATE.tar /tmp/pgdocker + create: yes + + - name: make backup_docker_AWX.sh executable + command: chmod +x /backup/backup_docker_AWX.sh + + - name: backup AWX's docker containers weekly at 5am on Mondays + cron: + name: "/backup/backup_docker_AWX.sh" + minute: "0" + hour: "5" + weekday: "mon" + job: "/backup/backup_docker_AWX.sh" diff --git a/ansible/plugins/inventory/nodejs_yaml.py b/ansible/plugins/inventory/nodejs_yaml.py index eb5aff917..8d1ece20b 100755 --- a/ansible/plugins/inventory/nodejs_yaml.py +++ b/ansible/plugins/inventory/nodejs_yaml.py @@ -72,7 +72,9 @@ def main(): export = {'_meta': {'hostvars': {}}} # get inventory - with open("inventory.yml", 'r') as stream: + basepath = path.dirname(__file__) + inventory_path = path.abspath(path.join(basepath, "..", "..", "inventory.yml")) + with open(inventory_path, 'r') as stream: try: hosts = yaml.load(stream) diff --git a/ansible/roles/bootstrap/tasks/partials/ubuntu1804.yml b/ansible/roles/bootstrap/tasks/partials/ubuntu1804.yml new file mode 100644 index 000000000..27c543317 --- /dev/null +++ b/ansible/roles/bootstrap/tasks/partials/ubuntu1804.yml @@ -0,0 +1,19 @@ +--- + +# +# ubuntu 16.04 +# + +- name: check for python + raw: stat /usr/bin/python + failed_when: has_python.rc > 1 + register: has_python + +- name: check for aptitude + raw: stat /usr/bin/aptitude + failed_when: has_aptitude.rc > 1 + register: has_aptitude + +- name: install python and aptitude + when: has_python.rc == 1 or has_aptitude.rc == 1 + raw: apt-get update && apt install -y python-minimal aptitude From a0945d5c707bcaf1f238e80412482a42ece28ef3 Mon Sep 17 00:00:00 2001 From: George Adams Date: Mon, 2 Jul 2018 13:16:59 +0100 Subject: [PATCH 2/3] Update ubuntu1804.yml --- ansible/roles/bootstrap/tasks/partials/ubuntu1804.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/bootstrap/tasks/partials/ubuntu1804.yml b/ansible/roles/bootstrap/tasks/partials/ubuntu1804.yml index 27c543317..8046038b9 100644 --- a/ansible/roles/bootstrap/tasks/partials/ubuntu1804.yml +++ b/ansible/roles/bootstrap/tasks/partials/ubuntu1804.yml @@ -1,7 +1,7 @@ --- # -# ubuntu 16.04 +# ubuntu 18.04 # - name: check for python From 0bf304c097b2f6876d71abe1288fd97fc675f4b6 Mon Sep 17 00:00:00 2001 From: George Adams Date: Tue, 3 Jul 2018 10:43:24 +0100 Subject: [PATCH 3/3] Update create-ansible-tower.yml --- ansible/playbooks/create-ansible-tower.yml | 162 +------------------- ansible/playbooks/jenkins/worker/create.yml | 1 + ansible/plugins/inventory/nodejs_yaml.py | 1 + ansible/roles/ansible-tower/tasks/main.yml | 160 +++++++++++++++++++ 4 files changed, 163 insertions(+), 161 deletions(-) create mode 100644 ansible/roles/ansible-tower/tasks/main.yml diff --git a/ansible/playbooks/create-ansible-tower.yml b/ansible/playbooks/create-ansible-tower.yml index b58adf0b0..7a67ac99a 100644 --- a/ansible/playbooks/create-ansible-tower.yml +++ b/ansible/playbooks/create-ansible-tower.yml @@ -9,6 +9,7 @@ roles: - bootstrap - package-upgrade + - ansible-tower pre_tasks: - name: check if secrets are properly set @@ -19,164 +20,3 @@ with_items: - awx_password - awx_org - - tasks: - - - name: set hostname - hostname: name="ansible.nodejs.org" - - - name: update /etc/hosts file - 127.0.0.1 - lineinfile: - dest: /etc/hosts - regexp: "^(.*)127.0.0.1 ansible.nodejs.org(.*)$" - line: "127.0.0.1 ansible.nodejs.org ansible-tower" - state: present - backup: yes - - - name: import docker gpg key - apt_key: - url: https://download.docker.com/linux/ubuntu/gpg - state: present - - - name: import docker apt repostory - apt_repository: - repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable - state: present - - - name: import ansible apt repostory - apt_repository: - repo: ppa:ansible/ansible - state: present - - - name: install apt dependencies - apt: - name: "{{ packages }}" - update_cache: yes - vars: - packages: - - ansible - - apt-transport-https - - binutils - - ca-certificates - - curl - - dkms - - docker-ce - - gcc - - git - - make - - patch - - python-pip - - python3-pip - - software-properties-common - - vim - - - name: install pip dependencies - pip: - name: "{{ packages }}" - vars: - packages: - - ansible-tower-cli - - django - - docker - - pexpect - - - name: start the docker service - service: - name: docker - state: started - - - name: create /opt/awx_install_files - file: path=/opt/awx_install_files state=directory - - - name: clone awx github repository - git: - repo: https://github.com/ansible/awx.git - dest: /opt/awx_install_files/awx - force: yes - - - name: run awx install files, this can take ~20 mins - command: chdir=/opt/awx_install_files/awx/installer/ ansible-playbook -i inventory install.yml - - - name: change awx admin password - expect: - command: docker exec -it awx_task awx-manage changepassword admin - responses: - (?i)password: "{{ awx_password }}" - - #TODO: change this to be https://ansible.nodejs.org and remove verify_ssl - - name: create /root/.tower_cli.cfg - blockinfile: - path: /root/.tower_cli.cfg - block: | - host: 169.62.77.235 - username: admin - password: {{ awx_password }} - verify_ssl: False - create: yes - mode: 0700 - - - name: delete Default organization created at install - command: tower-cli organization delete Default - - - name: create new organization (set in hosts file) - command: tower-cli organization create --name {{ awx_org }} - - - name: create new team belonging to organization - command: tower-cli team create --name {{ awx_org }} --organization {{ awx_org }} - - - name: add nodejs/build repo to AWX - command: tower-cli project create --organization {{ awx_org }} --name nodejs/build --scm-type git --scm-url https://github.com/nodejs/build --scm-branch master --scm-update-on-launch true - - - name: create inventory - command: tower-cli inventory create --organization {{ awx_org }} --name nodejs_inventory_github - - - name: create automate fetching inventory from inventory.yml - command: tower-cli inventory_source create --name github --inventory nodejs_inventory_github --source scm --source-project nodejs/build --source-path ansible/plugins/inventory/nodejs_yaml.py --overwrite true --overwrite-vars true --update-on-launch true - - - name: prevent ubuntu deleting /tmp folder on reboot - command: echo > /etc/init/mounted-tmp.conf - - - name: patch ubuntu weekly at 5 Sundays - cron: - name: "apt-get update" - minute: "0" - hour: "5" - weekday: "sun" - job: "apt-get update" - - - name: upgrade ubuntu weekly at 5:15 Sundays - cron: - name: "apt-get -y upgrade" - minute: "15" - hour: "5" - weekday: "sun" - job: "apt-get -y upgrade" - - - name: create /backup - file: path=/backup state=directory - - - name: setup Backups for AWX docker containers - blockinfile: - path: /backup/backup_docker_AWX.sh - block: | - DATE=`date +%m-%d-%Y` - - # Create Tar backups from the containers - docker save -o /backup/awx_test_backup_$DATE.tar ansible/awx_task - docker save -o /backup/awx_web_backup_$DATE.tar ansible/awx_web - docker save -o /backup/rabbitmq_backup_$DATE.tar rabbitmq - docker save -o /backup/postgres_backup_$DATE.tar postgres - docker save -o /backup/memcached_backup_$DATE.tar memcached - tar -cvf /backup/postgres_tmp_backup_$DATE.tar /tmp/pgdocker - create: yes - - - name: make backup_docker_AWX.sh executable - command: chmod +x /backup/backup_docker_AWX.sh - - - name: backup AWX's docker containers weekly at 5am on Mondays - cron: - name: "/backup/backup_docker_AWX.sh" - minute: "0" - hour: "5" - weekday: "mon" - job: "/backup/backup_docker_AWX.sh" diff --git a/ansible/playbooks/jenkins/worker/create.yml b/ansible/playbooks/jenkins/worker/create.yml index 775b34835..28b625421 100644 --- a/ansible/playbooks/jenkins/worker/create.yml +++ b/ansible/playbooks/jenkins/worker/create.yml @@ -9,6 +9,7 @@ - release # Benchmarking machine - infra-softlayer-ubuntu1404-x64-2 + - "!infra-softlayer-ubuntu1804-x64-2" - "!*-win*" roles: diff --git a/ansible/plugins/inventory/nodejs_yaml.py b/ansible/plugins/inventory/nodejs_yaml.py index 8d1ece20b..c00a077a3 100755 --- a/ansible/plugins/inventory/nodejs_yaml.py +++ b/ansible/plugins/inventory/nodejs_yaml.py @@ -35,6 +35,7 @@ import yaml import os import sys +from os import path valid = { diff --git a/ansible/roles/ansible-tower/tasks/main.yml b/ansible/roles/ansible-tower/tasks/main.yml new file mode 100644 index 000000000..35c9c8829 --- /dev/null +++ b/ansible/roles/ansible-tower/tasks/main.yml @@ -0,0 +1,160 @@ +--- + +- name: set hostname + hostname: name="ansible.nodejs.org" + +- name: update /etc/hosts file - 127.0.0.1 + lineinfile: + dest: /etc/hosts + regexp: "^(.*)127.0.0.1 ansible.nodejs.org(.*)$" + line: "127.0.0.1 ansible.nodejs.org ansible-tower" + state: present + backup: yes + +- name: import docker gpg key + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + state: present + +- name: import docker apt repostory + apt_repository: + repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable + state: present + +- name: import ansible apt repostory + apt_repository: + repo: ppa:ansible/ansible + state: present + +- name: install apt dependencies + apt: + name: "{{ packages }}" + update_cache: yes + vars: + packages: + - ansible + - apt-transport-https + - binutils + - ca-certificates + - curl + - dkms + - docker-ce + - gcc + - git + - make + - patch + - python-pip + - python3-pip + - software-properties-common + - vim + +- name: install pip dependencies + pip: + name: "{{ packages }}" + vars: + packages: + - ansible-tower-cli + - django + - docker + - pexpect + +- name: start the docker service + service: + name: docker + state: started + +- name: create /opt/awx_install_files + file: path=/opt/awx_install_files state=directory + +- name: clone awx github repository + git: + repo: https://github.com/ansible/awx.git + dest: /opt/awx_install_files/awx + force: yes + +- name: run awx install files, this can take ~20 mins + command: chdir=/opt/awx_install_files/awx/installer/ ansible-playbook -i inventory install.yml + +- name: change awx admin password + expect: + command: docker exec -it awx_task awx-manage changepassword admin + responses: + (?i)password: "{{ awx_password }}" + +#TODO: change this to be https://ansible.nodejs.org and remove verify_ssl +- name: create /root/.tower_cli.cfg + blockinfile: + path: /root/.tower_cli.cfg + block: | + host: http://169.62.77.235 + username: admin + password: {{ awx_password }} + verify_ssl: False + create: yes + mode: 0700 + +- name: delete Default organization created at install + command: tower-cli organization delete Default + +- name: create new organization (set in hosts file) + command: tower-cli organization create --name {{ awx_org }} + +- name: create new team belonging to organization + command: tower-cli team create --name {{ awx_org }} --organization {{ awx_org }} + +- name: add nodejs/build repo to AWX + command: tower-cli project create --organization {{ awx_org }} --name nodejs/build --scm-type git --scm-url https://github.com/nodejs/build --scm-branch master --scm-update-on-launch true + +- name: create inventory + command: tower-cli inventory create --organization {{ awx_org }} --name nodejs_inventory_github + +- name: create automate fetching inventory from inventory.yml + command: tower-cli inventory_source create --name github --inventory nodejs_inventory_github --source scm --source-project nodejs/build --source-path ansible/plugins/inventory/nodejs_yaml.py --overwrite true --overwrite-vars true --update-on-launch true + +- name: prevent ubuntu deleting /tmp folder on reboot + command: echo > /etc/init/mounted-tmp.conf + +- name: patch ubuntu weekly at 5 Sundays + cron: + name: "apt-get update" + minute: "0" + hour: "5" + weekday: "sun" + job: "apt-get update" + +- name: upgrade ubuntu weekly at 5:15 Sundays + cron: + name: "apt-get -y upgrade" + minute: "15" + hour: "5" + weekday: "sun" + job: "apt-get -y upgrade" + +- name: create /backup + file: path=/backup state=directory + +- name: setup Backups for AWX docker containers + blockinfile: + path: /backup/backup_docker_AWX.sh + block: | + DATE=`date +%m-%d-%Y` + + # Create Tar backups from the containers + docker save -o /backup/awx_test_backup_$DATE.tar ansible/awx_task + docker save -o /backup/awx_web_backup_$DATE.tar ansible/awx_web + docker save -o /backup/rabbitmq_backup_$DATE.tar rabbitmq + docker save -o /backup/postgres_backup_$DATE.tar postgres + docker save -o /backup/memcached_backup_$DATE.tar memcached + tar -cvf /backup/postgres_tmp_backup_$DATE.tar /tmp/pgdocker + create: yes + +- name: make backup_docker_AWX.sh executable + command: chmod +x /backup/backup_docker_AWX.sh + +- name: backup AWX's docker containers weekly at 5am on Mondays + cron: + name: "/backup/backup_docker_AWX.sh" + minute: "0" + hour: "5" + weekday: "mon" + job: "/backup/backup_docker_AWX.sh"