diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b678442 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +out/* +env \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1cc5603 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:22.04 + +#Get dependencies +RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt update && apt-get -y install tzdata +ENV LIBGUESTFS_BACKEND=direct + +WORKDIR /tmp +RUN apt-get install --no-install-recommends --no-install-suggests -y libguestfs-tools qemu-utils linux-image-generic wget unzip zip +# RUN git clone https://github.com/cl0-de/riasc-provisioning.git -b development +ENV FLAVOR=raspios +ENV REPOFOLDER=/tmp +# ENV LIBGUESTFS_DEBUG=1 +# ENV LIBGUESTFS_TRACE=1 +# RUN ./riasc-provisioning/rpi/create_image.sh + +CMD ${REPOFOLDER}/riasc/rpi/create_image.sh \ No newline at end of file diff --git a/README.md b/README.md index fcb55f1..b76c9a5 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,88 @@ -# RIasC Provisioning Scripts +# Raspberry PI Image generation -[![GitHub](https://img.shields.io/github/license/ERIGrid2/riasc-provisioning)](https://github.com/ERIGrid2/riasc-provisioning/blob/master/LICENSE) +This project generates and customizes a Raspberry Pi image, either for Ubuntu or Raspberry Pi OS. The script is build for running within a docker container. + +The customizations include: +* Generating or adding a `vaultkey.secret` file +* Updating Ansible configuration in `/boot/firmware/riasc.yaml` +* Updating cloud-init file in `/boot/firmware/user-data` +* Setting GIT Ansible repor in `/boot/firmware/riasc.yaml` +* Adding and enabling GIT based Ansible updates on reboot -- **Based on:** -## Introduction +## Usage +1) Switch to direcory with Dockerfile to build image +``` +docker build --tag "imagebuilder" . +``` +2) Create `env` file: +``` +GIT_URL=https://mygiturl +FLAVOR=ubuntu22.04 +GIT_BRANCH=mybranch +NODENAME=myhost +TAG=test +``` +3) Run docker container to generate image +``` +docker run \ +--volume ./:/tmp/riasc \ +--volume ./out/:/tmp/data \ +--env-file ./env \ +imagebuilder +``` +4) Image is placed in ´out/output´ folder +5) Copy image to SD card. Either using dd or the Raspberry Pi Imiger -## Documentation -For further documentation, please consult: https://riasc.eu/docs/ +## List of available variables +| Variable | Info | +| - | - | +|GIT_URL | URL to ansible git repository| +|FLAVOR | Falvor of os. See list of flavors| +|GIT_BRANCH | Branch used in ansible git pull| +|NODENAME | The hostname of the device| +|TAG | A tag that is added to the name| +|RAW_OUTPOUT | Set to yes to get the .img file as output| +|TOKEN | A token used by Ansible| +|VAULT_KEY | Key to use in the vaultkey.secret file| -## System requirements +### List of flavors -The scripts have been tested with the following operating systems: +ubuntu24.04 -- Ubuntu 20.03 -- Raspbian Buster +ubuntu22.04 -## Usage +ubuntu20.04 + +raspios + + +# Help + +## How to mount the generated image to check the content? + +Check for the partitions in the image file: + +`fdisk -lu ubuntu-22.04.4-preinstalled-server-arm64+raspi.img` + +Run mount command. Make sure to update the offset (526336) for the correct value + +`mount ubuntu-22.04.4-preinstalled-server-arm64+raspi.img -o loop,offset=$(( 512 * 526336)) /mnt/` + +## How to add my custom secrets file for ansible vaults? + +To use a custom secret the VAULT_KEY variable can be set. If a vaultkey file of the name NODENAME-vaultkey.secret already exists the variable will be ignored. + +[![GitHub](https://img.shields.io/github/license/ERIGrid2/riasc-provisioning)](https://github.com/ERIGrid2/riasc-provisioning/blob/master/LICENSE) -See: https://riasc.eu/docs/setup/agent/manual ## Credits - [Steffen Vogel](https://github.com/stv0g) [📧](mailto:post@steffenvogel.de), [Institute for Automation of Complex Power Systems](https://www.acs.eonerc.rwth-aachen.de), [RWTH Aachen University](https://www.rwth-aachen.de) +- [Vincent Bareiß]() [📧](mailto:), [Institute for Automation of Complex Power Systems](https://www.acs.eonerc.rwth-aachen.de), [RWTH Aachen University](https://www.rwth-aachen.de) +- [Manuel Pitz](https://https://github.com/windrad6) [📧](mailto:post@cl0.de), [Institute for Automation of Complex Power Systems](https://www.acs.eonerc.rwth-aachen.de), [RWTH Aachen University](https://www.rwth-aachen.de) ### Funding acknowledment diff --git a/common/riasc-update.sh b/common/riasc-update.sh index d2dc60b..674de80 100755 --- a/common/riasc-update.sh +++ b/common/riasc-update.sh @@ -150,11 +150,19 @@ ANSIBLE_EXTRA_VARS="$(config --tojson --indent 0 .ansible.variables)" ANSIBLE_OPTS=" --url $(config .ansible.url)" ANSIBLE_OPTS+=" --inventory $(config .ansible.inventory)" ANSIBLE_OPTS+=" $(config '.ansible.extra_args // [ ] | join(" ")')" +if [ -f /boot/firmware/vaultkey.secret ]; then + ANSIBLE_OPTS+=" --vault-password-file /boot/firmware/vaultkey.secret" +fi + if [ $(config '.ansible.verify_commit') == "true" ]; then ANSIBLE_OPTS+="--verify-commit" fi +if ! [ $(config '.ansible.branch') = null ]; then + ANSIBLE_OPTS+=" --checkout $(config '.ansible.branch')" +fi + # Run Ansible playbook log "Running Ansible playbook..." ANSIBLE_FORCE_COLOR=1 \ diff --git a/common/riasc.edgeflex.yaml b/common/riasc.edgeflex.yaml deleted file mode 100644 index 0e5db46..0000000 --- a/common/riasc.edgeflex.yaml +++ /dev/null @@ -1,37 +0,0 @@ ---- -# RIasC configuration file -# See also: https://erigrid2.github.io/riasc/docs/setup/config - -# A unique hostname to identify the node -hostname: edgepmu - -ansible: - # List of PGP keys which are used to verify the commits in the Ansible repo - keys: [] - - keyserver: keys.openpgp.org - - # A repository containing ansible playbooks which will be fetched via ansible-pull - url: https://git.rwth-aachen.de/acs/public/software/pmu/pmu-ansible.git - - verify_commit: false - - # The playbook which should be provision the node - playbook: playbook.yml - - # A path to the Ansible inventory within the repo from above - inventory: inventory/edgeflex/hosts.yml - - # extra_args: - # - --only-if-changed - - # Additional variables which are passed to the Ansible playbook for provisioning - variables: - - # A list of SSH keys which will be added to the 'pi' user - #additional_ssh_keys: - #- "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDOmn+2RUo3XrHmm7h3w647+f6DlV5wXzBJSYLa7NBVlEasBd6Gxp4HDh1iKmNgZrneqXWOAH19P41k1qiwAx8/gUYDlnoah30RWp3qlXXIN+RUCUyUx34tatGTSDynuAyYKAOg0CawK066pSaRsMan2JdhL+r0YAKDXswMU8NVdc32AFTJLrZyGzlrFn4y7hSgHCtOy2RvoULWcjVtcF0GLuSr3WCLUwm1Qy83tft++5FJCcNtg986or5OOeWtZxFU035Q+2Khd2JqyweFTmCMRXEXAzUMv6Lqxuw19qpK4eOOU59oybXlga5yLFZY0CVBK8XVVnA0pkCnabQPHvVJWL7tcDVvzfnd0JdHpQ6oMsYFM7MdncKbHay5/g/tUz34WXPPAPWvciyLkI1NMV++JpaPeMEkAvrsUUZMko+4EQ75ZErUOY3PAJG+2Jilb79QTRMzpCVWfQ0A/Zx1O7Y0sLks+f5VDJaPnHl0vXetFIkEpv9XjomNZgp81m3XRItJh0+BwGqoYtc+MTLelAu0oap1LyHSbpBWGzKxi2tW9VOrX7/pI18f0AhhCc/BG/6UFjMhyYzGj9wg6HWAo1N8hlhdeuNic004hbADX7qzXJMyVEol5AL0rpu3j3j5zTeULY3HxiE+FC6yKg1DKvcxHkmwXoWJYwW9yAgYDyjhbw==" # Manuel Pitz - # Set this to true if you want to login via SSH keys only. - # If you dont have an SSH key, set this to false. - # Important: Dont forget to change the default password after your first login! - disable_password_login: false diff --git a/common/riasc.yaml b/common/riasc.raspios.yaml similarity index 100% rename from common/riasc.yaml rename to common/riasc.raspios.yaml diff --git a/common/riasc.ubuntu.yaml b/common/riasc.ubuntu.yaml new file mode 100644 index 0000000..96b0e80 --- /dev/null +++ b/common/riasc.ubuntu.yaml @@ -0,0 +1,40 @@ +--- +# RIasC configuration file +# See also: https://erigrid2.github.io/riasc/docs/setup/config + +# A unique hostname to identify the node +hostname: dummyHostname + +ansible: + # List of PGP keys which are used to verify the commits in the Ansible repo + keys: [] + + keyserver: keys.openpgp.org + + # A repository containing ansible playbooks which will be fetched via ansible-pull + url: dummyGitUrl + + verify_commit: false + + # The playbook which should be provision the node + playbook: playbook.yml + + # A path to the Ansible inventory within the repo from above + inventory: inventory/edgeflex/hosts.yml + + # extra_args: + # - --only-if-changed + + # Additional variables which are passed to the Ansible playbook for provisioning + variables: + + # A list of SSH keys which will be added to the 'pi' user + #additional_ssh_keys: + #- "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDOmn+2RUo3XrHmm7h3w647+f6DlV5wXzBJSYLa7NBVlEasBd6Gxp4HDh1iKmNgZrneqXWOAH19P41k1qiwAx8/gUYDlnoah30RWp3qlXXIN+RUCUyUx34tatGTSDynuAyYKAOg0CawK066pSaRsMan2JdhL+r0YAKDXswMU8NVdc32AFTJLrZyGzlrFn4y7hSgHCtOy2RvoULWcjVtcF0GLuSr3WCLUwm1Qy83tft++5FJCcNtg986or5OOeWtZxFU035Q+2Khd2JqyweFTmCMRXEXAzUMv6Lqxuw19qpK4eOOU59oybXlga5yLFZY0CVBK8XVVnA0pkCnabQPHvVJWL7tcDVvzfnd0JdHpQ6oMsYFM7MdncKbHay5/g/tUz34WXPPAPWvciyLkI1NMV++JpaPeMEkAvrsUUZMko+4EQ75ZErUOY3PAJG+2Jilb79QTRMzpCVWfQ0A/Zx1O7Y0sLks+f5VDJaPnHl0vXetFIkEpv9XjomNZgp81m3XRItJh0+BwGqoYtc+MTLelAu0oap1LyHSbpBWGzKxi2tW9VOrX7/pI18f0AhhCc/BG/6UFjMhyYzGj9wg6HWAo1N8hlhdeuNic004hbADX7qzXJMyVEol5AL0rpu3j3j5zTeULY3HxiE+FC6yKg1DKvcxHkmwXoWJYwW9yAgYDyjhbw==" # Manuel Pitz + # Set this to true if you want to login via SSH keys only. + # If you dont have an SSH key, set this to false. + # Important: Dont forget to change the default password after your first login! + disable_password_login: false + + # Replace this token with the token provided by your RIasC provider + token: XXXXX # changeme! diff --git a/common/villas_docker.yaml b/common/villas_docker.yaml new file mode 100644 index 0000000..aa98faa --- /dev/null +++ b/common/villas_docker.yaml @@ -0,0 +1,3 @@ +hostname: villas-box + +ansible: \ No newline at end of file diff --git a/rpi/.gitignore b/rpi/.gitignore index 40b333e..c0fba10 100644 --- a/rpi/.gitignore +++ b/rpi/.gitignore @@ -5,3 +5,4 @@ patch.fish riasc.yaml fallback-ntp.conf keys/ +edgepmu*/ \ No newline at end of file diff --git a/rpi/create_image.sh b/rpi/create_image.sh old mode 100644 new mode 100755 index b07826f..8c94d5c --- a/rpi/create_image.sh +++ b/rpi/create_image.sh @@ -3,31 +3,67 @@ set -e SCRIPT_PATH=$(dirname $(realpath "${BASH_SOURCE[0]}")) -pushd ${SCRIPT_PATH} +cd "${SCRIPT_PATH}" # Settings NODENAME="${NODENAME:-riasc-agent}" TOKEN="${TOKEN:-XXXXX}" -FLAVOR=${FLAVOR:-erigrid} +DOWNLOAD_FOLDER="/tmp/data/download/" +OUTPUT_FOLDER="/tmp/data/output/" +IMG_FOLDER="/tmp/data/images/" +WORKDIR="/tmp/" +REPOFOLDER="/tmp/riasc/" +FLAVOR=${FLAVOR:-raspios} + +if [ ! -d "$DOWNLOAD_FOLDER" ]; then + mkdir ${DOWNLOAD_FOLDER} +fi +if [ ! -d "$OUTPUT_FOLDER" ]; then + mkdir ${OUTPUT_FOLDER} +fi +if [ ! -d "$IMG_FOLDER" ]; then + mkdir ${IMG_FOLDER} +fi case ${FLAVOR} in - edgeflex) + ubuntu24.04) OS="ubuntu" ;; - - erigrid) + ubuntu22.04) + OS="ubuntu" + ;; + ubuntu20.04) + OS="ubuntu" + ;; + raspios) OS="raspios" ;; + *) + echo "Flavor $FLAVOR not known!" + exit 0 + ;; esac -case ${OS} in - ubuntu) +case ${FLAVOR} in + ubuntu20.04) IMAGE_FILE="ubuntu-20.04.2-preinstalled-server-arm64+raspi" IMAGE_SUFFIX="img.xz" IMAGE_URL="https://cdimage.ubuntu.com/releases/20.04.2/release/${IMAGE_FILE}.${IMAGE_SUFFIX}" ;; + ubuntu22.04) + IMAGE_FILE="ubuntu-22.04.4-preinstalled-server-arm64+raspi" + IMAGE_SUFFIX="img.xz" + IMAGE_URL="https://cdimage.ubuntu.com/releases/22.04/release/${IMAGE_FILE}.${IMAGE_SUFFIX}" + ;; + + ubuntu24.04) + IMAGE_FILE="ubuntu-24.04-preinstalled-server-arm64+raspi" + IMAGE_SUFFIX="img.xz" + IMAGE_URL="https://cdimage.ubuntu.com/releases/24.04/release/${IMAGE_FILE}.${IMAGE_SUFFIX}" + ;; + raspios) IMAGE_FILE="2021-05-07-raspios-buster-armhf-lite" IMAGE_SUFFIX="zip" @@ -35,10 +71,10 @@ case ${OS} in ;; esac -RIASC_IMAGE_FILE="$(date +%Y-%m-%d)-riasc-${OS}" +RIASC_IMAGE_FILE="$(date +%Y-%m-%d)-riasc-${NODENAME}${TAG}" function check_command() { - if ! command -v $1 &> /dev/null; then + if ! command -v "$1" &> /dev/null; then echo "$1 could not be found" exit fi @@ -47,6 +83,12 @@ function check_command() { # Show config echo "Using hostname: ${NODENAME}" echo "Using token: ${TOKEN}" +echo "Using flavor: ${FLAVOR}" +echo "Using repo: ${GIT_URL}" +echo "Using branch: ${GIT_BRANCH}" +if [! -z "$VAULT_KEY" ]; then + echo "Using ansible secret ${VAULT_KEY}" +fi # Check that required commands exist echo "Check if required commands are installed..." @@ -59,44 +101,89 @@ check_command xz # apt-get install libguestfs-tools wget unzip zip xz-utils # Download image -if [ ! -f ${IMAGE_FILE}.${IMAGE_SUFFIX} ]; then +cd ${DOWNLOAD_FOLDER} +if [ ! -f "${IMAGE_FILE}"."${IMAGE_SUFFIX}" ]; then echo "Downloading image..." - wget ${IMAGE_URL} + wget \ + --progress=bar:force \ + "${IMAGE_URL}" +else + echo "${IMAGE_FILE}.${IMAGE_SUFFIX} exists skipping download" fi + # Unzip image -if [ ! -f ${IMAGE_FILE}.img ]; then +cd ${IMG_FOLDER} + +if [ ! -f "${IMAGE_FILE}".img ]; then echo "Unzipping image..." case ${IMAGE_SUFFIX} in img.xz) - unxz --keep --threads=0 ${IMAGE_FILE}.${IMAGE_SUFFIX} + unxz --keep --threads=0 ${DOWNLOAD_FOLDER}/"${IMAGE_FILE}"."${IMAGE_SUFFIX}" + mv ${DOWNLOAD_FOLDER}/"${IMAGE_FILE}".img ./ ;; zip) - unzip ${IMAGE_FILE}.${IMAGE_SUFFIX} + unzip "${DOWNLOAD_FOLDER}"/"${IMAGE_FILE}"."${IMAGE_SUFFIX}" ;; esac +else + echo "${IMAGE_FILE}.img exists skipping unpack" fi echo "Copying image..." -cp ${IMAGE_FILE}.img ${RIASC_IMAGE_FILE}.img +cd ${WORKDIR} + +cp ${IMG_FOLDER}/"${IMAGE_FILE}".img "${RIASC_IMAGE_FILE}".img # Prepare config -case ${FLAVOR} in - erigrid) - CONFIG_FILE="riasc.yaml" - ;; - *) - CONFIG_FILE="riasc.${FLAVOR}.yaml" - ;; -esac -cp ../common/${CONFIG_FILE} riasc.yaml + + +CONFIG_FILE="riasc.${OS}.yaml" + +cp "${REPOFOLDER}"/common/${CONFIG_FILE} riasc.yaml # Patch config sed -i \ -e "s/XXXXX/${TOKEN}/g" \ - -e "s/riasc-agent/${NODENAME}/g" \ + -e "s/dummyHostname/${NODENAME}/g" \ riasc.yaml +if [ "${OS}" = "ubuntu" ]; then + cp "${REPOFOLDER}"/rpi/user-data ${WORKDIR}/ + sed -i \ + -e "s/dummyHostname/${NODENAME}/g" \ + user-data +fi + +#Select branch +if [[ -n ${GIT_BRANCH} ]]; then + sed -i \ + -e "/url: /a\\ branch: ${GIT_BRANCH}" \ + riasc.yaml +fi + +#Git url +sed -i \ + -e "s,dummyGitUrl,${GIT_URL},g" \ + riasc.yaml + +#Generate ansible secret +if [ ! -f ${OUTPUT_FOLDER}/"${NODENAME}"-vaultkey.secret ]; then + if [ -z "$VAULT_KEY" ]; then + echo "Generate ansible secret" + VAULT_KEY=$(tr -dc A-Za-z0-9 ${OUTPUT_FOLDER}/"${NODENAME}"-vaultkey.secret +#!/bin/bash +echo "${VAULT_KEY}" +EOF + chmod +x ${OUTPUT_FOLDER}/"${NODENAME}"-vaultkey.secret +else + echo "Skip ansible secret generation use existing key" +fi +cp ${OUTPUT_FOLDER}/"${NODENAME}"-vaultkey.secret ${WORKDIR}/vaultkey.secret + + # Prepare systemd-timesyncd config cat > fallback-ntp.conf < patch.fish echo "Loading image..." @@ -127,39 +214,51 @@ echo "Available space:" df-h echo "Copy files into image..." -copy-in rootfs/etc/ / -copy-in riasc.yaml /boot +copy-in ${REPOFOLDER}/rpi/rootfs/etc/ / +copy-in ${WORKDIR}/riasc.yaml /boot +copy-in ${WORKDIR}/vaultkey.secret /boot mkdir-p /etc/systemd/timesyncd.conf.d/ -copy-in fallback-ntp.conf /etc/systemd/timesyncd.conf.d/ +copy-in ${REPOFOLDER}/rpi/fallback-ntp.conf /etc/systemd/timesyncd.conf.d/ mkdir-p /usr/local/bin -copy-in ../common/riasc-update.sh ../common/riasc-set-hostname.sh /usr/local/bin/ +copy-in ${REPOFOLDER}/common/riasc-update.sh ${REPOFOLDER}/common/riasc-set-hostname.sh /usr/local/bin/ glob chmod 755 /usr/local/bin/riasc-*.sh -copy-in keys/ /boot/ +copy-in ${REPOFOLDER}/rpi/keys/ /boot/ -echo "Enable SSH on boot..." -touch /boot/ssh - -echo "Setting hostname..." -write /etc/hostname "${NODENAME}" +echo "Disable daily APT timers" +rm /etc/systemd/system/timers.target.wants/apt-daily-upgrade.timer +rm /etc/systemd/system/timers.target.wants/apt-daily.timer echo "Updating os-release" write-append /etc/os-release "VARIANT=\"RIasC\"\n" write-append /etc/os-release "BUILD_ID=\"$(date)\"\n" write-append /etc/os-release "DOCUMENTATION_URL=\"https://riasc.eu\"\n" +EOF + +case ${OS} in + ubuntu) +cat <> patch.fish +copy-in ${WORKDIR}/user-data /boot +EOF + ;; + *) +cat <> patch.fish +echo "Enable SSH on boot..." +touch /boot/ssh + +echo "Setting hostname..." +write /etc/hostname "${NODENAME}" echo "Enable systemd risac services..." ln-sf /etc/systemd/system/risac-update.service /etc/systemd/system/multi-user.target.wants/riasc-update.service ln-sf /etc/systemd/system/risac-set-hostname.service /etc/systemd/system/multi-user.target.wants/riasc-set-hostname.service - -echo "Disable daily APT timers" -rm /etc/systemd/system/timers.target.wants/apt-daily-upgrade.timer -rm /etc/systemd/system/timers.target.wants/apt-daily.timer EOF + ;; +esac -if [ "${FLAVOR}" = "edgeflex" -a "${OS}" = "ubuntu" ]; then +if [ "${OS}" = "ubuntu" ]; then cat <> patch.fish echo "Disable Grub boot" write-append /boot/usercfg.txt "[all]\ninitramfs initrd.img followkernel\nkernel=vmlinuz\n" @@ -169,10 +268,16 @@ fi echo "Patching image with guestfish..." guestfish < patch.fish +if [ "${RAW_OUTPOUT}" = "yes" ]; then + echo "Copy raw image..." + cp "${RIASC_IMAGE_FILE}".img ${OUTPUT_FOLDER}/ +fi + # Zip image echo "Zipping image..." -rm -f ${RIASC_IMAGE_FILE}.zip -zip ${RIASC_IMAGE_FILE}.zip ${RIASC_IMAGE_FILE}.img +rm -f "${RIASC_IMAGE_FILE}".zip +zip ${OUTPUT_FOLDER}/"${RIASC_IMAGE_FILE}".zip "${RIASC_IMAGE_FILE}".img +chmod o+w ${OUTPUT_FOLDER}/"${RIASC_IMAGE_FILE}".zip echo "Please write the new image to an SD card:" -echo " dd bs=1M if=${RIASC_IMAGE_FILE}.img of=/dev/sdX" +echo " dd bs=1M if=${RIASC_IMAGE_FILE}.img of=/dev/sdX" \ No newline at end of file diff --git a/rpi/rootfs/etc/systemd/system/flash b/rpi/rootfs/etc/systemd/system/flash new file mode 100644 index 0000000..e69de29 diff --git a/rpi/update_image.sh b/rpi/update_image.sh new file mode 100755 index 0000000..6bd2d67 --- /dev/null +++ b/rpi/update_image.sh @@ -0,0 +1,428 @@ +#!/bin/bash +set -e + + +SCRIPT_PATH=$(dirname $(realpath "${BASH_SOURCE[0]}")) +SCRIPT_OWNER=$(stat -c '%U' ${SCRIPT_PATH}) +pushd ${SCRIPT_PATH} + +#====================== Required Packages ======================== +#jq, ansible-vault (ansible), pass, gpg, curl, guestfish, git +function check_command() { + if ! command -v $1 &> /dev/null; then + echo "$1 could not be found" + exit + fi +} + +check_command jq +check_command ansible-vault +check_command pass +check_command curl +check_command gpg +check_command git +check_command guestfish + +#======================= Predefined settings ===================== +CONFIG_FILE="riasc.edgeflex.yaml" +SSL_SEARCH_PATH="./ssl" +USE_SNMP=false +USE_OVPN=false +ASK_CONFIRM=true +UPDATE=false +DEBUG=false + +GIT_SERVER="git.rwth-aachen.de" +GIT_USE_KEY=false +GIT_MIN_ACCESS_LEVEL=30 + +GIT_ANSIBLE_REPO_NAME="pmu-ansible" +GIT_ANSIBLE_REPO_ID=61980 +GIT_ANSIBLE_REPO="${GIT_SERVER}/acs/public/software/pmu/${GIT_ANSIBLE_REPO_NAME}.git" + +GIT_PASS_REPO_NAME="PMU_pass" +GIT_PASS_REPO_ID=67640 +GIT_PASS_REPO="${GIT_SERVER}/acs/public/software/pmu/${GIT_PASS_REPO_NAME}.git" + +PASS_GPG_DIR="gpg" +PASS_GPG_KEYRING="${PASS_GPG_DIR}/keyring.gpg" +PASS_GPG_OPTIONS="--no-default-keyring --keyring ${PASS_GPG_KEYRING} --homedir ${PASS_GPG_DIR}" + +#======================= Convenience Functions ==================== +function pass_cmd() { + PASSWORD_STORE_DIR="${SCRIPT_PATH}/${NODENAME}/${GIT_PASS_REPO_NAME}" PASSWORD_STORE_GPG_OPTS=${PASS_GPG_OPTIONS} PASSWORD_STORE_CLIP_TIME=1 pass $@ +} + +#========================= Get User input ========================= +usage(){ + echo "Usage:" + echo " -I [Path to Image: -I /path/to/image/]" + echo " -N [Hostname to use: -N name]" + echo " -B [Git branch for ansible repo -B development]" + echo " -C [Path to OpenVPN Cert: -C /path/to/cert]" + echo " -S [Create SNMP config]" + echo " -y [Dont ask for confirmations]" + echo " -d [Debug mode. Dont delete temp files. Dont push to Repos]" + echo " -u [Update mode. Dont delete temp files but push to Repos]" + echo "" + echo "Credentials for ansible/pass repo" + echo " -U [${GIT_SERVER} username -U myName]" #Needed? + echo " -P [${GIT_SERVER} pass/token -P Token]" + exit +} + + +while getopts ":I:N:B:C::SU::P::ydu" opt +do + case "${opt}" in + I) IMAGE_FILE=${OPTARG};; + C) OVPN_CERT_FILE=${OPTARG} ;; + S) USE_SNMP=true ;; + N) NODENAME=${OPTARG} ;; + y) ASK_CONFIRM=false ;; + d) DEBUG=true ;; + u) UPDATE=true ;; + B) PMU_GIT_BRANCH=${OPTARG} ;; + U) GIT_USERNAME=${OPTARG} ;; + P) GIT_PASS=${OPTARG} ;; + *) echo "Unknown argument ${OPTARG}" + usage ;; + :) usage ;; + esac +done + + +#if [ $OPTIND -eq 1 ]; then +# echo "Not enougth options" +# usage; +#fi + +#Ensure RIASC Image file is found +if ! [[ -r ${IMAGE_FILE} ]]; then + echo "Image file '${IMAGE_FILE}' does not exist" + usage +fi + +#Ensure Nodename is supplied +if ! [[ -n ${NODENAME} ]]; then + echo "No node name supplied" + usage +fi + +#Check if OVPN cert file has been supplied and is valid +if [ -z ${OVPN_CERT_FILE} ]; then + USE_OVPN=false +else + USE_OVPN=true + if ! [[ -r ${OVPN_CERT_FILE} ]]; then + echo "OVPN cert file '${OVPN_CERT_FILE}' does not exist" + usage + fi +fi + +#Ensure default config is found +if [[ -r "${SCRIPT_PATH}/../common/${CONFIG_FILE}" ]]; then + CONFIG_PATH="${SCRIPT_PATH}/../common/${CONFIG_FILE}" +else + echo "Could not find default config at:" + echo "${SCRIPT_PATH}/../common/${CONFIG_FILE}" + exit +fi + + +#========================= Check if we can access Git repos ========================= +GIT_API_URL="https://${GIT_SERVER}/api/v4" +GIT_API_AUTH_HEADER="--header 'PRIVATE-TOKEN: ${GIT_PASS}'" + +#Check Permissions on ansible repo via gitlab api +GIT_ANSIBLE_PERM_J=$(curl -s --header "PRIVATE-TOKEN: ${GIT_PASS}" ${GIT_API_URL}/projects/${GIT_ANSIBLE_REPO_ID} | jq -r '.permissions') +if [[ $(echo ${GIT_ANSIBLE_PERM_J} | jq -r '.group_access.access_level') -lt ${GIT_MIN_ACCESS_LEVEL} ]] && [[ $(echo ${GIT_ANSIBLE_PERM_J} | jq -r '.project_access.access_level') -lt ${GIT_MIN_ACCESS_LEVEL} ]]; then + echo "You appear to not have the right permissions for the ${GIT_ANSIBLE_REPO_NAME} repository" + echo "Please make sure that you have an access of at least ${GIT_MIN_ACCESS_LEVEL}" + exit +fi + +#Check Permissions on pass repo via gitlab api +GIT_PASS_PERM_J=$(curl -s --header "PRIVATE-TOKEN: ${GIT_PASS}" ${GIT_API_URL}/projects/${GIT_PASS_REPO_ID} | jq -r '.permissions') +if [[ $(echo ${GIT_PASS_PERM_J} | jq -r '.group_access.access_level') -lt ${GIT_MIN_ACCESS_LEVEL} ]] && [[ $(echo ${GIT_PASS_PERM_J} | jq -r '.project_access.access_level') -lt ${GIT_MIN_ACCESS_LEVEL} ]]; then + echo "You appear to not have the right permissions for the ${GIT_PASS_REPO_NAME} repository" + echo "Please make sure that you have an access of at least ${GIT_MIN_ACCESS_LEVEL}" + exit +fi + +#================================= Confirm settings ================================= +echo "Gathered following configuration:" +echo "Nodename: ${NODENAME}" +echo "Image: ${IMAGE_FILE}" + +if [[ ${USE_OVPN} == true ]]; then + echo "OpenVPN Cert: ${OVPN_CERT_FILE}" +else + echo "Not using OpenVPN" +fi + +echo "Config: ${CONFIG_PATH}" +if [[ ${USE_SNMP} == true ]]; then + echo "Creating SNMP files" +else + echo "No SNMP" +fi + + +echo "Git User: ${GIT_USERNAME}" +if [[ -n ${PMU_GIT_BRANCH} ]]; then + echo "Branch: ${PMU_GIT_BRANCH}" +fi + +if [[ ${DEBUG} = true ]]; then + echo "Running in Debug mode" +fi + +if [[ ${UPDATE} = true ]]; then + echo "Running in Update mode" +fi + +if [[ ${ASK_CONFIRM} == true ]]; then + echo "Continue? (Y,n)" + read inp + if ! [[ ${inp} == "Y" || ${inp} == "" ]]; then + exit + fi +fi + +#============================== Setup to create patch ============================== + +#1. Create temp directory +echo "Creating temporary work directory" +if [[ -d ${NODENAME} ]]; then + echo "Allready exists. Deleting" + rm ${NODENAME} -r +fi +mkdir ${NODENAME} + +#2. Copy files to work directory +echo "Copying files" +NODE_IMAGE_FILE="${NODENAME}_IMAGE" +cp ${IMAGE_FILE} "${NODENAME}/${NODE_IMAGE_FILE}.img" +if [[ ${USE_OVPN} == true ]]; then + cp ${OVPN_CERT_FILE} "${NODENAME}/openvpn.conf" +fi +cp ${CONFIG_PATH} "${NODENAME}/riasc.yaml" +cp "user-data" "${NODENAME}/user-data" +echo "Done" + +#3. Enter working directory +pushd ${NODENAME} + +#4. Make sure repos are here +echo "Cloning GIT repos" +git clone "https://${GIT_USERNAME}:${GIT_PASS}@${GIT_PASS_REPO}" +echo ${PMU_GIT_BRANCH} +git clone "https://${GIT_USERNAME}:${GIT_PASS}@${GIT_ANSIBLE_REPO}" -b ${PMU_GIT_BRANCH:-master} +pass_cmd git init #tell pass we want to create commits when working with passwords +echo "Done" + +#5. Import GPG keys +echo "Setting up GPG keyring / trustdb" +mkdir ${PASS_GPG_DIR} +touch ${PASS_GPG_KEYRING} +chmod 600 ${PASS_GPG_DIR}/* +chmod 700 ${PASS_GPG_DIR} + +gpg ${PASS_GPG_OPTIONS} --import $(ls -1 ${GIT_PASS_REPO_NAME}/keys/*.asc) +for keyfile in $(ls -1 ${GIT_PASS_REPO_NAME}/keys/*.asc| xargs basename -a -s .asc); do + echo "${keyfile}:6:" | gpg ${PASS_GPG_OPTIONS} --import-ownertrust; +done +echo "Done" + +#6. Generate secrets and write to files +echo "Generating secrets" + +#Vault Key +#backup existing password if exists +if [[ -r ${GIT_PASS_REPO_NAME}/${NODENAME}.gpg ]]; then + echo "Backing old PW up" + pass_cmd mv ${NODENAME} "old/${NODENAME}_$(date '+%Y-%m-%d_%H:%M:%S')" +fi + +VAULT_KEY=$(pass_cmd generate ${NODENAME} -n 20 | tail -1 | sed -r "s/\x1B\[([0-9]{1,3}((;[0-9]{1,3})*)?)?[m|K]//g" ) #TODO: this is not a pretty way to do this... +#SED curtesy of: https://gist.github.com/stevenh512/2245881 + +cat < vaultkey.secret +#!/bin/bash +echo "${VAULT_KEY}" +EOF +chmod +x vaultkey.secret + +#Git token +#request git token from API +if [[ ${DEBUG} = false ]]; then + TOKEN_RESP_J=$(curl -s --request POST --header "PRIVATE-TOKEN: ${GIT_PASS}" --header "Content-Type:application/json" --data "{ \"name\":\"${NODENAME}\", \"scopes\":[\"read_repository\"]}" "${GIT_API_URL}/projects/${GIT_ANSIBLE_REPO_ID}/access_tokens") + PMU_GIT_TOKEN=$(echo ${TOKEN_RESP_J} | jq -r '.token') +else + PMU_GIT_TOKEN="TEMPTOKEN" + echo "DID NOT GENERATE TOKEN DUE TO DEBUG MODE" +fi +echo ${PMU_GIT_TOKEN} +if [ -z ${PMU_GIT_TOKEN} ]; then + echo "Error while creating git access token" + exit +fi + +echo ${PMU_GIT_TOKEN} > "git_token.secret" + +#SNMP pass +SNMP_PASS=$(openssl rand -hex 10) + +echo "Done" + +#============================ Edit Files for Boot partition ============================ + +#1. Edit configuration files +echo "Writing configuration" + +sed -i \ + -e "s/exampleHostname/${NODENAME}/g" \ + riasc.yaml + +#Select branch +if [[ -n ${PMU_GIT_BRANCH} ]]; then + sed -i \ + -e "/url: /a\\ branch: ${PMU_GIT_BRANCH}" riasc.yaml +fi + +#Git token +sed -i -e "s/git.rwth-aachen/pmu-acs:${PMU_GIT_TOKEN}@git.rwth-aachen/g" riasc.yaml + +#Node Name +sed -i \ + -e "s/exampleHost/${NODENAME}/g" \ + user-data + +echo "Done" + +#============================== Edit Files in git repo ============================= + +#1. Encrypt with ansible +#OpenVPN +if [[ ${USE_OVPN} == true ]]; then + ansible-vault encrypt --vault-password-file ./vaultkey.secret openvpn.conf --output openvpn.conf.secret +fi +#SNMP +if [[ ${USE_SNMP} == true ]]; then + SNMP_PASS_VAULT=$(ansible-vault encrypt_string --vault-password-file ./vaultkey.secret --name SNMP_PASS ${SNMP_PASS}) +fi + +#2. Write variables to ansible-repo +#make sure host bin exists +HOST_BIN="./${GIT_ANSIBLE_REPO_NAME}/inventory/edgeflex/host_vars/${NODENAME}" +if ! [[ -d ${HOST_BIN} ]]; then + mkdir ${HOST_BIN} +fi + +#Replace SNMP Pass +if [[ ${USE_SNMP} == true ]]; then + cat < ./${GIT_ANSIBLE_REPO_NAME}/inventory/edgeflex/host_vars/${NODENAME}/snmp.yml + ${SNMP_PASS_VAULT} + SNMP_USR: ${NODENAME} +EOF +else + #Loesche alte SNMP config if it exists to not confuse ansible + rm -f ${HOST_BIN}/snmp.yml +fi + +#Replace openVPN config +if [[ ${USE_OVPN} == true ]]; then + cp ./openvpn.conf.secret ${HOST_BIN} +else + rm -f ${HOST_BIN}/openvpn.conf.secret +fi + +#3. Commit and push ansible-repo +pushd ${GIT_ANSIBLE_REPO_NAME} +git add . +git commit -m "Running update_image on $(date) for ${NODENAME}" --author="Update Image Script" +if ! [ ${DEBUG} = true ]; then + git push +else + echo "Didnt push to ansible repo due to debug mode" +fi +popd + +#5. Push to pass repo +pushd ${GIT_PASS_REPO_NAME} +if ! [ ${DEBUG} = true ]; then + pass_cmd git push +else + echo "Didnt push to pass repo due to debug mode" +fi +popd + +#================================== Write to Image ================================== +#1. Write patch file +echo "Writing Patch file" + +cat < edgeflex.fish +echo "Loading image..." +add ${NODE_IMAGE_FILE}.img + +echo "Start virtual environment..." +run + +echo "Available filesystems:" +list-filesystems + +echo "Mounting filesystems..." +mount /dev/sda2 / +mount /dev/sda1 /boot + +echo "Available space:" +df-h + +echo "Copy files into image..." +copy-in riasc.yaml /boot +copy-in user-data /boot +copy-in vaultkey.secret /boot +copy-in git_token.secret /boot +EOF + +if [[ ${USE_OVPN} == true ]]; then + cat <> edgeflex.fish + mkdir /boot/openvpn/ + copy-in openvpn.conf /boot/openvpn +EOF +fi + +#2. Write patch to image +echo "Patching image with guestfish..." +guestfish < edgeflex.fish + +#3. Zip image + +#echo "Zipping image..." +#rm -f ${NODE_IMAGE_FILE}.zip +#zip ${NODE_IMAGE_FILE}.zip ${NODE_IMAGE_FILE}.img +#rm -f ${NODE_IMAGE_FILE}.img +#echo "Done" + + +#4. Clean up +if [[ ${DEBUG} == false ]] && [[ ${UPDATE} == false ]]; then + echo "removing files" + if [[ ${USE_OVPN} == true ]]; then + rm "openvpn.conf" + fi + rm "edgeflex.fish" + rm "riasc.yaml" + rm "user-data" + rm *.secret + rm ${PASS_GPG_DIR} -r + rm ${GIT_PASS_REPO_NAME} -r + rm ${GIT_ANSIBLE_REPO_NAME} -r +fi + + +#4. Final outputs +echo "Please write the new image to an SD card:" +lsblk |grep sd +echo " dd bs=1M if=${NODE_IMAGE_FILE}.img of=/dev/sdX" diff --git a/rpi/user-data b/rpi/user-data new file mode 100644 index 0000000..bf161f2 --- /dev/null +++ b/rpi/user-data @@ -0,0 +1,35 @@ +#cloud-config + +# This is the user-data configuration file for cloud-init. By default this sets +# up an initial user called "ubuntu" with password "ubuntu", which must be +# changed at first login. However, many additional actions can be initiated on +# first boot from this file. The cloud-init documentation has more details: +# +# https://cloudinit.readthedocs.io/ +# +# Some additional examples are provided in comments below the default +# configuration. + +# On first boot, set the (default) ubuntu user's password to "ubuntu" and +# expire user passwords +chpasswd: + expire: true + list: + - ubuntu:ubuntu + +# Enable password authentication with the SSH daemon +ssh_pwauth: true + +locale: C.UTF-8 +timezone: Europe/Berlin +hostname: dummyHostname + +## Run arbitrary commands at rc.local like time +runcmd: + - [ ln, -sf, /etc/systemd/system/risac-update.service, /etc/systemd/system/multi-user.target.wants/riasc-update.service] + +power_state: + mode: reboot + message: Reboot after inital setup + timeout: 20 + condition: True