Release v1.4.2

.github/workflows/deploy-prod.yml

@@ -14,7 +14,7 @@ on:
       core-image-tag:
         description: Core DockerHub image tag
         required: true
-        default: 'v1.4.1'
+        default: 'v1.4.2'
       countryconfig-image-tag:
         description: Your Country Config DockerHub image tag
         required: true

.github/workflows/deploy.yml

@@ -15,7 +15,7 @@ on:
       core-image-tag:
         description: Core DockerHub image tag
         required: true
-        default: 'v1.4.1'
+        default: 'v1.4.2'
       countryconfig-image-tag:
         description: Your Country Config DockerHub image tag
         required: true

.github/workflows/publish-to-dockerhub.yml

@@ -6,6 +6,7 @@ on:
       - master
       - develop
       - main
+      - release-*
   workflow_dispatch:
     inputs:
       branch_name:

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## [1.4.2](https://github.com/opencrvs/opencrvs-farajaland/compare/v1.4.0...v1.4.2)
+
+- Bugfix: fix critical bug stopping backups from being sent to target backup server
+- Bugfix: add a mechanism for controlling the amount of backups stored on the backup server.
+
 ## [1.4.1](https://github.com/opencrvs/opencrvs-farajaland/compare/v1.4.0...v1.4.1)
 
 - Improved logging for emails being sent

infrastructure/backups/backup.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
@@ -38,10 +40,6 @@ for i in "$@"; do
     SSH_PORT="${i#*=}"
     shift
     ;;
-  --production_ip=*)
-    PRODUCTION_IP="${i#*=}"
-    shift
-    ;;
   --remote_dir=*)
     REMOTE_DIR="${i#*=}"
     shift
@@ -63,7 +61,7 @@ for i in "$@"; do
 done
 
 print_usage_and_exit() {
-  echo 'Usage: ./backup.sh --passphrase=XXX --ssh_user=XXX --ssh_host=XXX --ssh_port=XXX --production_ip=XXX --remote_dir=XXX --replicas=XXX --label=XXX'
+  echo 'Usage: ./backup.sh --passphrase=XXX --ssh_user=XXX --ssh_host=XXX --ssh_port=XXX --remote_dir=XXX --replicas=XXX --label=XXX'
   echo "Script must receive SSH details and a target directory of a remote server to copy backup files to."
   echo "Optionally a LABEL i.e. 'v1.0.1' can be provided to be appended to the backup file labels"
   echo "7 days of backup data will be retained in the manager node"
@@ -96,10 +94,6 @@ if [ "$IS_LOCAL" = false ]; then
     echo "Error: Argument for the --ssh_port is required."
     print_usage_and_exit
   fi
-  if [ -z "$PRODUCTION_IP" ]; then
-    echo "Error: Argument for the --production_ip is required."
-    print_usage_and_exit
-  fi
   if [ -z "$REMOTE_DIR" ]; then
     echo "Error: Argument for the --remote_dir is required."
     print_usage_and_exit
@@ -299,47 +293,41 @@ fi
 
 # Copy the backups to an offsite server in production
 #----------------------------------------------------
-if [[ "$OWN_IP" = "$PRODUCTION_IP" || "$OWN_IP" = "$(dig $PRODUCTION_IP +short)" ]]; then
 
-  # Create a temporary directory to store the backup files before packaging
-  BACKUP_RAW_FILES_DIR=/tmp/backup-${LABEL:-$BACKUP_DATE}/
-  mkdir -p $BACKUP_RAW_FILES_DIR
+# Create a temporary directory to store the backup files before packaging
+BACKUP_RAW_FILES_DIR=/tmp/backup-${LABEL:-$BACKUP_DATE}/
+mkdir -p $BACKUP_RAW_FILES_DIR
 
-  # Copy full directories to the temporary directory
-  cp -r $ROOT_PATH/backups/elasticsearch/ $BACKUP_RAW_FILES_DIR/elasticsearch/
-  cp -r $ROOT_PATH/backups/influxdb/${LABEL:-$BACKUP_DATE} $BACKUP_RAW_FILES_DIR/influxdb/
+# Copy full directories to the temporary directory
+cp -r $ROOT_PATH/backups/elasticsearch/ $BACKUP_RAW_FILES_DIR/elasticsearch/
+cp -r $ROOT_PATH/backups/influxdb/${LABEL:-$BACKUP_DATE} $BACKUP_RAW_FILES_DIR/influxdb/
 
 
-  mkdir -p $BACKUP_RAW_FILES_DIR/minio/ && cp $ROOT_PATH/backups/minio/ocrvs-${LABEL:-$BACKUP_DATE}.tar.gz $BACKUP_RAW_FILES_DIR/minio/
-  mkdir -p $BACKUP_RAW_FILES_DIR/metabase/ && cp $ROOT_PATH/backups/metabase/ocrvs-${LABEL:-$BACKUP_DATE}.tar.gz $BACKUP_RAW_FILES_DIR/metabase/
-  mkdir -p $BACKUP_RAW_FILES_DIR/vsexport/ && cp $ROOT_PATH/backups/vsexport/ocrvs-${LABEL:-$BACKUP_DATE}.tar.gz $BACKUP_RAW_FILES_DIR/vsexport/
-  mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/hearth-dev-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
-  mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/user-mgnt-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
-  mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/openhim-dev-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
-  mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/application-config-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
-  mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/metrics-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
-  mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/webhooks-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
-  mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/performance-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
+mkdir -p $BACKUP_RAW_FILES_DIR/minio/ && cp $ROOT_PATH/backups/minio/ocrvs-${LABEL:-$BACKUP_DATE}.tar.gz $BACKUP_RAW_FILES_DIR/minio/
+mkdir -p $BACKUP_RAW_FILES_DIR/metabase/ && cp $ROOT_PATH/backups/metabase/ocrvs-${LABEL:-$BACKUP_DATE}.tar.gz $BACKUP_RAW_FILES_DIR/metabase/
+mkdir -p $BACKUP_RAW_FILES_DIR/vsexport/ && cp $ROOT_PATH/backups/vsexport/ocrvs-${LABEL:-$BACKUP_DATE}.tar.gz $BACKUP_RAW_FILES_DIR/vsexport/
+mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/hearth-dev-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
+mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/user-mgnt-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
+mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/openhim-dev-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
+mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/application-config-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
+mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/metrics-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
+mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/webhooks-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
+mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/performance-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
 
-  tar -czf /tmp/${LABEL:-$BACKUP_DATE}.tar.gz -C "$BACKUP_RAW_FILES_DIR" .
+tar -czf /tmp/${LABEL:-$BACKUP_DATE}.tar.gz -C "$BACKUP_RAW_FILES_DIR" .
 
-  openssl enc -aes-256-cbc -salt -pbkdf2 -in /tmp/${LABEL:-$BACKUP_DATE}.tar.gz -out /tmp/${LABEL:-$BACKUP_DATE}.tar.gz.enc -pass pass:$PASSPHRASE
+openssl enc -aes-256-cbc -salt -pbkdf2 -in /tmp/${LABEL:-$BACKUP_DATE}.tar.gz -out /tmp/${LABEL:-$BACKUP_DATE}.tar.gz.enc -pass pass:$PASSPHRASE
 
+if [ "$IS_LOCAL" = false ]; then
+  set +e
   rsync -a -r --rsync-path="mkdir -p $REMOTE_DIR/ && rsync" --progress --rsh="ssh -o StrictHostKeyChecking=no -p $SSH_PORT" /tmp/${LABEL:-$BACKUP_DATE}.tar.gz.enc $SSH_USER@$SSH_HOST:$REMOTE_DIR/
-
-  echo "Copied backup files to remote server."
-
-  rm /tmp/${LABEL:-$BACKUP_DATE}.tar.gz.enc
-  rm /tmp/${LABEL:-$BACKUP_DATE}.tar.gz
-  rm -r $BACKUP_RAW_FILES_DIR
+  if [ $? -eq 0 ]; then
+    echo "Copied backup files to remote server."
+  fi
+  set -e
 fi
 
-# Cleanup any old backups from influx or mongo. Keep previous 7 days of data and all elastic data
-# Elastic snapshots require a random selection of files in the data/backups/elasticsearch/indices
-# folder
-#------------------------------------------------------------------------------------------------
-find $ROOT_PATH/backups/influxdb -mtime +7 -exec rm {} \;
-find $ROOT_PATH/backups/mongo -mtime +7 -exec rm {} \;
-find $ROOT_PATH/backups/minio -mtime +7 -exec rm {} \;
-find $ROOT_PATH/backups/metabase -mtime +7 -exec rm {} \;
-find $ROOT_PATH/backups/vsexport -mtime +7 -exec rm {} \;
+rm /tmp/${LABEL:-$BACKUP_DATE}.tar.gz.enc
+rm /tmp/${LABEL:-$BACKUP_DATE}.tar.gz
+rm -r $BACKUP_RAW_FILES_DIR
+

infrastructure/backups/download.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
@@ -9,7 +11,7 @@
 
 #------------------------------------------------------------------------------------------------------------------
 # By default OpenCRVS saves a backup of all data on a cron job every day in case of an emergency data loss incident
-# This script clears all data and restores a specific day's data.  It is irreversable, so use with caution.
+# This script downloads all the data based on --label (defaults to current day)
 #------------------------------------------------------------------------------------------------------------------
 
 set -e
@@ -28,10 +30,6 @@ for i in "$@"; do
     SSH_PORT="${i#*=}"
     shift
     ;;
-  --replicas=*)
-    REPLICAS="${i#*=}"
-    shift
-    ;;
   --label=*)
     LABEL="${i#*=}"
     shift
@@ -100,8 +98,8 @@ openssl enc -d -aes-256-cbc -salt -pbkdf2 -in $BACKUP_RAW_FILES_DIR/${LABEL}.tar
 mkdir -p $BACKUP_RAW_FILES_DIR/extract
 tar -xvf $BACKUP_RAW_FILES_DIR/${LABEL}.tar.gz -C $BACKUP_RAW_FILES_DIR/extract
 
-# Move folders
-rm -r /data/backups/elasticsearch
+# Delete previous days restore(s) and move the newly downloaded one in place
+rm -rf /data/backups/*
 mv $BACKUP_RAW_FILES_DIR/extract/elasticsearch /data/backups/elasticsearch
 
 mv $BACKUP_RAW_FILES_DIR/extract/influxdb /data/backups/influxdb/${LABEL}

infrastructure/backups/rotate_backups.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# OpenCRVS is also distributed under the terms of the Civil Registration
+# & Healthcare Disclaimer located at http://opencrvs.org/license.
+#
+# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
+
+#------------------------------------------------------------------------------------------------------------------
+# By default OpenCRVS saves a backup of all data on a cron job every day in case of an emergency data loss incident
+# This script downloads all the data based on --label (defaults to current day)
+#------------------------------------------------------------------------------------------------------------------
+
+set -e
+
+print_usage_and_exit() {
+  echo 'Usage: ./rotate_backups.sh --backup_dir=/home/backup/backups --amount_to_keep=7'
+  exit 1
+}
+
+for i in "$@"; do
+  case $i in
+  --backup_dir=*)
+    BACKUP_DIR="${i#*=}"
+    shift
+    ;;
+  --amount_to_keep=*)
+    AMOUNT_TO_KEEP="${i#*=}"
+    shift
+    ;;
+  *) ;;
+  esac
+done
+
+
+if ! [[ "$AMOUNT_TO_KEEP" =~ ^[0-9]+$ ]]; then
+  echo "Script must be passed a positive integer number of backups to keep, got $AMOUNT_TO_KEEP"
+  print_usage_and_exit
+fi
+
+BACKUP_DIR=${BACKUP_DIR:-/home/backup/backups}
+
+if [ ! -d "$BACKUP_DIR" ]; then
+    echo "Error: BACKUP_DIR ($BACKUP_DIR) doesn't exist"
+    print_usage_and_exit
+fi
+
+# Delete subdirectories but keep latest according to AMOUNT_TO_KEEP
+find "$BACKUP_DIR" -mindepth 1 -type d -print | sort -r | tail -n +$(("$AMOUNT_TO_KEEP" + 1)) | xargs rm -rf --

infrastructure/docker-compose.staging-deploy.yml

@@ -195,10 +195,10 @@ services:
     #   - --tls.stores.default.defaultcertificate.keyfile=/certs/crvs.cm.key
 
     environment:
-      - GOOGLE_DOMAINS_ACCESS_TOKEN=${GOOGLE_DOMAINS_ACCESS_TOKEN}
+      - CLOUDFLARE_DNS_API_TOKEN=${CLOUDFLARE_DNS_API_TOKEN}
     command:
       - --certificatesresolvers.certResolver.acme.dnschallenge=true
-      - --certificatesresolvers.certResolver.acme.dnschallenge.provider=googledomains
+      - --certificatesresolvers.certResolver.acme.dnschallenge.provider=cloudflare
       - --certificatesresolvers.certResolver.acme.email=riku@opencrvs.org
       - --certificatesresolvers.certResolver.acme.storage=acme.json
 

infrastructure/logrotate.conf

@@ -44,7 +44,7 @@ include /etc/logrotate.d
 
 # system-specific logs may be configured here
 
-/var/log/opencrvs-backup.log {
+/var/log/opencrvs-rotate-backups.log {
     missingok
     monthly
     create 0660 root application

infrastructure/server-setup/backups.yml

@@ -86,6 +86,7 @@
   become_method: sudo
   vars:
     manager_hostname: "{{ groups['docker-manager-first'][0] }}"
+    crontab_user: root
   tasks:
     - name: Ensure backup user is present
       user:
@@ -136,3 +137,23 @@
         owner: '{{ external_backup_server_user }}'
       tags:
         - backups
+
+    - name: Copy rotate_backups.sh file to external_backup_server_user's home directory
+      copy:
+        src: ../backups/rotate_backups.sh
+        dest: '{{ external_backup_server_user_home }}/rotate_backups.sh'
+        owner: '{{ external_backup_server_user }}'
+        mode: 0755
+      tags:
+        - backups
+
+    - name: 'Setup backup rotation'
+      cron:
+        user: '{{ crontab_user }}'
+        name: 'rotate backups'
+        minute: '0'
+        hour: '0'
+        job: 'bash {{ external_backup_server_user_home }}/rotate_backups.sh --backup_dir={{ external_backup_server_remote_directory }} --amount_to_keep={{ amount_of_backups_to_keep }} >> /var/log/opencrvs-rotate-backups.log 2>&1'
+        state: "{{ 'present' if (amount_of_backups_to_keep) else 'absent' }}"
+      tags:
+        - backups

infrastructure/server-setup/group_vars/all.yml

@@ -10,5 +10,4 @@ ansible_python_interpreter: /usr/bin/python3
 encrypt_data: False
 swap_file_path: /swapfile
 swap_file_size_mb: 8000
-external_backup_server_remote_directory: /home/backup/backups
 external_backup_server_user: 'backup'

infrastructure/server-setup/production.yml

@@ -12,11 +12,13 @@ all:
     # This configuration variable blocks all access to the server, including SSH, except from the IP addresses specified below.
     # This should always be set when configuring a production server if there is no other firewall in front of the server.
     # SSH and other services should never be exposed to the public internet.
+    only_allow_access_from_addresses:
+      - 165.22.110.53
+    # Enable backups
     enable_backups: true
+    external_backup_server_remote_directory: /home/backup/backups
     # external_backup_server_ssh_port: Defined in --extra-vars by the provisioning pipeline
     # external_backup_server_ip: Defined in --extra-vars by the provisioning pipeline
-    only_allow_access_from_addresses:
-      - 165.22.110.53
     users:
       # If you need to remove access from someone, do not remove them from this list, but instead set their state: absent
       - name: riku
@@ -50,3 +52,5 @@ backups:
   hosts:
     farajaland-qa:
       ansible_host: '165.22.110.53'
+      # @todo how many days to store backups for?
+      amount_of_backups_to_keep: 3

infrastructure/server-setup/staging.yml

@@ -13,7 +13,9 @@ all:
     # SSH and other services should never be exposed to the public internet.
     only_allow_access_from_addresses:
       - 165.22.110.53
-    enable_backups: false
+    # Enable backups but write them to a different location from where production writes them
+    enable_backups: true
+    external_backup_server_remote_directory: /home/backup/staging-backups
     periodic_restore_from_backup: true
     # external_backup_server_ssh_port: Defined in --extra-vars by the provisioning pipeline
     # external_backup_server_ip: Defined in --extra-vars by the provisioning pipeline
@@ -52,3 +54,5 @@ backups:
   hosts:
     farajaland-qa:
       ansible_host: '165.22.110.53'
+      # @todo how many days to store backups for?
+      amount_of_backups_to_keep: 3

infrastructure/server-setup/tasks/backups/crontab.yml

@@ -1,10 +1,17 @@
+- name: Copy backups.sh file to external_backup_server_user's home directory
+  copy:
+    src: ../backups/backup.sh
+    dest: '{{ crontab_user_home }}/backup.sh'
+    owner: 'root'
+    mode: 0755
+
 - name: 'Setup crontab to backup the opencrvs data'
   cron:
     user: '{{ crontab_user }}'
     name: 'backup opencrvs'
     minute: '0'
     hour: '0'
-    job: 'cd / && bash /opt/opencrvs/infrastructure/backups/backup.sh --passphrase={{ backup_encryption_passphrase }} --ssh_user={{ external_backup_server_user }} --ssh_host={{ external_backup_server_ip }} --ssh_port={{ external_backup_server_ssh_port }} --production_ip={{ manager_production_server_ip }} --remote_dir={{ external_backup_server_remote_directory }} --replicas=1 >> /var/log/opencrvs-backup.log 2>&1'
+    job: 'bash {{ crontab_user_home }}/backup.sh --passphrase={{ backup_encryption_passphrase }} --ssh_user={{ external_backup_server_user }} --ssh_host={{ external_backup_server_ip }} --ssh_port={{ external_backup_server_ssh_port }} --remote_dir={{ external_backup_server_remote_directory }} --replicas=1 >> /var/log/opencrvs-backup.log 2>&1'
     state: "{{ 'present' if (external_backup_server_ip is defined and backup_encryption_passphrase and (enable_backups | default(false))) else 'absent' }}"
 
 ##

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencrvs/countryconfig",
-  "version": "1.4.1",
+  "version": "1.4.2",
   "description": "OpenCRVS country configuration for reference data",
   "license": "MPL-2.0",
   "husky": {