From cae68a17edc4c5b87ec281f2b8420eefca7f5837 Mon Sep 17 00:00:00 2001 From: Shamzic Date: Wed, 14 Aug 2024 18:53:04 +0200 Subject: [PATCH 1/6] feat: ajoute un wrapper pour capturer les erreurs des crons sur sentry --- roles/bootstrap/tasks/setup_cron.yaml | 12 ++++++---- scripts/sentry_wrapper_cron.sh | 33 +++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) create mode 100755 scripts/sentry_wrapper_cron.sh diff --git a/roles/bootstrap/tasks/setup_cron.yaml b/roles/bootstrap/tasks/setup_cron.yaml index 7a2bdc81..4c765f16 100644 --- a/roles/bootstrap/tasks/setup_cron.yaml +++ b/roles/bootstrap/tasks/setup_cron.yaml @@ -2,13 +2,15 @@ - name: Set cron env variables ansible.builtin.set_fact: envvar_prefix: NODE_ENV=production MONGODB_URL=mongodb://127.0.0.1/db_{{ item.name }} + env_file_path: "{{ repository_folder }}/.env" + wrapper_script_path: "{{ playbook_dir }}/script/sentry_wrapper_cron.sh" - name: Add a cron job for stats generation become_user: "{{ server_user_name }}" ansible.builtin.cron: name: generate stats for {{ item.name }} minute: "23" hour: "2" - job: ({{ envvar_prefix }} /usr/bin/node {{ repository_folder }}/dist-server/backend/lib/stats) + job: ({{ wrapper_script_path }} {{ env_file_path }} {{ envvar_prefix }} /usr/bin/node {{ repository_folder }}/dist-server/backend/lib/stats) - name: Add a cron job to send initial survey emails become_user: "{{ server_user_name }}" ansible.builtin.cron: @@ -17,7 +19,7 @@ hour: "4" job: >- (cd {{ repository_folder }} && - {{ envvar_prefix }} /usr/bin/node + {{ wrapper_script_path }} {{ env_file_path }} {{ envvar_prefix }} /usr/bin/node ./dist-server/tools/email-sending-tool.js send initial-survey --multiple 1000 >> /var/log/{{ server_user_name }}/{{ item.name }}_emails.log 2>&1) - name: Add a cron job to send initial survey by sms @@ -28,7 +30,7 @@ hour: "17" job: >- (cd {{ repository_folder }} && - {{ envvar_prefix }} /usr/bin/node + {{ wrapper_script_path }} {{ env_file_path }} {{ envvar_prefix }} /usr/bin/node ./dist-server/tools/sms-sending-tool.js send initial-survey --multiple 100 >> /var/log/{{ server_user_name }}/{{ item.name }}_sms.log 2>&1) - name: Add a cron job to anonymize Simulation and Followup data collections @@ -37,7 +39,7 @@ name: anonymize simulation and followup data collections for {{ item.name }} minute: "0" hour: "5" - job: (cd {{ repository_folder }} && {{ envvar_prefix }} npm run tools:cleaner) + job: (cd {{ repository_folder }} && {{ wrapper_script_path }} {{ env_file_path }} {{ envvar_prefix }} npm run tools:cleaner) - name: Create cron job to execute generate_mongo_stats.sh monthly become_user: "{{ server_user_name }}" ansible.builtin.cron: @@ -45,4 +47,4 @@ minute: "0" hour: "0" day: "1" - job: (cd {{ repository_folder }} && {{ envvar_prefix }} npm run tools:generate-mongo-stats) + job: (cd {{ repository_folder }} && {{ wrapper_script_path }} {{ env_file_path }} {{ envvar_prefix }} npm run tools:generate-mongo-stats) diff --git a/scripts/sentry_wrapper_cron.sh b/scripts/sentry_wrapper_cron.sh new file mode 100755 index 00000000..6f093648 --- /dev/null +++ b/scripts/sentry_wrapper_cron.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# This script is executing the command passed as 2nd argument +# and send an event to Sentry if the command fails. +# The .env file path is passed as the first argument. + + +ENV_FILE_PATH=$1 +shift # Remove the first argument +if [ -z "$ENV_FILE_PATH" ]; then + echo "Error: .env file path is missing" + exit 1 +fi +source $ENV_FILE_PATH + +if [ -z "$SENTRY_CRON_DSN" ]; then + echo "Error: SENTRY_CRON_DSN is not set in .env file" + exit 1 +fi + +echo "SENTRY_CRON_DSN is set to: $SENTRY_CRON_DSN" +export SENTRY_DSN=$SENTRY_CRON_DSN + +# The wrapped command is executed and the result status is stored +"$@" +STATUS=$? + +if [ $STATUS -ne 0 ]; then + echo "Cron job failed: $@" + sentry-cli send-event -m "Cron job failed: $@" +fi + +exit $STATUS \ No newline at end of file From 1c6327f5580bdfc1dc9b886027a42b54dd7d4850 Mon Sep 17 00:00:00 2001 From: Shamzic Date: Thu, 15 Aug 2024 20:05:51 +0200 Subject: [PATCH 2/6] feat: add sentry-cli --- roles/bootstrap/tasks/initialize_server.yaml | 4 ++++ roles/bootstrap/tasks/setup_application.yaml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/roles/bootstrap/tasks/initialize_server.yaml b/roles/bootstrap/tasks/initialize_server.yaml index 8ee8097d..07c0086c 100644 --- a/roles/bootstrap/tasks/initialize_server.yaml +++ b/roles/bootstrap/tasks/initialize_server.yaml @@ -15,6 +15,10 @@ ansible.builtin.apt: name: [rsync, python3-pip, build-essential, git, man, ntp] state: present +- name: Install sentry-cli + ansible.builtin.pip: + name: sentry-sdk + state: present - name: Add gnupg in order to add PPA with apt # required for node, python, mongo ansible.builtin.apt: name: [gnupg] diff --git a/roles/bootstrap/tasks/setup_application.yaml b/roles/bootstrap/tasks/setup_application.yaml index 72fd5000..c40ac1d7 100644 --- a/roles/bootstrap/tasks/setup_application.yaml +++ b/roles/bootstrap/tasks/setup_application.yaml @@ -48,3 +48,5 @@ mode: "0600" - name: Add cron for application ansible.builtin.include_tasks: setup_cron.yaml + tags: + - setup_cron From a86e3bd7aaed50f0985f10569f75d3e12a74ee62 Mon Sep 17 00:00:00 2001 From: Shamzic Date: Thu, 15 Aug 2024 23:13:03 +0200 Subject: [PATCH 3/6] fix: indentation --- roles/bootstrap/tasks/setup_application.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/bootstrap/tasks/setup_application.yaml b/roles/bootstrap/tasks/setup_application.yaml index c40ac1d7..72fd5000 100644 --- a/roles/bootstrap/tasks/setup_application.yaml +++ b/roles/bootstrap/tasks/setup_application.yaml @@ -48,5 +48,3 @@ mode: "0600" - name: Add cron for application ansible.builtin.include_tasks: setup_cron.yaml - tags: - - setup_cron From e4b2f9fbc48b9e4ef1eb01ad748d90139429ed87 Mon Sep 17 00:00:00 2001 From: Shamzic Date: Fri, 16 Aug 2024 01:09:20 +0200 Subject: [PATCH 4/6] fix: sentry-cli install --- roles/bootstrap/tasks/initialize_server.yaml | 4 ---- roles/bootstrap/tasks/install_sentry_cli.yaml | 5 +++++ roles/bootstrap/tasks/main.yaml | 4 ++++ 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 roles/bootstrap/tasks/install_sentry_cli.yaml diff --git a/roles/bootstrap/tasks/initialize_server.yaml b/roles/bootstrap/tasks/initialize_server.yaml index 07c0086c..8ee8097d 100644 --- a/roles/bootstrap/tasks/initialize_server.yaml +++ b/roles/bootstrap/tasks/initialize_server.yaml @@ -15,10 +15,6 @@ ansible.builtin.apt: name: [rsync, python3-pip, build-essential, git, man, ntp] state: present -- name: Install sentry-cli - ansible.builtin.pip: - name: sentry-sdk - state: present - name: Add gnupg in order to add PPA with apt # required for node, python, mongo ansible.builtin.apt: name: [gnupg] diff --git a/roles/bootstrap/tasks/install_sentry_cli.yaml b/roles/bootstrap/tasks/install_sentry_cli.yaml new file mode 100644 index 00000000..8450dab2 --- /dev/null +++ b/roles/bootstrap/tasks/install_sentry_cli.yaml @@ -0,0 +1,5 @@ +--- +- name: Install @sentry/cli using npm + ansible.builtin.npm: + name: '@sentry/cli' + global: yes \ No newline at end of file diff --git a/roles/bootstrap/tasks/main.yaml b/roles/bootstrap/tasks/main.yaml index 0fd36cde..586e5e9a 100644 --- a/roles/bootstrap/tasks/main.yaml +++ b/roles/bootstrap/tasks/main.yaml @@ -11,6 +11,10 @@ - name: Install nodeJS 18.x ansible.builtin.include_tasks: install_node.yaml +## setup sentry-cli +- name: Install sentry-cli + ansible.builtin.include_tasks: install_sentry_cli.yaml + ## Setup pm2 - name: Install and setup pm2 ansible.builtin.include_tasks: install_pm2.yaml From 83da09f58a54c5a118701491b27e381f5045b866 Mon Sep 17 00:00:00 2001 From: Shamzic Date: Fri, 16 Aug 2024 01:09:43 +0200 Subject: [PATCH 5/6] fix: sentry_wrapper_cron path --- roles/bootstrap/tasks/setup_cron.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bootstrap/tasks/setup_cron.yaml b/roles/bootstrap/tasks/setup_cron.yaml index 4c765f16..5b1de000 100644 --- a/roles/bootstrap/tasks/setup_cron.yaml +++ b/roles/bootstrap/tasks/setup_cron.yaml @@ -3,7 +3,7 @@ ansible.builtin.set_fact: envvar_prefix: NODE_ENV=production MONGODB_URL=mongodb://127.0.0.1/db_{{ item.name }} env_file_path: "{{ repository_folder }}/.env" - wrapper_script_path: "{{ playbook_dir }}/script/sentry_wrapper_cron.sh" + wrapper_script_path: "/opt/mes-aides/scripts/sentry_wrapper_cron.sh" - name: Add a cron job for stats generation become_user: "{{ server_user_name }}" ansible.builtin.cron: From 0938f0f3988ac87d77c08a9ae51306f61de01d9a Mon Sep 17 00:00:00 2001 From: Shamzic Date: Fri, 16 Aug 2024 11:57:26 +0200 Subject: [PATCH 6/6] fix: sentry wrapper fonctionnel --- roles/bootstrap/tasks/install_sentry_cli.yaml | 4 ++-- roles/bootstrap/tasks/setup_cron.yaml | 10 +++++----- scripts/sentry_wrapper_cron.sh | 11 ++++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/roles/bootstrap/tasks/install_sentry_cli.yaml b/roles/bootstrap/tasks/install_sentry_cli.yaml index 8450dab2..47da610a 100644 --- a/roles/bootstrap/tasks/install_sentry_cli.yaml +++ b/roles/bootstrap/tasks/install_sentry_cli.yaml @@ -1,5 +1,5 @@ --- - name: Install @sentry/cli using npm - ansible.builtin.npm: + community.general.npm: name: '@sentry/cli' - global: yes \ No newline at end of file + global: true diff --git a/roles/bootstrap/tasks/setup_cron.yaml b/roles/bootstrap/tasks/setup_cron.yaml index 5b1de000..e0d0e9c4 100644 --- a/roles/bootstrap/tasks/setup_cron.yaml +++ b/roles/bootstrap/tasks/setup_cron.yaml @@ -10,7 +10,7 @@ name: generate stats for {{ item.name }} minute: "23" hour: "2" - job: ({{ wrapper_script_path }} {{ env_file_path }} {{ envvar_prefix }} /usr/bin/node {{ repository_folder }}/dist-server/backend/lib/stats) + job: ({{ envvar_prefix }} {{ wrapper_script_path }} {{ env_file_path }} /usr/bin/node {{ repository_folder }}/dist-server/backend/lib/stats) - name: Add a cron job to send initial survey emails become_user: "{{ server_user_name }}" ansible.builtin.cron: @@ -19,7 +19,7 @@ hour: "4" job: >- (cd {{ repository_folder }} && - {{ wrapper_script_path }} {{ env_file_path }} {{ envvar_prefix }} /usr/bin/node + {{ envvar_prefix }} {{ wrapper_script_path }} {{ env_file_path }} /usr/bin/node ./dist-server/tools/email-sending-tool.js send initial-survey --multiple 1000 >> /var/log/{{ server_user_name }}/{{ item.name }}_emails.log 2>&1) - name: Add a cron job to send initial survey by sms @@ -30,7 +30,7 @@ hour: "17" job: >- (cd {{ repository_folder }} && - {{ wrapper_script_path }} {{ env_file_path }} {{ envvar_prefix }} /usr/bin/node + {{ envvar_prefix }} {{ wrapper_script_path }} {{ env_file_path }} /usr/bin/node ./dist-server/tools/sms-sending-tool.js send initial-survey --multiple 100 >> /var/log/{{ server_user_name }}/{{ item.name }}_sms.log 2>&1) - name: Add a cron job to anonymize Simulation and Followup data collections @@ -39,7 +39,7 @@ name: anonymize simulation and followup data collections for {{ item.name }} minute: "0" hour: "5" - job: (cd {{ repository_folder }} && {{ wrapper_script_path }} {{ env_file_path }} {{ envvar_prefix }} npm run tools:cleaner) + job: (cd {{ repository_folder }} && {{ envvar_prefix }} {{ wrapper_script_path }} {{ env_file_path }} npm run tools:cleaner) - name: Create cron job to execute generate_mongo_stats.sh monthly become_user: "{{ server_user_name }}" ansible.builtin.cron: @@ -47,4 +47,4 @@ minute: "0" hour: "0" day: "1" - job: (cd {{ repository_folder }} && {{ wrapper_script_path }} {{ env_file_path }} {{ envvar_prefix }} npm run tools:generate-mongo-stats) + job: (cd {{ repository_folder }} && {{ envvar_prefix }} {{ wrapper_script_path }} {{ env_file_path }} npm run tools:generate-mongo-stats) diff --git a/scripts/sentry_wrapper_cron.sh b/scripts/sentry_wrapper_cron.sh index 6f093648..1d157df3 100755 --- a/scripts/sentry_wrapper_cron.sh +++ b/scripts/sentry_wrapper_cron.sh @@ -19,15 +19,16 @@ if [ -z "$SENTRY_CRON_DSN" ]; then fi echo "SENTRY_CRON_DSN is set to: $SENTRY_CRON_DSN" -export SENTRY_DSN=$SENTRY_CRON_DSN # The wrapped command is executed and the result status is stored -"$@" +COMMAND="$@" +OUTPUT=$(eval "$COMMAND" 2>&1) STATUS=$? if [ $STATUS -ne 0 ]; then - echo "Cron job failed: $@" - sentry-cli send-event -m "Cron job failed: $@" + echo "Cron job failed: $COMMAND" + echo "Error output: $OUTPUT" + SENTRY_DSN="$SENTRY_CRON_DSN" sentry-cli send-event -m "Cron job failed: $COMMAND | Error: $OUTPUT" fi -exit $STATUS \ No newline at end of file +exit $STATUS