From 0d173a0625e81615da8301a673942c61d37c5ba3 Mon Sep 17 00:00:00 2001 From: Scott Mebberson <74628+smebberson@users.noreply.github.com> Date: Fri, 5 Jul 2024 14:46:57 +0930 Subject: [PATCH] Refactored c mongo download in bash and Node.js. --- bin/c-mongo-download | 54 ++++++++++++ bin/c-mongo-download-config.js | 69 +++++++++++++++ bin/c-mongo-download.js | 156 --------------------------------- 3 files changed, 123 insertions(+), 156 deletions(-) create mode 100755 bin/c-mongo-download create mode 100644 bin/c-mongo-download-config.js delete mode 100755 bin/c-mongo-download.js diff --git a/bin/c-mongo-download b/bin/c-mongo-download new file mode 100755 index 0000000..3aef3d8 --- /dev/null +++ b/bin/c-mongo-download @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +set -e; + +SCRIPT_DIR=$(dirname "$0") + +report_error() { + echo "Error: $1" >&2 + exit 1 +} + +usage() { + printf "\n\tUsage: c mongo download [collection]\n\n" + printf "\tDownload all or a specific collection from a Mongo database.\n" + printf "\tIf you don't provide a collection, all will be downloaded.\n\n" + printf "\tOptions:\n\n" + printf "\t-h, --help Display this help message\n\n" +} + +if [ "$#" -lt 1 ]; then + usage + exit 1 +fi + +if [ "$1" == "-h" ] || [ "$1" == "--help" ]; then + usage + exit 0 +fi + +ENV=$1 +COLLECTION=$2 + +# Disallow "local" environment +if [ "$ENV" == "local" ]; then + report_error "You cannot download the local database" +fi + +CONFIG=$(node "$SCRIPT_DIR/c-mongo-download-config.js" "$ENV" "$COLLECTION") + +DOCKER_VOLUME_NAME=$(echo $CONFIG | jq -r '.dockerVolumeName') +OUTPUT_PATH="/data/$(echo $CONFIG | jq -r '.dbName')" +CONNECTION_STRING=$(echo $CONFIG | jq -r '.connectionString') + +docker volume create "$DOCKER_VOLUME_NAME" >/dev/null 2>&1 + +docker run --rm -v "$DOCKER_VOLUME_NAME:/data" busybox sh -c "mkdir -p $OUTPUT_PATH && chmod -R 777 /data" + +docker run --rm -v "$DOCKER_VOLUME_NAME:/data" mongo:7 mongodump $CONNECTION_STRING -o $OUTPUT_PATH + +docker run --rm -v "$DOCKER_VOLUME_NAME:/data" -v "$(pwd)/data:/host_data" busybox sh -c "cp -r /data/* /host_data" + +docker volume rm "$DOCKER_VOLUME_NAME" >/dev/null 2>&1 + +sudo chown -R "$(id -u):$(id -g)" "$(pwd)/data" diff --git a/bin/c-mongo-download-config.js b/bin/c-mongo-download-config.js new file mode 100644 index 0000000..9330950 --- /dev/null +++ b/bin/c-mongo-download-config.js @@ -0,0 +1,69 @@ +const { loadConfig } = require('./lib/c'); +const { connectionParts } = require('./lib/c-mongo'); + +const connectionStringWithAddress = ({ + address, + name, + params, + password, + user, +}) => { + const url = new URL(address); + + url.password = password; + url.username = user; + + const cmd = []; + + if (params) { + cmd.push(params); + } + + return [...cmd, '--uri', `${url.href}/${name}`].join(' '); +}; + +const connectionStringWithHost = ({ auth, host, name, params }) => { + const cmd = []; + + if (auth) { + cmd.push(auth); + } + + if (params) { + cmd.push(params); + } + + return [...cmd, '--host', host, '-d', name].join(' '); +}; + +const volumeName = ({ collection, env, name }) => { + const parts = ['mongodump_data', name, env]; + + if (collection) { + parts.push(collection); + } + + return parts.join('_'); +}; + +(async () => { + const [, , env, collection] = process.argv; + const details = await loadConfig(`mongo.${env}`); + const connection = connectionParts(details); + const dockerVolumeName = volumeName({ + collection, + env, + name: details.name, + }); + const connectionString = connection.host + ? connectionStringWithHost(connection) + : connectionStringWithAddress(connection); + + console.log( + JSON.stringify({ + dbName: details.name, + dockerVolumeName, + connectionString, + }) + ); +})(); diff --git a/bin/c-mongo-download.js b/bin/c-mongo-download.js deleted file mode 100755 index f5c01c9..0000000 --- a/bin/c-mongo-download.js +++ /dev/null @@ -1,156 +0,0 @@ -'use strict'; - -const { $ } = require('zx'); - -const program = require('commander'); -const { loadConfig, reportError } = require('./lib/c'); -const { connectionParts } = require('./lib/c-mongo'); - -$.shell = '/usr/bin/zsh'; - -// The basic program, which uses sub-commands. -program - .arguments('[env]') - .arguments('') - .description( - "Download all or a specific collection from a Mongo database. If you don't provide a collection, all will be downloaded." - ) - .parse(process.argv); - -if (!program.args.length) { - return reportError(new Error('Please provide an environment'), program); -} - -const [env, collection] = program.args; - -if (env.toLowerCase() === 'local') { - return reportError( - new Error('You cannot download the local database'), - program - ); -} - -const connectionStringWithAddress = ({ - address, - name, - params, - password, - user, -}) => { - const url = new URL(address); - - url.password = password; - url.username = user; - - const cmd = []; - - if (params) { - cmd.push(params); - } - - return [...cmd, '--uri', `${url.href}/${name}`]; -}; - -const connectionStringWithHost = ({ auth, host, name, params }) => - [auth, params, '--host', host, '-d', name].filter(Boolean); - -const handleProcessOutput = async ({ - debug = false, - errorMessage = 'An error occurred', - processOutput, - okayCodes = [0], -}) => { - let err; - let processOutputResult; - - try { - processOutputResult = await processOutput; - } catch (e) { - err = e; - } - - if (err) { - return reportError(new Error(err.stdout), program); - } - - if (debug === true) { - console.log(processOutputResult); - } - - const { stdout, stderr, exitCode } = processOutputResult; - - if (!okayCodes.includes(exitCode)) { - return reportError(new Error(errorMessage), program); - } - - return { stdout, stderr }; -}; - -const streamProcessOutput = async ({ processOutput }) => { - await processOutput.pipe(process.stdout); -}; - -(async () => { - const details = await loadConfig(`mongo.${env}`); - const connection = connectionParts(details); - - const dockerVolumeName = `mongodump_data_${details.name}_${env}${ - collection ? `_${collection}` : '' - }`; - - await handleProcessOutput({ - processOutput: $`docker volume ${['create', dockerVolumeName]}`, - errorMessage: 'Failed to create Docker volume', - }); - - await handleProcessOutput({ - processOutput: $`docker run -it --rm -v ${dockerVolumeName}:/data busybox sh -c "mkdir -p /data/${details.name} && chown -R root:root /data && chmod -R 777 /data"`, - errorMessage: 'Failed to setup Docker volume directories', - }); - - await handleProcessOutput({ - processOutput: $`docker run -it --rm -v ${dockerVolumeName}:/data busybox sh -c "ls -l /data"`, - errorMessage: 'Failed to run mongodump', - }); - - await streamProcessOutput({ - processOutput: $`docker run ${[ - '-it', - '--rm', - '-v', - `${dockerVolumeName}:/data`, - 'mongo:7', - 'mongodump', - ...(connection.host - ? connectionStringWithHost(connection) - : connectionStringWithAddress(connection)), - '-o', - `/data/${details.name}`, - ]}`, - }); - - await handleProcessOutput({ - processOutput: $`docker run -it --rm -v ${dockerVolumeName}:/data -v ${process.cwd()}/data:/host_data busybox sh -c "cp -r /data/* /host_data"`, - errorMessage: 'Failed to copy data to host', - }); - - await handleProcessOutput({ - processOutput: $`docker volume ${['rm', dockerVolumeName]}`, - errorMessage: 'Failed to remove Docker volume', - }); - - const user = await $`id -u`; - const group = await $`id -u`; - - await handleProcessOutput({ - processOutput: $`sudo chown ${[ - '-R', - `${user.stdout.replace('\n', '')}:${group.stdout.replace( - '\n', - '' - )}`, - `${process.cwd()}/data`, - ]}`, - errorMessage: 'Failed to remove Docker volume', - }); -})();