manage

#!/usr/bin/env bash

#
# Litter Map service management toolkit
#

# MacOS by default has BSD grep, and brew installs GNU grep as ggrep
if command -v ggrep &>/dev/null; then
  grep="ggrep"
else
  grep="grep"
fi

# Check if a command exists in the shell session
command_exists() {
  command -v "$1" &>/dev/null
}

# Exit if a particular command is not available
required_command() {
  if ! command_exists "$1"; then
    echo "Install $1: $2"
    exit 1
  fi
}

jq_required() {
  required_command jq https://stedolan.github.io/jq/
}

# Return samconfig configuration parameter
get_config() {
  if [ -f samconfig.toml ]; then
    # Read "sam deploy" configuration parameter
    #
    # FIXME: This should only read parameters defined in the correct section of the samconfig.toml file
    $grep -Po "(?<=$1 = \\\").*(?=\\\")" samconfig.toml | head -1
  fi
}

get_stack_param_slashes_escaped() {
  params=$(get_config parameter_overrides)

  # Find the specific parameter value (backslashes remain escaped)
  echo "$params" | $grep -Po "(?<=$1=\\\\\\\").*?(?=\\\\\\\")"
}

get_stack_param() {
  # Unescape backslashes in the returned parameter value
  get_stack_param_slashes_escaped "$1" | sed -e 's/\\\\/\\/g'
}

profile=$(get_config profile)

if [ -n "$profile" ]; then
  # Use AWS with the profile name used to deploy the stack, if found in samconfig.toml
  aws="aws --profile $profile"
else
  aws=aws
fi

nargs=$#

# If no arguments, list available commands and exit
if [ $nargs -eq 0 ]; then
  echo "Available commands:"
  echo

  # Finds the commands by inspecting the code
  items=$($grep '"[^"]\+" '$'\051' "$0")

  # Set the input field separator to split only on new lines
  IFS=$'\n'

  for i in $items; do
    # Print the command name
    echo -ne "- \e[1;37m$(echo "$i" | cut -d\" -f 2)\e[0m"

    # Print the arguments
    echo -e "\e[33m$(echo "$i" | awk -F '# args:' '{print $2}')\e[0m"
  done

  if ! command_exists jq; then
    echo
    echo "jq not found, please install it: https://stedolan.github.io/jq/download/"
  fi

  exit
fi

if [ ! -f "$(basename "$0")" ]; then
  echo "Please run $0 from its own directory"
  exit 1
fi

# Set how many arguments are strictly required by a command
args () {
  if [[ $((nargs-1)) -lt $1 ]]; then
    if [[ $1 -eq 1 ]]; then
      what=argument
    else
      what=arguments
    fi

    echo "This command requires $1 $what"
    exit 1
  fi
}

# Base name for a custom lambda build environment docker image
build_env_image_base_name () {
  echo "lambda-$1-build-environment"
}

# Format and colorize JSON output with jq (if it's available)
format_json=$(command_exists jq && echo jq || echo cat)

case $1 in
  "list-functions" )
    $aws lambda list-functions --output json | $format_json
    ;;

  "list-apis" )
    $aws apigatewayv2 get-apis --output json | $format_json
    ;;

  "list-api-ids" )
    jq_required

    $0 list-apis | jq -r ".Items[].ApiId"
    ;;

  "list-api-stages" ) # args: [api-id]
    api=${2:-$($0 list-api-ids | head -1)}
    $aws apigatewayv2 get-stages --api-id "$api"
    ;;

  "list-api-stage-names" ) # args: [api-id]
    jq_required

    api=${2:-$($0 list-api-ids | head -1)}
    $0 list-api-stages "$api" | jq -r '.Items[] | .StageName'
    ;;

  "list-api-urls" )
    region=$(get_config region)

    if [[ -z $region ]]; then
      echo "Please configure your AWS deployment region by running: aws configure"
      exit 1
    fi

    for api in $($0 list-api-ids); do
      for stage in $($0 list-api-stage-names "$api"); do
        echo "https://$api.execute-api.$region.amazonaws.com/$stage/"
      done
    done
    ;;

  "list-rds-dbs" )
    $aws rds describe-db-instances --output json | $format_json
    ;;

  "list-rds-db-ids" )
    jq_required

    $0 list-rds-dbs | jq -r "try(.DBInstances[].DBInstanceIdentifier)"
    ;;

  "list-rds-db-endpoints" )
    jq_required

    $0 list-rds-dbs | jq -r "try(.DBInstances[].Endpoint)"
    ;;

  "list-rds-postgres-versions" )
    for ver in $($aws rds describe-db-engine-versions --engine postgres --query "DBEngineVersions[].EngineVersion" --output=text); do
      echo "$ver"
    done
    ;;

  "list-rds-db-instance-options" )
    $aws rds describe-orderable-db-instance-options --engine postgres --engine-version 13.3 --query "*[].{DBInstanceClass:DBInstanceClass,StorageType:StorageType}|[?StorageType=='gp2']|[].{DBInstanceClass:DBInstanceClass}" --output text
    ;;

  "list-stacks" )
    jq_required

    $aws cloudformation describe-stacks --output json | jq -r '.Stacks[].StackName'
    ;;

  "list-stack-params" ) # args: [stack-name]
    jq_required

    $aws cloudformation describe-stacks --stack-name "${2:-$(get_config stack_name)}" | jq -r 'try(.Stacks[0].Parameters[]) | "\(.ParameterKey)=\(.ParameterValue)"'
    ;;

  "list-stack-outputs" ) # args: [stack-name]
    jq_required

    $aws cloudformation describe-stacks --stack-name "${2:-$(get_config stack_name)}" | jq -r 'try(.Stacks[0].Outputs)'
    ;;

  "function-run" ) # args: func-name [...params]
    args 1
    jq_required

    name=$2
    shift 2

    out=$(mktemp)
    res=$($aws lambda invoke --function-name "$name" --log-type Tail "$out" "$@")

    echo "Execution information:"
    echo
    echo "$res" | jq '.LogResult=(.LogResult | @base64d | gsub("[\t\n]$";"") | gsub("\t";"\n  ") | split("\n"))'
    echo
    echo "Function response: [raw: $out]"
    echo

    status=$(jq -r '.statusCode // empty' < "$out")

    if [ -z "$status" ]; then
      jq < "$out"
    else
      echo "HTTP code: $status"
      echo
      jq '.body | fromjson' < "$out"
    fi
    ;;

  "function-logs" ) # args: func-name
    args 1
    jq_required

    logstreams=$($aws logs describe-log-streams --log-group-name "/aws/lambda/$2" --query 'logStreams[*].logStreamName' | jq -r '.[-1]')

    if [ -z "$logstreams" ]; then
      echo
      echo "No logs for this function"
    else
      for i in $logstreams; do
        $aws logs get-log-events --log-group-name "/aws/lambda/$2" --log-stream-name "$i" | jq '.events[]'
      done
    fi
    ;;

  "make-cpp-build-environment" ) # args: [arch = arm|x86]
    required_command docker https://docs.docker.com/get-docker/
    jq_required

    set -e

    # Determine the host CPU architecture
    host_arch=$(uname -m)

    #
    # Determine which CPU architecture we're building the docker build environment for and whether it's the same or
    # different from the host CPU architecture.
    #
    case $2 in
      "" )
        case "$host_arch" in
          aarch64 )
            platform=linux/arm64
            arch=arm
            ;;

          x86_64 )
            platform=linux/amd64
            arch=x86
            ;;

          * )
            echo "Your system architecture ($host_arch) is not supported on AWS Lambda. Choose one of the supported architectures."

            exit 1
            ;;
        esac

        echo "Building for your host CPU architecture: 64-bit $arch"
        echo
        ;;

      arm )
        platform=linux/arm64
        arch=arm

        if [ "$host_arch" != "arm64" ]; then
          echo "Not building for your host architecture (building for 64-bit $arch)"
          echo
        fi
        ;;

      x86 )
        platform=linux/amd64
        arch=x86

        if [ "$host_arch" != "x86_64" ]; then
          echo "Not building for your host architecture (building for 64-bit $arch)"
          echo
        fi
        ;;

      * )
        echo "Architecture must be one of: arm|x86 (assumed 64-bit)"
        echo
        echo "See: https://docs.aws.amazon.com/lambda/latest/dg/foundation-arch.html"

        exit 1
        ;;
    esac

    #
    # If the 'buildx' module is present, use it to enable use of the new BuildKit engine, which is
    # required for cross-architecture builds. Once it becomes the standard build engine for Docker,
    # use of the 'buildx' module will be deprecated and the module might be removed.
    #
    if docker buildx &>/dev/null; then
      buildx=buildx
    fi

    image_base_name=$(build_env_image_base_name cpp)

    image_name="$image_base_name-$arch"

    old_image=$(docker images "$image_name" -q)

    if [ -n "$old_image" ]; then
      echo "Existing image found: $old_image ($image_name)"
      echo
    fi

    echo "== Building self-contained environment for building and packaging C++ lambdas for deployment"
    echo

    # Build the Docker image that contains the build environment that is ready to build native Lambda functions
    docker $buildx build \
      --progress plain \
      --tag "$image_name" \
      --platform="$platform" \
      build-environments/cpp

    # Look up current default build environment image
    current_default_inspect=$(docker image inspect "$image_base_name" 2>/dev/null)

    # If default wasn't set, then set this as default
    if [ "$current_default_inspect" = '[]' ]; then
      echo
      echo "Setting this as the default build environment image"

      docker tag "$image_name" "$image_base_name"
    else
      tags=$(echo "$current_default_inspect" | jq -r '.[].RepoTags[]')

      # See if the default points to a still tagged build environment image
      current_default=$(echo "$tags" | grep "$image_base_name-" | cut -d: -f1 | tr '\n' ' ')

      # If the default no longer points to a tagged image, show the image id instead
      if [ -z "$current_default" ]; then
        current_default=$(echo "$current_default_inspect" | jq -r '.[].Id' | cut -d: -f2)
      fi

      echo
      echo "Default build environment is still currently: $current_default"
    fi

    if [ -n "$old_image" ]; then
      # Check if the old image has any name aliases (relevant field is named "RepoTags")
      tags=$(docker image inspect "$old_image" | jq -r '.[].RepoTags[]')

      # If the old image has no name aliases, it can be discarded
      if [ -z "$tags" ]; then
        echo
        echo "Deleting the previously existing Docker image: $old_image (used to be $image_name)"
        echo

        docker rmi "$old_image" || true # Don't crash if image can't be deleted
      fi
    fi

    echo
    echo "Finished building: $image_name"
    echo
    echo "Image metadata:"
    echo

    docker image inspect "$image_name" | jq '.[].Config.Labels'
    ;;

  "set-default-cpp-build-environment" ) # args: [arch]
    required_command docker https://docs.docker.com/get-docker/
    jq_required

    image_base_name=$(build_env_image_base_name cpp)

    #
    # If no architecture was specified or cannot set default, list available images; otherwise set default
    #
    if [ -z "$2" ]; then
      current_default_inspect=$(docker image inspect "$image_base_name" 2>/dev/null)

      if [ "$current_default_inspect" = '[]' ]; then
        echo "Current default build environment isn't set."
        echo
      else
        tags=$(echo "$current_default_inspect" | jq -r '.[].RepoTags[]')

        # Default alias might point to more than one other name, so concatenate them into one line
        echo "Current default: $(echo "$tags" | grep "$image_base_name-" | cut -d: -f1 | tr '\n' ' ')"
        echo
        echo "Image metadata:"
        echo
        echo "$current_default_inspect" | jq '.[].Config.Labels'
        echo
      fi

      echo "Available build environment images:"
      echo

      docker images "$image_base_name-*"
    else
      image_name="$image_base_name-$2"

      image_inspect=$(docker image inspect "$image_name" 2>/dev/null)

      # Cannot set default if this specific build environment image doesn't exist
      if [ "$image_inspect" = '[]' ]; then
        echo "Cannot set default to: $image_name"
        echo
        echo "Available build environment images:"
        echo

        docker images "$image_base_name-*"

        exit 1
      fi

      echo "Setting default build environment image: $image_name"

      docker tag "$image_name" "$image_base_name"
    fi
    ;;

  "build-cpp-function" ) # args: func-name [arch] [build = debug|release]
    args 1

    required_command docker https://docs.docker.com/get-docker/

    set -e

    # You can pass "-" for 'arch' to use the current default container image
    arch=${3:--}

    image_base_name=$(build_env_image_base_name cpp)

    if [ "$arch" != '-' ]; then
      image="$image_base_name-$arch"
    else
      # Default build environment alias
      image="$image_base_name"
    fi

    # If the build environment image does not exist, show available images and exit
    if [ "$(docker image inspect "$image" 2>/dev/null)" = '[]' ]; then
      echo "Build environment image not found: $image"
      echo
      echo "Available build environment images:"
      echo

      docker images "$image_base_name-*"

      exit 1
    fi

    # Build a release release by default
    build_type=${4:-release}

    # Capitalize the build type
    build_type=${build_type^}

    cd "functions/$2"

    echo "Building function $2 insider a Docker container"
    echo
    echo "Function source: functions/$2/src/"
    echo "Docker container image: $image"
    echo
    echo "-- Enter Docker"
    echo

    # The docker container finds the source in /mnt/src and produces the package in /mnt/out
    docker run \
      -it --rm \
      -v "$(pwd)":/mnt/src \
      -v /tmp:/mnt/out \
      "$image" \
      /mnt/src/build.sh "$build_type"

    echo
    echo "-- Leave Docker"
    echo

    mkdir -p build && cp -v "/tmp/$2.zip" build/

    package_size=$(du -sh build/"$2".zip | cut -f1)

    echo
    echo "The package is ready: functions/$2/build/$2.zip (size: $package_size)"
    ;;

  "rds-db-action" ) # args: action [instance-id]
    args 1

    #
    # For list of possible actions: `aws rds help`
    #

    if [ -z "$3" ]; then
      ids=$($0 list-rds-db-ids)

      if [[ -z "$ids" ]]; then
        echo "No databases found"
      elif [ "$(echo "$ids" | wc -l)" -gt 1 ]; then
        echo "Found more than one database, please specify an instance id:"
        echo
        echo "$ids"
      else
        id="$ids"
      fi
    else
      id="$3"
    fi

    if [ -n "$id" ]; then
      $aws rds "$2" --db-instance-identifier "$id"
    else
      exit 1
    fi
    ;;

  "rds-db-status" ) # args: [instance-id]
    jq_required

    if [ -n "$2" ]; then
      id="--db-instance-identifier $2"
    fi

    res=$($aws rds describe-db-instances "$id")

    echo "$(echo "$res" | jq -r '.DBInstances[].DBName')": "$(echo "$res" | jq -r '.DBInstances[].DBInstanceStatus')"
    ;;

  "rds-db-hibernate" ) # args: [instance-id]
    jq_required

    set -eo pipefail

    status="$($0 rds-db-action stop-db-instance "$2" | jq -r '.DBInstance.DBInstanceStatus')"

    echo "Status: $status"
    echo
    echo "This may take some time"
    echo
    echo "Check the current status with: $0 rds-db-status"
    echo
    echo "AWS will stop the database cluster for seven days, after which is will automatically be restarted."
    echo
    echo "According to your system clock, 7 days from now will be: $(date -d "+7 days")"
    ;;

  "rds-db-wake" ) # args: [instance-id]
    jq_required

    set -eo pipefail

    status="$($0 rds-db-action start-db-instance "$2" | jq -r '.DBInstance.DBInstanceStatus')"

    echo "Status: $status"
    echo
    echo "This may take some time"
    echo
    echo "Check the status with: $0 rds-db-status"
    ;;

  "rds-db-init" )
    echo "WARNING: If you have an existing database, it will lose all stored information."
    echo

    $0 function-run db-init

    echo
    echo "!IMPORTANT! - The value of geometry_type_objectid changes on every database init"
    echo
    echo "Don't forget to run \`sam deploy -g\` and manually specify the DBGeometryTypeObjectID parameter"
    ;;

  "rds-db-run" ) # args: query [user]
    args 1

    # Disable filename expansion so that the SQL query will be accepted verbatim
    set -f

    # Escape double quotes in the query, since it is being passed as a JSON value
    a1="\"query\":\"${2//\"/\\\"}\""

    # Include optional user parameter
    [ $# -ge 3 ] && a2=",\"user\": \"$3\""

    json="{$a1$a2}"

    echo "Invoking query function with parameters:"
    echo
    echo "$json" | $format_json
    echo

    $0 function-run db-run --payload "$json"
    ;;

  "rds-db-run-file" ) # args: query-file
    args 1
    jq_required

    echo "Invoking query function with parameters:"
    echo
    jq < "$2"
    echo

    $0 function-run db-run --payload "fileb://$2"
    ;;

  "rds-db-connect" ) # args: [role = admin|reader|writer]
    required_command psql https://www.postgresql.org/download/
    jq_required

    # Database name (defined as a stack parameter)
    dbname=$(get_stack_param DBName)

    # Log in as admin by default
    role=${2:-admin}

    # The admin role has a configurable user name (so it can be different from just "admin")
    if [ "$role" = admin ]; then
      user=$(get_stack_param DBAdminUser)
    else
      user=$role
    fi

    # Get the password for the role from the stack configuration file
    password=$(get_stack_param_slashes_escaped "DB${role^}Password")

    if [ -z "$password" ]; then
      echo "No password found for database user \"$user\" in samconfig.toml"
      echo

      userinfo="user=$user"
    else
      userinfo="user=$user password=$password"
    fi

    # Current assumption is that only one database endpoint will be returned
    dbinfo=$($0 list-rds-db-endpoints)

    # Retrieve database connection info and connect to the database with `psql`
    psql "$(echo "$dbinfo" | jq -r "\"${userinfo//\\/\\\\} host=\" + .Address + \" port=\" + (.Port | tostring) + \" dbname=$dbname\"")"
    ;;

  "rds-db-get-security-group" )
    jq_required

    $aws rds describe-db-instances | jq -r ".DBInstances[].VpcSecurityGroups[].VpcSecurityGroupId"
    ;;

  "rds-db-authorize-remote-access" )
    sg=$($0 rds-db-get-security-group)

    # Authorize TCP access to the database from the public internet
    $aws ec2 authorize-security-group-ingress --group-id "$sg" --protocol tcp --port 5432 --cidr 0.0.0.0/0
    ;;

  "rds-db-deauthorize-remote-access" )
    sg=$($0 rds-db-get-security-group)

    # Deauthorize TCP access to the database from the public internet
    $aws ec2 revoke-security-group-ingress --group-id "$sg" --protocol tcp --port 5432 --cidr 0.0.0.0/0
    ;;

  "dynamo-table-info" ) # args: table = sessions|users|event-log
    args 1

    $aws dynamodb describe-table --table-name "$2" | $format_json
    ;;

  "s3-list-object-tags" ) # args: bucket-name object-key
    args 2

    $aws s3api get-object-tagging --bucket "$2" --key "$3" | $format_json
    ;;

  "s3-delete-all" ) # args: bucket-name
    args 1

    $aws s3 rm --recursive "s3://$2"
    ;;

  "event-log" ) # args: [utc-date]
    jq_required

    # Convert today or given UTC date to epoch time
    date=$(date --date="${2:-$(date --utc +"%Y-%m-%d")}" --utc +%s)

    $aws dynamodb query --table-name event-log --key-condition-expression 'date_key = :date' --expression-attribute-values "{\":date\": {\"S\": \"$date\"}}" --scan-index-forward | jq -r '.Items[] | [.timestamp.S, .event.M.message.S] | join(" :: ")'
    ;;

  "stack-cancel-update" ) # args: [stack-name]
    $aws cloudformation cancel-update-stack --stack-name "${2:-$(get_config stack_name)}"
    ;;

  "stack-delete" ) # args: [stack-name]
    sam delete --stack-name "${2:-$(get_config stack_name)}"
    ;;

  "api-export" ) # args: [api-id] [stage]
    api=${2:-$($0 list-api-ids | head -1)}
    stage=${3:-$($0 list-api-stage-names "$api" | head -1)}
    out=$(mktemp)
    $aws apigatewayv2 export-api --api-id "$api" --stage-name "$stage" --specification OAS30 --output-type=YAML "$out"

    cat "$out"
    rm "$out"
    ;;

  "gen-password" )
    $aws secretsmanager get-random-password --output=text
    ;;

  "gen-rds-db-password" )
    # As per documented constraints for RDS passwords, replace quotes with |,
    # @ with # and / with \, and then also escape \ as \\
    $0 gen-password | tr '/@"'"'" '\\#||' | sed 's/\\/\\\\/g'
    ;;

  "delete-stack-artifacts" )
    s3_bucket=$(get_config s3_bucket)

    if [ -z "$s3_bucket" ]; then
      echo "s3_bucket is not defined in samconfig.toml"
      exit 1
    fi

    $aws s3 rm --recursive "s3://$s3_bucket/$(get_config stack_name)/"
    ;;

  "check-service-quotas" ) # args: [service]
    if [ -z "$2" ]; then
      jq_required

      # List services for which service quota information can be queried
      $aws service-quotas list-services | jq -r ".Services[].ServiceCode"
    else
      $aws service-quotas list-service-quotas --service-code "$2" | $format_json
    fi
    ;;

  "whoami" )
    $aws iam get-user | $format_json
    ;;

  "lint" )
    required_command jshint https://jshint.com/install/
    required_command shellcheck https://github.com/koalaman/shellcheck

    # Lint js code
    find {functions,layers}/ -name '*.js' -print0 | xargs -0 jshint

    # Lint bash code
    shellcheck "$0"
    ;;

  "samconfig-get-stack-param" ) # args: name
    args 1

    get_stack_param "$2"
    ;;

  "frontend-prepare" ) # args: [git-ref]
    required_command pnpm https://pnpm.io

    if [ -d publish ]; then
      echo "It looks like you have previously used this command. Please delete the publish/ directory and re-run this."
      exit 1
    fi

    ref=${2:-master}

    git clone git@github.com:littermap/littermap.git publish
    cd publish
    echo

    git reset "$ref" --hard
    echo

    pnpm install
    echo

    echo Please configure the front-end in publish/ before publishing
    ;;

  "frontend-update" ) # args: [git-ref]
    if [ ! -d publish ]; then
      echo First acquire a copy of the front-end repository by running: "$0" frontend-prepare
      exit 1
    fi

    ref=${2:-master}

    cd publish
    git reset --hard
    echo
    git pull
    echo
    git reset "$ref" --hard
    echo

    pnpm install
    echo

    echo "Check the configuration and then publish the website"
    ;;

  "frontend-publish" )
    if [ ! -d publish ]; then
      echo First acquire a copy of the front-end repository by running: "$0" frontend-prepare
      exit 1
    fi

    cd publish

    if ! pnpm run build; then
      echo
      echo "Build failed"

      exit 1
    fi

    echo
    echo -n "Looking for S3 bucket that hosts the website..."

    bucket=$($aws s3 ls | $grep website | awk '{ print $3 }')

    if [ -z "$bucket" ]; then
      echo " not found."
      echo
      echo "Could not publish the website because the hosting S3 bucket was not found"

      exit 1
    else
      echo
      echo
    fi

    echo "Uploading files to S3 bucket: $bucket"
    echo

    $aws s3 sync --acl public-read --cache-control 'max-age=604800' build/ "s3://$bucket" --exclude index.html
    $aws s3 sync --acl public-read --cache-control 'no-cache'       build/ "s3://$bucket"

    echo
    echo "Website files published to: https://$bucket.s3.amazonaws.com"
    ;;

  * )
    echo "Command $1 not available"

    exit 1
    ;;
esac