start-splice-cluster

#!/bin/bash

##################################################################################
# Start Zookeeper and the Splice HBase master server and region servers and yarn
#
# Currently, if compilation fails, it still tries to start Zoo and RSes.
#   An enhancement would be to tee build output to a Named Pipe to monitor for
#   "[INFO] BUILD SUCCESS" before starting, like this example:
#   mkfifo -m 770 /tmp/myfifo
#   iter=0 ; while true ; do echo $iter 2>&1 | tee /tmp/myfifo ; iter=$((${iter}+1)) ; sleep 1 ; done
#   while true ; do sleep 1 ; grep -q 6 /tmp/myfifo && echo 'i found a 6!' ; done
##################################################################################

BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
DEFAULT_PROFILE="cdh6.3.0"  # default hbase platform profile
PROFILE=$DEFAULT_PROFILE
IN_MEM_PROFILE="mem"
RUN_DIR="${BASE_DIR}/platform_it"

BUILD="1"
CLEAN="1"
CHAOS="false"

MEMBERS=2
DEBUG_PATH=""



##################################################################################
# Function to kill all splice test processes - zoo, SpliceTestPlatform, YARN, and
# anything started from maven exec, i.e., mvn exec:exec
##################################################################################
function _kill_em_all {
  SIG=$1
   local P=$(ps -ef | awk '/SpliceTestPlatform|SpliceSinglePlatform|SpliceTestClusterParticipant|OlapServerMaster/ && !/awk/ {print $2}')
   [[ -n $P ]] && echo "Found Splice. Stopping it." && for pid in $P; do kill -$SIG `echo $pid`; done

   P=$(ps -ef | awk '/spliceYarn|SpliceTestYarnPlatform|CoarseGrainedScheduler|ExecutorLauncher/ && !/awk/ {print $2}')
   [[ -n $P ]] && echo "Found YARN. Stopping it." && for pid in $P; do kill -$SIG `echo $pid`; done

   P=$(ps -ef | awk '/TestKafkaCluster/ && !/awk/ {print $2}')
   [[ -n $P ]] && echo "Found Kafka. Stopping it." && for pid in $P; do kill -$SIG `echo $pid`; done

   P=$(ps -ef | awk '/exec:java/ && !/awk/ {print $2}')
   [[ -n $P ]] && echo "Found stray maven exec:java. Stopping it." && for pid in $P; do kill -$SIG `echo $pid`; done

   P=$(ps -ef | awk '/ZooKeeper/ && !/awk/ {print $2}')
   [[ -n $P ]] && echo "Found ZooKeeper. Stopping it." && for pid in $P; do kill -$SIG `echo $pid`; done
}

##################################################################################
# check if port ${2} on host {1} is open
#
# note there's two versions of nc: GNU and OpenBSD
# OpenBSD supports -z (only check port), but GNU not.
# GNU would work with 'exit', OpenBSD needs 'exit\n'
# this one works in both and on mac
##################################################################################
function is_port_open
{
  host1=$1
  port2=$2
  echo 'exit\n' | nc ${host1} ${port2} > /dev/null 2> /dev/null
}

##################################################################################
# print message ${1} and wait until port {2} is open
##################################################################################
function _wait_for_port
{
  msg=${1}
  port=${2}
  echo -n ${msg}
  echo -n " "
  counter=0
  until is_port_open localhost ${port}; do
    echo -n "${counter} - "
    counter=$((counter+1))
    sleep 2
  done
  echo "done!"
}

##################################################################################
# wait until file ${1} contains the text ${2}
##################################################################################
function _wait_for_file_content
{
  LOG_FILE=$1
  shift
  NEEDLE=$*

  echo -n "  Waiting. "
  counter=0
  while ! grep -q "$NEEDLE" $LOG_FILE ; do
    echo -n "${counter} - "
    counter=$((counter+1));
    sleep 2
  done
  echo
}

##################################################################################
# zookeeper
##################################################################################
function _start_zookeeper
{
  ZOO_LOG="${RUN_DIR}/zoo.log"
  echo "Starting ZooKeeper. Log file is ${ZOO_LOG}"
  (${MVN} exec:exec -Denv=${HBASE_PROFILE} -P${PROFILE},spliceZoo > ${ZOO_LOG} 2>&1) &    ## zookeeper

}

#######################################################################################################
# Wait for up to 65 seconds for zoo to start, checking nc periodically to see if we are connected
# This makes use of ZooKeeper's 'Four-Letter Words' commands to determine the liveness of ZooKeeper
# In particular, it uses netcat (nc) to pipe the command 'ruok' to zookeeper on it's connection port.
# If ZooKeeper responds with 'imok', then the server is up. If it returns empty, then zookeeper
# hasn't started yet. If the count goes to 0, and we never got 'imok', then we want to blow up because
# ZooKeeper didn't start up properly
#######################################################################################################
function _wait_zookeeper
{
  COUNT=65
  ZOO_UP=""
  until [ ${COUNT} -eq 0 ] || [ "${ZOO_UP}" == "imok" ]; do
      sleep 1
      ZOO_UP="$(echo 'ruok' | nc localhost 2181 2> /dev/null)"
      let COUNT-=1
  done

  if [ ${COUNT} -eq 0 ]; then
      echo "ZooKeeper did not start up properly, aborting startup. Please check ${ZOO_LOG} for more information"
      exit 5
  fi
}


##################################################################################
# # Start Kafka in background
##################################################################################
function _start_kafka {
  KAFKALOG="${RUN_DIR}"/kafka.log
  echo "Starting Kafka. Log file is ${KAFKALOG}"
  (${MVN} exec:exec -Denv=${HBASE_PROFILE} -P${PROFILE},spliceKafka > ${KAFKALOG} 2>&1) &
}

##################################################################################
#  yarn
##################################################################################
function _start_yarn
{
  pushd ../standalone/ > /dev/null
  YARN_LOG="${RUN_DIR}/yarn.log"
  echo "Starting YARN. Log file is ${YARN_LOG}"
  (${MVN} exec:exec -Denv=${HBASE_PROFILE} -P${PROFILE},spliceYarn > ${YARN_LOG} 2>&1) &  ## YARN
  popd > /dev/null
}

##################################################################################
# region servers
##################################################################################
function _start_region_servers {
  if [[ ${MEMBERS} -gt 0 ]]; then
    for (( MEMBER=1; MEMBER<${MEMBERS}; MEMBER++ )); do
      REGION_SVR_LOG="${RUN_DIR}/spliceRegionSvr$(($MEMBER +1)).log"
      echo "Starting Region Server ${REGION_SVR_LOG}"
      ## (region server, splice on 1528, 1529, ...)
      (${MVN} exec:exec -Denv=${HBASE_PROFILE} -P${PROFILE},spliceClusterMember ${SYSTEM_PROPS} \
                        -DmemberNumber=${MEMBER} -Dxml.plan.debug.path=${DEBUG_PATH} > ${REGION_SVR_LOG} 2>&1) &
    done
  fi
}

function _wait_for_region_servers {
  if [[ ${MEMBERS} -gt 0 ]]; then
    for (( MEMBER=1; MEMBER<${MEMBERS}; MEMBER++ )); do
      _wait_for_port "Waiting for Region Server $(($MEMBER +1)) to be ready ..." $(( 1527 + ${MEMBER} ))
    done
  fi
}

##################################################################################
# start master
##################################################################################
function _start_master {
  SPLICE_LOG="${RUN_DIR}/splice.log"
  echo "Starting Master and 1 Region Server. Log file is ${SPLICE_LOG}"
  ## (master + region server on 1527)

  if [ -z "${SERVER_SSLMODE}" ]; then
    (${MVN} exec:exec -Denv=${HBASE_PROFILE} -P${PROFILE},spliceFast ${SYSTEM_PROPS} \
                      -Dxml.plan.debug.path=${DEBUG_PATH} > ${SPLICE_LOG} 2>&1) &
  else
    (${MVN} exec:exec -Denv=${HBASE_PROFILE} -P${PROFILE},spliceFastSSL ${SYSTEM_PROPS} \
                      -Dxml.plan.debug.path=${DEBUG_PATH} > ${SPLICE_LOG} 2>&1) &
  fi
}

function _wait_for_master {
  _wait_for_port "Waiting for Master to be ready ..." 1527
  
}

##################################################################################
# start with KDFS and HDFS (DISTRIBUTED=1 mode)
##################################################################################
function _start_kdc_hdfs
{
  KDC_LOG="${RUN_DIR}/kdc.log"
  echo "Starting KDC. Log file is ${KDC_LOG}"
  (${MVN} exec:java -Denv=${HBASE_PROFILE} -P${PROFILE},spliceKDC > ${KDC_LOG} 2>&1) &  ## KDC
  _wait_for_file_content ${KDC_LOG} "KDC cluster started" 
  
  export MAVEN_OPTS="$MAVEN_OPTS -Djava.security.krb5.conf=${RUN_DIR}/target/krb5.conf -Dsun.security.krb5.debug=true"
  
  HDFS_LOG="${RUN_DIR}/hdfs.log"
  echo "Starting HDFS. Log file is ${HDFS_LOG}"
  (${MVN} exec:java -Denv=${HBASE_PROFILE} -P${PROFILE},spliceHdfs > ${HDFS_LOG} 2>&1) &  ## HDFS
  _wait_for_file_content ${HDFS_LOG} "HDFS cluster started"
}

##################################################################################
# mem platform
##################################################################################
function _start_mem
{
  MEM_LOG="${RUN_DIR}/spliceMem.log"
  echo "Starting MEM. Log file is ${MEM_LOG}"
  export JMX_CONFIG="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=10102"
  (MAVEN_OPTS="${MAVEN_OPTS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=4000 ${JMX_CONFIG}" \
                ${MVN} -Pmem exec:java > ${MEM_LOG} 2>&1) & ## IN MEMORY
}

function _wait_for_mem
{
  _wait_for_port "Waiting until Mem Platform is ready ..." 1527
}

export -f _kill_em_all


########################################################################################################
# Command Line Options
########################################################################################################

function usage
{
    # $1 is an error, if any
    if [[ -n "${1}" ]]; then
        echo "Error: ${1}"
    fi
    echo "Usage: $0 -p [<hbase_profile>] [-s <n>] [-c] [-k] -f -h[elp]"
    echo "Where: "
    echo "  -b is an optional argument specifying to NOT ${MVN} clean install -DskipTest -Dspark-prepare first. The default is to build first."
    echo "  -l is an optional argument specifying to NOT run clean the environment."
    echo "  -f is an optional argument specifying to run a fully distributed, secure environment, with Kerberos and HDFS."
    echo "  -s <n> is an optional number of additional cluster RegionServer members to start. The default is 1 master and 2 region servers."
    echo "  -c is an optional flag determining random task failures. Default is that the chaos monkey NOT run. To see if you have the chaos monkey running, execute: grep 'task fail' <hbase_profile>/splice_machine_test/splice-derby.log"
    echo "  -p <hbase_profile> is the optional splice hbase platform to run. Default is ${DEFAULT_PROFILE}. (available options are in the top level pom.xml file)"
    echo "  -e use ee (enterprise edition) profile"
    echo "  -k just KILL any and all splice processes currently running."
    echo "  -K like k, but will also reset database to clean. Needs some rebuild afterwards (use -l)."
    echo "  -T <n> use -T <n> with maven (build with n threads)."
    echo "  -h => print this message"
}

function verify_profiles
{
   local PROFILE=$1
   local MVN_PROFILES=$(mvn -T 20 help:all-profiles | grep "Profile Id" | awk '{print $3}')
   for arg in $(echo $PROFILE | tr "," "\n")
   do
      local in=0
      for mvn_profile in $(echo $MVN_PROFILES | tr " " "\n")
      do
          if [[ "$arg" == "$mvn_profile" ]]; then
             in=1
             break
          fi
      done
      if [[ $in -eq 0 ]]; then
          echo "unknown profile '$arg', available profiles: $( echo $MVN_PROFILES | tr '\n' ' ')"
          exit 1
      fi
   done
}

while getopts "chkp:s:bld:fT:Ke" flag ; do
    case $flag in
        h* | \?)
            usage
            exit 0 # This is not an error, User asked help. Don't do "exit 1"
        ;;
        c)
        # start server with the chaos monkey (random task failures)
            CHAOS="true"
        ;;
        b)
        # DO NOT clean build first
            BUILD="0"
        ;;
        l)
        # DO NOT clean first
            CLEAN="0"
        ;;
        f)
        # Fully distributed & secure
          DISTRIBUTED="1"
        ;;
        p)
        # the hbase profile
          PROFILE="${OPTARG}"
        ;;
        s)
        # number of cluster members
           MEMBERS=$(($MEMBERS + $OPTARG))
        ;;
        d)
        # path to write debug file
           DEBUG_PATH=$OPTARG
        ;;
        k)
        # KILL current processes
           _kill_em_all 9
           exit 0
        ;;
        K)
        # KILL current processes + reset to clean database
           _kill_em_all 9
           echo "reset database..."
           rm -rf platform_it/target/hbase
           rm -rf platform_it/target/zookeeper
           rm -rf platform_it/target/hbase-site.xml
           rm -rf platform_it/target/SpliceTestYarnPlatform
           exit 0
        ;;
        T)
        # number of threads
           MVN_THREADS="-T ${OPTARG}"
        ;;
        e)
        # enterprise edition
          ENTERPRISE_EDITION=1
        ;;
        ?)
            usage "Unknown option (ignored): ${OPTARG}"
            exit 1
        ;;
    esac
done

MVN="mvn -B ${MVN_THREADS}"

SYSTEM_PROPS="-Dlog4j.configuration=hbase-log4j.properties -DfailTasksRandomly=${CHAOS}"

########################################################################################################
# Run...
########################################################################################################
_kill_em_all 9

if [[ "${ENTERPRISE_EDITION}" == "1" ]]; then
  PROFILE="${PROFILE},ee"
fi

HBASE_PROFILE=${PROFILE%%,*}
if [[ ${HBASE_PROFILE} == ${IN_MEM_PROFILE} ]]; then
  RUN_DIR="${BASE_DIR}/mem_sql"
# Otherwise, set (in)secure option
elif [[ "${DISTRIBUTED}" == "1" ]]; then
    PROFILE="secure,${PROFILE}"
else
    PROFILE="insecure,${PROFILE}"
fi

verify_profiles $PROFILE

echo "Running Splice $PROFILE master and ${MEMBERS} regionservers with CHAOS = ${CHAOS} in:"
echo "   ${RUN_DIR}"

export MAVEN_OPTS="$MAVEN_OPTS -Xmx5000m"

if [[ "${BUILD}" == "1" ]]; then
  echo "Building first..."
  pushd "${BASE_DIR}" > /dev/null
  if [[ "${CLEAN}" == "1" ]]; then
    (${MVN} clean install -Pcore,${PROFILE} -DskipTests ) || exit
  else
    (${MVN} install -Pcore,${PROFILE} -DskipTests ) || exit
  fi
  popd > /dev/null
  echo "Running Splice $PROFILE master and ${MEMBERS} regionservers with CHAOS = $CHAOS in:"
  echo "   ${RUN_DIR}"
fi

pushd "${RUN_DIR}" > /dev/null
## delete old logs before we start fresh
/bin/rm -f *.log*

# Run IN MEMORY
if [[ ${HBASE_PROFILE} == ${IN_MEM_PROFILE} ]]; then
  _start_mem
  _wait_for_mem
  popd > /dev/null
  exit 0;
fi

_start_zookeeper
_wait_zookeeper # wait for zookeeper before starting others

if [[ "${DISTRIBUTED}" == "1" ]]; then
  _start_kdc_hdfs
fi 
  
_start_yarn
_start_kafka

_start_master
_wait_for_master

_start_region_servers

_wait_for_region_servers

popd > /dev/null