From 4980038d678a276bb19e8a91f3d193a073b02386 Mon Sep 17 00:00:00 2001 From: littlezhou Date: Fri, 21 Jul 2017 09:38:18 +0800 Subject: [PATCH] Solve #761, Refine boot scripts and provide standalone script for database formatting --- bin/common.sh | 52 ++++++++++++--- bin/format-database.sh | 60 +++++++++++++++++ bin/smart | 146 +++++++++++++++++++++++++++++++++++++++++ bin/start-smart.sh | 63 ++++++------------ bin/stop-smart.sh | 58 ++++++---------- 5 files changed, 288 insertions(+), 91 deletions(-) create mode 100755 bin/format-database.sh create mode 100755 bin/smart diff --git a/bin/common.sh b/bin/common.sh index fe18ce61262..1cffe256459 100755 --- a/bin/common.sh +++ b/bin/common.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -143,17 +143,16 @@ function start_smart_server() { if [[ "${SMART_VARGS}" =~ " -format" ]]; then echo "Start formatting database ..." - exec $SMART_RUNNER $JAVA_OPTS -cp "${SMART_CLASSPATH}" $SMART_SERVER $SMART_VARGS - exit $? + exec $SMART_RUNNER $JAVA_OPTS -cp "${SMART_CLASSPATH}" $SMART_CLASSNAME $SMART_VARGS fi - echo "Starting SmartServer ..." - ssh ${SSH_OPTIONS} ${servers} "cd ${SMART_HOME}; ${SMART_HOME}/bin/start-smart.sh ${SMART_VARGS} --daemon" + echo "Starting SmartServer ..." + smart_start_daemon ${SMART_PID_FILE} } function stop_smart_server() { local servers=localhost - ssh ${SSH_OPTIONS} ${servers} "cd ${SMART_HOME}; ${SMART_HOME}/bin/stop-smart.sh --daemon" + smart_stop_daemon ${SMART_PID_FILE} } function smart_start_daemon() { @@ -201,7 +200,7 @@ function start_daemon() { echo "ERROR: Can NOT write PID file ${pidfile}." fi - exec $SMART_RUNNER $JAVA_OPTS -cp "${SMART_CLASSPATH}" $SMART_SERVER $SMART_VARGS + exec $SMART_RUNNER $JAVA_OPTS -cp "${SMART_CLASSPATH}" $SMART_CLASSNAME $SMART_VARGS } function smart_stop_daemon() { @@ -238,4 +237,41 @@ function smart_stop_daemon() { else echo "Service not found on node '${HOSTNAME}'" fi -} \ No newline at end of file +} + +function init_command() { + local subcmd=$1 + shift + + case ${subcmd} in + formatdatabase) + SMART_CLASSNAME=org.smartdata.server.SmartDaemon + SMART_PID_FILE=/tmp/SmartServer.pid + ALLOW_DAEMON_OPT=true + SMART_VARGS+=" -format" + ;; + smartserver) + SMART_CLASSNAME=org.smartdata.server.SmartDaemon + SMART_PID_FILE=/tmp/SmartServer.pid + ALLOW_DAEMON_OPT=true + ;; + getconf) + SMART_CLASSNAME=org.smartdata.server.utils.tools.GetConf + ;; + *) + echo "Unkown command ${subcmd}" + exit 1; + ;; + esac +} + +function remote_execute() { + local host=$1 + shift + + ssh ${SSH_OPTIONS} ${host} "$@" +} + +function local_execute() { + exec $SMART_RUNNER $JAVA_OPTS -cp "${SMART_CLASSPATH}" $SMART_CLASSNAME $SMART_VARGS +} diff --git a/bin/format-database.sh b/bin/format-database.sh new file mode 100755 index 00000000000..69cd1b71023 --- /dev/null +++ b/bin/format-database.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +bin=$(dirname "${BASH_SOURCE-$0}") +bin=$(cd "${bin}">/dev/null; pwd) +HOSTNAME=$(hostname) + +SMART_VARGS= +while [ $# != 0 ]; do + case "$1" in + "--config") + shift + conf_dir="$1" + if [[ ! -d "${conf_dir}" ]]; then + echo "ERROR : ${conf_dir} is not a directory" + echo ${USAGE} + exit 1 + else + export SMART_CONF_DIR="${conf_dir}" + echo "SMART_CONF_DIR="$SMART_CONF_DIR + fi + shift + ;; + *) + SMART_VARGS+=" $1" + shift + ;; + esac +done + +. "${bin}/common.sh" + +#--------------------------------------------------------- +# + +echo -n "Start formatting database ... " + +$("${SMART_HOME}/bin/smart" --config "${SMART_CONF_DIR}" formatdatabase 2>/dev/null) + +if [ x"$?" = x"0" ]; then + echo "[Success]" +else + echo "[Failed]" +fi \ No newline at end of file diff --git a/bin/smart b/bin/smart new file mode 100755 index 00000000000..1cd42077cc0 --- /dev/null +++ b/bin/smart @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +Usage="smart --config [--daemon] [--hosts --hostsend] [--daemon start|stop ] [--remote] " + +SCRIPT_ARGS="$*" + +bin=$(dirname "${BASH_SOURCE-$0}") +bin=$(cd "${bin}">/dev/null; pwd) +HOSTNAME=$(hostname) + +REMOTE_COMMAND= +DAEMON_MOD= +DAEMON_MOD_OP= +REMOTE_MODE= +SMART_VARGS= +while [ $# != 0 ]; do + case "$1" in + "--config") + shift + conf_dir="$1" + if [[ ! -d "${conf_dir}" ]]; then + echo "ERROR : ${conf_dir} is not a directory" + echo ${USAGE} + exit 1 + else + export SMART_CONF_DIR="${conf_dir}" + fi + REMOTE_COMMAND+=" --config $1" + shift + ;; + "--debug") + JAVA_OPTS+=" -Xdebug -Xrunjdwp:transport=dt_socket,address=8008,server=y,suspend=y" + REMOTE_COMMAND+=" $1" + shift + ;; + "--daemon") + REMOTE_COMMAND+=" $1 $2" + DAEMON_MOD=true + shift + DAEMON_MOD_OP=$1 + shift + ;; + "--remote") + REMOTE_MODE=true + shift + ;; + "--hosts") + SMART_HOSTS= + shift + while [ x"$1" != x"--hostsend" ]; do + SMART_HOSTS+=" $1" + shift + done + shift + ;; + *) + break; + ;; + esac +done + +REMOTE_COMMAND+=" $@" +COMMAND=$1 +shift +SMART_VARGS="$@" + +. "${bin}/common.sh" + +init_command ${COMMAND} + +JAVA_OPTS+=" -Dsmart.log.dir=${SMART_LOG_DIR}" +JAVA_OPTS+=" -Dsmart.log.file=SmartServer.log" + +JAVA_VERSION=$($SMART_RUNNER -version 2>&1 | awk -F '.' '/version/ {print $2}') + +if [[ "$JAVA_VERSION" -ge 8 ]]; then + JAVA_OPTS+=" -XX:MaxMetaspaceSize=256m" +else + JAVA_OPTS+=" -XX:MaxPermSize=256m" +fi + +addJarInDir "${SMART_HOME}/smart-server/target/lib" +addNonTestJarInDir "${SMART_HOME}/smart-server/target" +addJarInDir "${SMART_HOME}/lib" + +if [ "$SMART_CLASSPATH" = "" ]; then + SMART_CLASSPATH="${SMART_CONF_DIR}" +else + SMART_CLASSPATH="${SMART_CONF_DIR}:${SMART_CLASSPATH}" +fi + +if [[ ! -d "${SMART_LOG_DIR}" ]]; then + $(mkdir -p "${SMART_LOG_DIR}") +fi + +SMART_VARGS+=" -D smart.conf.dir="${SMART_CONF_DIR} +SMART_VARGS+=" -D smart.log.dir="${SMART_LOG_DIR} + + +#echo "$SMART_RUNNER $JAVA_OPTS -cp ${SMART_CLASSPATH} ${SMART_CLASSNAME} $@" + +#exec $SMART_RUNNER $JAVA_OPTS -cp "${SMART_CLASSPATH}" ${SMART_CLASSNAME} $@ + +if [ x"${REMOTE_MODE}" = x"true" ]; then + for host in $SMART_HOSTS + do + echo "Processing on '${host}' ... " + remote_execute ${host} ${SMART_HOME}/bin/smart ${REMOTE_COMMAND} + if [ "$?" = "0" ]; then + echo "Processing on '${host}' completed successfully." + else + echo "ERROR: Processing on '${host}' failed." + fi + done +else + if [ x"${DAEMON_MOD}" = x"true" ]; then + case "${DAEMON_MOD_OP}" in + "start") + smart_start_daemon ${SMART_PID_FILE} + ;; + "stop") + smart_stop_daemon ${SMART_PID_FILE} + ;; + esac + else + local_execute + fi +fi +exit $? diff --git a/bin/start-smart.sh b/bin/start-smart.sh index 32e7ac2896f..647d7ef6892 100755 --- a/bin/start-smart.sh +++ b/bin/start-smart.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -16,17 +16,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Run SmartServer -# -#./bin/start-smart.sh -D smart.dfs.namenode.rpcserver=hdfs://localhost:9000 -#./bin/start-smart.sh -D smart.dfs.namenode.rpcserver=hdfs://localhost:9000 -D smart.metastore.db.url=jdbc:sqlite:file-sql.db - -USAGE="Usage: bin/start-smart.sh [--config ] [--debug] ..." bin=$(dirname "${BASH_SOURCE-$0}") bin=$(cd "${bin}">/dev/null; pwd) +HOSTNAME=$(hostname) -DAEMON_MOD= SMART_VARGS= while [ $# != 0 ]; do case "$1" in @@ -44,11 +38,7 @@ while [ $# != 0 ]; do shift ;; "--debug") - JAVA_OPTS+=" -Xdebug -Xrunjdwp:transport=dt_socket,address=8008,server=y,suspend=y" - shift - ;; - "--daemon") - DAEMON_MOD=1 + DEBUG_OPT=$1 shift ;; *) @@ -60,41 +50,26 @@ done . "${bin}/common.sh" +#--------------------------------------------------------- +# Start Smart Servers -HOSTNAME=$(hostname) - -SMART_SERVER=org.smartdata.server.SmartDaemon -JAVA_OPTS+=" -Dsmart.log.dir=${SMART_LOG_DIR}" -JAVA_OPTS+=" -Dsmart.log.file=SmartServer.log" - -JAVA_VERSION=$($SMART_RUNNER -version 2>&1 | awk -F '.' '/version/ {print $2}') +SMARTSERVERS=$("${SMART_HOME}/bin/smart" getconf SmartServers 2>/dev/null) -if [[ "$JAVA_VERSION" -ge 8 ]]; then - JAVA_OPTS+=" -XX:MaxMetaspaceSize=256m" -else - JAVA_OPTS+=" -XX:MaxPermSize=256m" +if [ "$?" != "0" ]; then + echo "${SMARTSERVERS}" + exit 1 fi -addJarInDir "${SMART_HOME}/smart-server/target/lib" -addNonTestJarInDir "${SMART_HOME}/smart-server/target" -addJarInDir "${SMART_HOME}/lib" - -if [ "$SMART_CLASSPATH" = "" ]; then - SMART_CLASSPATH="${SMART_CONF_DIR}" -else - SMART_CLASSPATH="${SMART_CONF_DIR}:${SMART_CLASSPATH}" -fi +#if [[ -z "${SMARTSERVERS}" ]]; then +#SMARTSERVERS=$HOSTNAME +#fi -if [[ ! -d "${SMART_LOG_DIR}" ]]; then - echo "Log dir doesn't exist, create ${SMART_LOG_DIR}" - $(mkdir -p "${SMART_LOG_DIR}") -fi +echo "Starting SmartServers on [${SMARTSERVERS}]" -SMART_VARGS+=" -D smart.conf.dir="${SMART_CONF_DIR} -SMART_VARGS+=" -D smart.log.dir="${SMART_LOG_DIR} +. "${SMART_HOME}/bin/smart" \ + --remote \ + --config "${SMART_CONF_DIR}" \ + --hosts "${SMARTSERVERS}" --hostsend \ + --daemon start ${DEBUG_OPT} \ + smartserver -if [ -z "$DAEMON_MOD" ]; then - start_smart_server -else - smart_start_daemon ${SMART_SERVER_PID_FILE} -fi diff --git a/bin/stop-smart.sh b/bin/stop-smart.sh index 7f40d614914..17d2cb36d20 100755 --- a/bin/stop-smart.sh +++ b/bin/stop-smart.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -16,17 +16,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Stop SmartServer -# -#./bin/start-smart.sh -D smart.dfs.namenode.rpcserver=hdfs://localhost:9000 -#./bin/start-smart.sh -D smart.dfs.namenode.rpcserver=hdfs://localhost:9000 -D smart.metastore.db.url=jdbc:sqlite:file-sql.db - -USAGE="Usage: bin/stop-smart.sh [--config ] [--debug] ..." bin=$(dirname "${BASH_SOURCE-$0}") bin=$(cd "${bin}">/dev/null; pwd) +HOSTNAME=$(hostname) -DAEMON_MOD= SMART_VARGS= while [ $# != 0 ]; do case "$1" in @@ -43,14 +37,6 @@ while [ $# != 0 ]; do fi shift ;; - "--debug") - JAVA_OPTS+=" -Xdebug -Xrunjdwp:transport=dt_socket,address=8008,server=y,suspend=y" - shift - ;; - "--daemon") - DAEMON_MOD=1 - shift - ;; *) SMART_VARGS+=" $1" shift @@ -60,32 +46,26 @@ done . "${bin}/common.sh" +#--------------------------------------------------------- +# Stop Smart Servers -HOSTNAME=$(hostname) - -SMART_SERVER=org.smartdata.server.SmartDaemon -JAVA_OPTS+=" -Dsmart.log.dir=${SMART_LOG_DIR}" - -addJarInDir "${SMART_HOME}/smart-server/target/lib" -addNonTestJarInDir "${SMART_HOME}/smart-server/target" -addJarInDir "${SMART_HOME}/lib" +SMARTSERVERS=$("${SMART_HOME}/bin/smart" getconf SmartServers 2>/dev/null) -if [ "$SMART_CLASSPATH" = "" ]; then - SMART_CLASSPATH="${SMART_CONF_DIR}" -else - SMART_CLASSPATH="${SMART_CONF_DIR}:${SMART_CLASSPATH}" +if [ "$?" != "0" ]; then + echo "${SMARTSERVERS}" + exit 1 fi -if [[ ! -d "${SMART_LOG_DIR}" ]]; then - echo "Log dir doesn't exist, create ${SMART_LOG_DIR}" - $(mkdir -p "${SMART_LOG_DIR}") -fi +#if [[ -z "${SMARTSERVERS}" ]]; then +#SMARTSERVERS=$HOSTNAME +#fi -SMART_VARGS+=" -D smart.conf.dir="${SMART_CONF_DIR} -SMART_VARGS+=" -D smart.log.dir="${SMART_LOG_DIR} +echo "Stopping SmartServers on [${SMARTSERVERS}]" + +. "${SMART_HOME}/bin/smart" \ + --remote \ + --config "${SMART_CONF_DIR}" \ + --hosts "${SMARTSERVERS}" --hostsend \ + --daemon stop ${DEBUG_OPT} \ + smartserver -if [ -z "$DAEMON_MOD" ]; then - stop_smart_server -else - smart_stop_daemon ${SMART_SERVER_PID_FILE} -fi