From 49c219ac074be92496ca67cb3ee185ac5174ef2a Mon Sep 17 00:00:00 2001 From: "Sungjun, Kim" Date: Fri, 10 May 2019 22:17:58 +0900 Subject: [PATCH 1/4] [CI] Randomize RPCService's port in test_container_rpc.sh We should avoid the port collisions between test cases as much as possible. https://amplab.cs.berkeley.edu/jenkins//job/Clipper-PRB/1934/ shows the port collision to us. --- containers/test/test_container_rpc.sh | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/containers/test/test_container_rpc.sh b/containers/test/test_container_rpc.sh index 61857fff2..c3df58e86 100755 --- a/containers/test/test_container_rpc.sh +++ b/containers/test/test_container_rpc.sh @@ -22,7 +22,9 @@ then fi success=false -rpc_service_port=17000 # for test only + +PORT_RANGE_START=10000 +PORT_RANGE_END=20000 function clean_up { # Perform program exit housekeeping @@ -48,8 +50,9 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $DIR # Start python rpc test container -echo "Starting python RPC test container..." -python ../python/rpc_test_container.py --rpc_service_port $rpc_service_port & +RPC_SERVICE_PORT=`perl -e "print int(rand($PORT_RANGE_END-$PORT_RANGE_START)) + $PORT_RANGE_START"` +echo "Starting python RPC test container... (port:$RPC_SERVICE_PORT)" +python ../python/rpc_test_container.py --rpc_service_port $RPC_SERVICE_PORT & # Deprecate JVM containers # cd ../jvm @@ -68,20 +71,24 @@ cd $DIR/../../ cd container make container_rpc_test container_uptime_seconds=180 -./container_rpc_test -t $container_uptime_seconds -p $rpc_service_port & +RPC_SERVICE_PORT=`perl -e "print int(rand($PORT_RANGE_END-$PORT_RANGE_START)) + $PORT_RANGE_START"` +echo "Starting cpp RPC test container... (port:$RPC_SERVICE_PORT)" +./container_rpc_test -t $container_uptime_seconds -p $RPC_SERVICE_PORT & sleep 10s cd $DIR/../../debug/src/benchmarks make rpctest -echo "Executing RPC test (first iteration)..." REDIS_PORT=$1 -./rpctest --num_containers=2 --timeout_seconds=30 --redis_port $REDIS_PORT --rpc_service_port $rpc_service_port +RPC_SERVICE_PORT=`perl -e "print int(rand($PORT_RANGE_END-$PORT_RANGE_START)) + $PORT_RANGE_START"`` +echo "Executing RPC test (first iteration)... (redis port:$REDIS_PORT, rpc_service_port:$RPC_SERVICE_PORT)" +./rpctest --num_containers=2 --timeout_seconds=30 --redis_port $REDIS_PORT --rpc_service_port $RPC_SERVICE_PORT redis-cli -p $REDIS_PORT "flushall" echo "Sleeping for 5 seconds..." sleep 5s -echo "Executing RPC test (second iteration)..." -./rpctest --num_containers=2 --timeout_seconds=30 --redis_port $REDIS_PORT --rpc_service_port $rpc_service_port +RPC_SERVICE_PORT=`perl -e "print int(rand($PORT_RANGE_END-$PORT_RANGE_START)) + $PORT_RANGE_START"` +echo "Executing RPC test (second iteration)... (redis port:$REDIS_PORT, rpc_service_port:$RPC_SERVICE_PORT)" +./rpctest --num_containers=2 --timeout_seconds=30 --redis_port $REDIS_PORT --rpc_service_port $RPC_SERVICE_PORT redis-cli -p $REDIS_PORT "flushall" echo "TEST PASSED!" success=true From 7f0e77a56c4b6caa2f7132bf0b9cda32c6de5d77 Mon Sep 17 00:00:00 2001 From: "Sungjun, Kim" Date: Fri, 10 May 2019 23:40:59 +0900 Subject: [PATCH 2/4] Fix my mistake --- containers/test/test_container_rpc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/test/test_container_rpc.sh b/containers/test/test_container_rpc.sh index c3df58e86..280f075d5 100755 --- a/containers/test/test_container_rpc.sh +++ b/containers/test/test_container_rpc.sh @@ -80,7 +80,7 @@ sleep 10s cd $DIR/../../debug/src/benchmarks make rpctest REDIS_PORT=$1 -RPC_SERVICE_PORT=`perl -e "print int(rand($PORT_RANGE_END-$PORT_RANGE_START)) + $PORT_RANGE_START"`` +RPC_SERVICE_PORT=`perl -e "print int(rand($PORT_RANGE_END-$PORT_RANGE_START)) + $PORT_RANGE_START"` echo "Executing RPC test (first iteration)... (redis port:$REDIS_PORT, rpc_service_port:$RPC_SERVICE_PORT)" ./rpctest --num_containers=2 --timeout_seconds=30 --redis_port $REDIS_PORT --rpc_service_port $RPC_SERVICE_PORT redis-cli -p $REDIS_PORT "flushall" From 406fd34e7eaa1c3aa658cb4a7aca1218c731c13d Mon Sep 17 00:00:00 2001 From: "Sungjun, Kim" Date: Sat, 11 May 2019 00:44:04 +0900 Subject: [PATCH 3/4] Handle psutil.NoSuchProcess error properly Current Jenkins CI shows the following errors to us. ``` [unittest_rpc_container] Traceback (most recent call last): [unittest_rpc_container] File ../python/rpc_test_container.py, line 1, in [unittest_rpc_container] import rpc [unittest_rpc_container] File /clipper/containers/python/rpc.py, line 20, in [unittest_rpc_container] import clipper_admin.metrics as metrics [unittest_rpc_container] File /clipper/clipper_admin/clipper_admin/metrics/__init__.py, line 6, in [unittest_rpc_container] if not server.redis_daemon_exist(): [unittest_rpc_container] File /clipper/clipper_admin/clipper_admin/metrics/server.py, line 134, in redis_daemon_exist [unittest_rpc_container] process_names = [psutil.Process(pid).name() for pid in pids] [unittest_rpc_container] File /usr/local/lib/python2.7/dist-packages/psutil/__init__.py, line 338, in __init__ [unittest_rpc_container] self._init(pid) [unittest_rpc_container] File /usr/local/lib/python2.7/dist-packages/psutil/__init__.py, line 378, in _init [unittest_rpc_container] raise NoSuchProcess(pid, None, msg) [unittest_rpc_container] psutil._exceptions.NoSuchProcess: psutil.NoSuchProcess no process found with pid 162 ``` --- clipper_admin/clipper_admin/metrics/server.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/clipper_admin/clipper_admin/metrics/server.py b/clipper_admin/clipper_admin/metrics/server.py index 4295ced84..0fc2706b1 100644 --- a/clipper_admin/clipper_admin/metrics/server.py +++ b/clipper_admin/clipper_admin/metrics/server.py @@ -131,7 +131,15 @@ def redis_daemon_exist(): # situation is that we are in a container without any other python2 # process. pids = psutil.pids() - process_names = [psutil.Process(pid).name() for pid in pids] + process_names = [] + + for pid in pids: + try: + name = psutil.Process(pid).name() + except psutil.NoSuchProcess: + name = None + process_names.append(name) + return 'redis-server' in process_names From 691e67ed693d4177261caa0aa3c15405ce05f5a7 Mon Sep 17 00:00:00 2001 From: "Sungjun, Kim" Date: Sat, 11 May 2019 22:26:48 +0900 Subject: [PATCH 4/4] Fix location --- containers/test/test_container_rpc.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/containers/test/test_container_rpc.sh b/containers/test/test_container_rpc.sh index 280f075d5..04c5e2547 100755 --- a/containers/test/test_container_rpc.sh +++ b/containers/test/test_container_rpc.sh @@ -25,6 +25,7 @@ success=false PORT_RANGE_START=10000 PORT_RANGE_END=20000 +RPC_SERVICE_PORT=`perl -e "print int(rand($PORT_RANGE_END-$PORT_RANGE_START)) + $PORT_RANGE_START"` function clean_up { # Perform program exit housekeeping @@ -50,7 +51,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $DIR # Start python rpc test container -RPC_SERVICE_PORT=`perl -e "print int(rand($PORT_RANGE_END-$PORT_RANGE_START)) + $PORT_RANGE_START"` echo "Starting python RPC test container... (port:$RPC_SERVICE_PORT)" python ../python/rpc_test_container.py --rpc_service_port $RPC_SERVICE_PORT & @@ -71,7 +71,6 @@ cd $DIR/../../ cd container make container_rpc_test container_uptime_seconds=180 -RPC_SERVICE_PORT=`perl -e "print int(rand($PORT_RANGE_END-$PORT_RANGE_START)) + $PORT_RANGE_START"` echo "Starting cpp RPC test container... (port:$RPC_SERVICE_PORT)" ./container_rpc_test -t $container_uptime_seconds -p $RPC_SERVICE_PORT & @@ -80,13 +79,11 @@ sleep 10s cd $DIR/../../debug/src/benchmarks make rpctest REDIS_PORT=$1 -RPC_SERVICE_PORT=`perl -e "print int(rand($PORT_RANGE_END-$PORT_RANGE_START)) + $PORT_RANGE_START"` echo "Executing RPC test (first iteration)... (redis port:$REDIS_PORT, rpc_service_port:$RPC_SERVICE_PORT)" ./rpctest --num_containers=2 --timeout_seconds=30 --redis_port $REDIS_PORT --rpc_service_port $RPC_SERVICE_PORT redis-cli -p $REDIS_PORT "flushall" echo "Sleeping for 5 seconds..." sleep 5s -RPC_SERVICE_PORT=`perl -e "print int(rand($PORT_RANGE_END-$PORT_RANGE_START)) + $PORT_RANGE_START"` echo "Executing RPC test (second iteration)... (redis port:$REDIS_PORT, rpc_service_port:$RPC_SERVICE_PORT)" ./rpctest --num_containers=2 --timeout_seconds=30 --redis_port $REDIS_PORT --rpc_service_port $RPC_SERVICE_PORT redis-cli -p $REDIS_PORT "flushall"