Skip to content
Test

test: add a case for node recovery in a broken cluster #982

Sign in to view logs

test: add a case for node recovery in a broken cluster

test: add a case for node recovery in a broken cluster #982

Workflow file for this run

.github/workflows/test.yaml at adaf4fd
---
# @see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
#
# Services feature of GitHub Actions isn't fit for our purposes for testing.
# We cannot overwrite arguments of ENTRYPOINT.
# @see https://docs.docker.com/engine/reference/commandline/create/#options
#
# @see https://gh.neting.ccmunity/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662
name: Test
on:
push:
branches:
- master
pull_request:
branches:
- master
defaults:
run:
shell: bash
jobs:
main:
name: Main
timeout-minutes: 10
runs-on: ubuntu-latest
strategy:
fail-fast: false
max-parallel: 10
matrix:
include:
- {redis: '7.2', ruby: '3.3'}
- {task: test_cluster_broken, restart: 'no', startup: '6'}
- {redis: '7.2', ruby: '3.3', compose: compose.ssl.yaml}
- {redis: '7.2', ruby: '3.3', driver: 'hiredis'}
- {redis: '7.2', ruby: '3.3', driver: 'hiredis', compose: compose.ssl.yaml}
- {redis: '7.2', ruby: '3.3', compose: compose.replica.yaml, replica: '2'}
- {task: test_cluster_state, pattern: 'PrimaryOnly', compose: compose.valkey.yaml, redis: '8', replica: '2', startup: '9'}
- {task: test_cluster_state, pattern: 'Pooled', compose: compose.valkey.yaml, redis: '8', replica: '2', startup: '9'}
- {task: test_cluster_state, pattern: 'ScaleReadRandom', compose: compose.valkey.yaml, redis: '8', replica: '2', startup: '9'}
- {task: test_cluster_state, pattern: 'ScaleReadRandomWithPrimary', compose: compose.valkey.yaml, redis: '8', replica: '2', startup: '9'}
- {task: test_cluster_state, pattern: 'ScaleReadLatency', compose: compose.valkey.yaml, redis: '8', replica: '2', startup: '9'}
- {ruby: 'jruby'}
- {ruby: 'truffleruby'}
- {task: test_cluster_down}
- {redis: '8', ruby: '3.3', compose: compose.valkey.yaml, replica: '2'}
- {redis: '7.2', ruby: '3.2', compose: compose.auth.yaml}
- {redis: '7.0', ruby: '3.1'}
- {redis: '6.2', ruby: '3.0'}
- {redis: '5.0', ruby: '2.7'}
- {task: test_cluster_scale, pattern: 'Single', compose: compose.scale.yaml, startup: '8'}
- {task: test_cluster_scale, pattern: 'Pipeline', compose: compose.scale.yaml, startup: '8'}
- {task: test_cluster_scale, pattern: 'Transaction', compose: compose.scale.yaml, startup: '8'}
- {task: test_cluster_scale, pattern: 'PubSub', compose: compose.scale.yaml, startup: '8'}
env:
REDIS_VERSION: ${{ matrix.redis || '7.2' }}
DOCKER_COMPOSE_FILE: ${{ matrix.compose || 'compose.yaml' }}
REDIS_CONNECTION_DRIVER: ${{ matrix.driver || 'ruby' }}
REDIS_REPLICA_SIZE: ${{ matrix.replica || '1' }}
RESTART_POLICY: ${{ matrix.restart || 'always' }}
REDIS_CLIENT_MAX_STARTUP_SAMPLE: ${{ matrix.startup || '3' }}
TEST_CLASS_PATTERN: ${{ matrix.pattern || '' }}
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: ${{ matrix.ruby || '3.3' }}
bundler-cache: true
- name: Pull Docker images
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull
- name: Run containers
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d
- name: Wait for Redis cluster to be ready
run: bundle exec rake wait
- name: Print containers
run: docker compose -f $DOCKER_COMPOSE_FILE ps
- name: Run minitest
run: bundle exec rake ${{ matrix.task || 'test' }}
- name: Stop containers
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true
nat-ted-env:
name: NAT-ted Environments
timeout-minutes: 5
runs-on: ubuntu-latest
env:
REDIS_VERSION: '7.2'
DOCKER_COMPOSE_FILE: 'compose.nat.yaml'
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: '3.3'
bundler-cache: true
- name: Get IP address of host
run: |
host_ip_addr=$(ip a | grep eth0 | grep inet | awk '{print $2}' | cut -d'/' -f1)
echo "HOST_IP_ADDR=$host_ip_addr" >> $GITHUB_ENV
- name: Pull Docker images
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull
- name: Run containers
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d
env:
HOST_ADDR: ${{ env.HOST_IP_ADDR }}
- name: Wait for nodes to be ready
run: |
node_cnt=$(docker compose -f $DOCKER_COMPOSE_FILE ps --format json | jq -s 'length')
i=0
while :
do
if [[ $i -gt $MAX_ATTEMPTS ]]
then
echo "Max attempts exceeded: $i times"
exit 1
fi
healthy_cnt=$(docker compose -f $DOCKER_COMPOSE_FILE ps --format json | jq -s 'map(select(.Health == "healthy")) | length')
if [[ $healthy_cnt -eq $node_cnt ]]
then
break
fi
echo 'Waiting for nodes to be ready...'
sleep 3
: $((++i))
done
env:
MAX_ATTEMPTS: "10"
- name: Print containers
run: docker compose -f $DOCKER_COMPOSE_FILE ps
- name: Build cluster
run: bundle exec rake "build_cluster[$HOST_ADDR]"
env:
HOST_ADDR: ${{ env.HOST_IP_ADDR }}
DEBUG: "1"
- name: Run minitest
run: bundle exec rake test
- name: Stop containers
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true
lint:
name: Lint
timeout-minutes: 5
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: '3.3'
bundler-cache: true
- name: Run rubocop
run: bundle exec rubocop
benchmark:
name: Benchmark
timeout-minutes: 10
runs-on: ubuntu-latest
env:
REDIS_VERSION: '7.2'
DOCKER_COMPOSE_FILE: 'compose.latency.yaml'
REDIS_REPLICA_SIZE: '2'
REDIS_CLIENT_MAX_THREADS: '10'
DELAY_TIME: '1ms'
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: '3.3'
bundler-cache: true
- name: Pull Docker images
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull
- name: Run containers
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d
- name: Wait for Redis cluster to be ready
run: bundle exec rake wait
- name: Print containers
run: docker compose -f $DOCKER_COMPOSE_FILE ps
- name: Rebuild cluster for balancing of replicas
run: bundle exec rake build_cluster_for_bench
env:
DEBUG: '1'
- name: Print topology
run: |
for i in {1..9}
do
echo "node$i: $(docker compose -f $DOCKER_COMPOSE_FILE exec node$i redis-cli cluster nodes | grep myself)"
done
- name: Ping nodes
run: |
for i in {1..9}
do
node_addr="$(docker compose -f $DOCKER_COMPOSE_FILE exec node$i redis-cli cluster nodes | grep myself | awk '{print $2}' | cut -d'@' -f1 | cut -d':' -f1)"
echo "node$i:"
ping -c 5 $node_addr
done
- name: Print cpu info
run: grep 'model name' /proc/cpuinfo
- name: Print memory info
run: free -w
- name: Print disk info
run: df -h
- name: Run minitest
run: bundle exec rake bench | grep BenchCommand | grep -v 'Envoy#bench_pipeline_echo\|Envoy#bench_single_echo' | sort
- name: Reset qdisc
run: |
for i in {5..9..2}
do
docker compose -f $DOCKER_COMPOSE_FILE exec node$i tc qdisc del dev eth0 root netem || true
done
- name: Stop containers
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true
ips:
name: IPS
timeout-minutes: 10
runs-on: ubuntu-latest
env:
REDIS_VERSION: '7.2'
DOCKER_COMPOSE_FILE: 'compose.latency.yaml'
REDIS_REPLICA_SIZE: '2'
REDIS_CLIENT_MAX_THREADS: '10'
DELAY_TIME: '0ms'
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: '3.3'
bundler-cache: true
- name: Pull Docker images
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull
- name: Run containers
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d
- name: Wait for Redis cluster to be ready
run: bundle exec rake wait
- name: Print containers
run: docker compose -f $DOCKER_COMPOSE_FILE ps
- name: Print cpu info
run: grep 'model name' /proc/cpuinfo
- name: Run iteration per second
run: bundle exec rake ips
- name: Stop containers
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true
profiling:
name: Profiling
timeout-minutes: 5
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
mode:
- single
- excessive_pipelining
- pipelining_in_moderation
- original_mget
- emulated_mget
env:
REDIS_VERSION: '7.2'
DOCKER_COMPOSE_FILE: 'compose.yaml'
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: '3.3'
bundler-cache: true
- name: Pull Docker images
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull
- name: Run containers
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d
- name: Wait for Redis cluster to be ready
run: bundle exec rake wait
- name: Print containers
run: docker compose -f $DOCKER_COMPOSE_FILE ps
- name: Run profiler
run: bundle exec rake prof
env:
PROFILE_MODE: ${{ matrix.mode }}
- name: Stop containers
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true
massive:
name: Massive Cluster
timeout-minutes: 10
runs-on: ubuntu-latest
env:
REDIS_VERSION: '7.2'
DOCKER_COMPOSE_FILE: 'compose.massive.yaml'
REDIS_SHARD_SIZE: '10'
REDIS_REPLICA_SIZE: '2'
REDIS_CLIENT_MAX_STARTUP_SAMPLE: '5'
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: '3.3'
bundler-cache: true
- name: Print user limits
run: ulimit -a
- name: Print kernel params
run: |
sysctl fs.file-max
sysctl vm.swappiness
sysctl vm.overcommit_memory
sysctl net.ipv4.tcp_sack
sysctl net.ipv4.tcp_timestamps
sysctl net.ipv4.tcp_window_scaling
sysctl net.ipv4.tcp_congestion_control
sysctl net.ipv4.tcp_syncookies
sysctl net.ipv4.tcp_tw_reuse
sysctl net.ipv4.tcp_max_syn_backlog
sysctl net.core.somaxconn
sysctl net.core.rmem_max
sysctl net.core.wmem_max
- name: Tune kernel params for redis
run: |
# https://developer.redis.com/operate/redis-at-scale/talking-to-redis/initial-tuning/
sudo sysctl -w vm.overcommit_memory=1
sudo sysctl -w net.ipv4.tcp_tw_reuse=1 # reuse sockets quickly
sudo sysctl -w net.ipv4.tcp_max_syn_backlog=1024 # backlog setting
sudo sysctl -w net.core.somaxconn=1024 # up the number of connections per port
- name: Pull Docker images
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull
- name: Run containers
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d
- name: Print memory info
run: free -w
- name: Wait for Redis cluster to be ready
run: bundle exec rake wait
env:
DEBUG: '1'
- name: Print containers
run: docker compose -f $DOCKER_COMPOSE_FILE ps
- name: Run profiler
run: bundle exec rake prof
env:
PROFILE_MODE: pipelining_in_moderation
- name: Stop containers
run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true