-
Notifications
You must be signed in to change notification settings - Fork 454
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ToxicStub allow to unblocking write to Output
It could happen when Link has no reciever and there is some packets in buffer. It produces deadlock.
- Loading branch information
Showing
9 changed files
with
351 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
|
|
||
| <hazelcast xmlns="http://www.hazelcast.com/schema/config" | ||
| xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | ||
| xsi:schemaLocation="http://www.hazelcast.com/schema/config | ||
| http://www.hazelcast.com/schema/config/hazelcast-config-5.1.xsd"> | ||
|
|
||
| <properties> | ||
| <property name="hazelcast.merge.next.run.delay.seconds">15</property> | ||
| <property name="hazelcast.merge.first.run.delay.seconds">20</property> | ||
| <property name="hazelcast.partition.migration.chunks.enabled">false</property> | ||
| <property name="hazelcast.heartbeat.failuredetector.type">deadline</property> | ||
| <property name="hazelcast.heartbeat.interval.seconds">3</property> | ||
| <property name="hazelcast.max.no.heartbeat.seconds">10</property> | ||
| </properties> | ||
|
|
||
| <network> | ||
| <public-address>member-proxy:${proxyPort}</public-address> | ||
| <port auto-increment="false">5701</port> | ||
| <join> | ||
| <auto-detection enabled="false"/> | ||
| <tcp-ip enabled="true"> | ||
| <member-list> | ||
| <member>member-proxy:${proxyPort0}</member> | ||
| <member>member-proxy:${proxyPort1}</member> | ||
| <member>member-proxy:${proxyPort2}</member> | ||
| </member-list> | ||
| </tcp-ip> | ||
| </join> | ||
| </network> | ||
| </hazelcast> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Usage: | ||
| # test-e2e-hazelcast [docker image name for toxiproxy] | ||
|
|
||
| set -ueo pipefail | ||
|
|
||
| cd "$(dirname "$0")" | ||
|
|
||
| toxiproxy="../dist/toxiproxy-cli" | ||
| state="started" | ||
|
|
||
| wait_for_url() { | ||
| curl -s --retry-connrefused --retry 5 --retry-delay 2 --retry-max-time 30 \ | ||
| --max-time 1 -L -I -X GET "${1}" | ||
| } | ||
|
|
||
| # Stop all background jobs on exit | ||
| function cleanup() { | ||
| echo -e "\n\n== Teardown: state=${state}" | ||
| if [[ $state != "success" ]]; then | ||
| docker kill -s SIGQUIT member-proxy | ||
| docker logs -t member-proxy | ||
| fi | ||
| docker stop member-proxy member0 member1 member2 || true | ||
| docker network rm toxiproxy-e2e || true | ||
| } | ||
| trap "cleanup" EXIT SIGINT SIGTERM | ||
|
|
||
| IMAGE_HAZELCAST="hazelcast/hazelcast:5.1.2-slim" | ||
| IMAGE_TOXIPROXY="${1:-ghcr.io/shopify/toxiproxy:2.4.0}" | ||
| TOXIPROXY_BASE_URL="http://localhost:8474" | ||
|
|
||
| echo "= Toxiproxy E2E tests with Hazelcast cluster" | ||
| echo | ||
| echo "== Setup" | ||
| echo | ||
| echo "=== Starting Toxiproxy" | ||
|
|
||
| docker rm -f member-proxy member0 member1 member2 &>/dev/null | ||
| docker network rm toxiproxy-e2e &>/dev/null || true | ||
|
|
||
| docker network create --subnet 172.18.5.0/24 toxiproxy-e2e | ||
|
|
||
| docker run --rm -t "${IMAGE_TOXIPROXY}" --version | ||
| docker run -d \ | ||
| --name member-proxy \ | ||
| --network toxiproxy-e2e \ | ||
| --ip 172.18.5.2 \ | ||
| -p 8474:8474 \ | ||
| -e LOG_LEVEL=trace \ | ||
| "$IMAGE_TOXIPROXY" | ||
|
|
||
| echo "=== Wait Toxiproxy API is available" | ||
| wait_for_url "${TOXIPROXY_BASE_URL}/version" | ||
|
|
||
| echo "=== Prepare proxies for Hazelcast cluster" | ||
| for i in {0..2}; do | ||
| echo "> Create proxy for member${i} on port 600${i}" | ||
| # curl --data "{\"name\": \"member${i}\", \"upstream\": \"member${i}:5701\", \"listen\": \"0.0.0.0:600${i}\"}" "${TOXIPROXY_BASE_URL}/proxies" | ||
| $toxiproxy create -l "0.0.0.0:600${i}" -u "member${i}:5701" "member${i}" | ||
| echo | ||
| done | ||
|
|
||
| echo | ||
| echo "=== Strating Hazelcast containers" | ||
| for i in {0..2}; do | ||
| echo "> Start Hazelcast on host member${i}" | ||
| docker run -d --rm \ | ||
| --name "member${i}" \ | ||
| --network toxiproxy-e2e \ | ||
| --ip "172.18.5.1${i}" \ | ||
| --volume "${PWD}/hazelcast.xml:/opt/hazelcast/config/hazelcast-docker.xml" \ | ||
| --env HZ_PHONE_HOME_ENABLED=false \ | ||
| --env JAVA_OPTS="-DproxyPort=600${i} -DproxyPort0=6000 -DproxyPort1=6001 -DproxyPort2=6002" \ | ||
| "$IMAGE_HAZELCAST" | ||
| done | ||
|
|
||
| echo "> Wait for cluster join (30s)..." | ||
| sleep 30 | ||
|
|
||
| echo "> Output of member0" | ||
| docker logs -t -n 10 member0 | ||
|
|
||
| echo | ||
| echo "=== Initialize toxics for cluster" | ||
| for i in {0..2}; do | ||
| echo "> Adding toxics to member${i} proxy" | ||
| # curl --data "{\"name\": \"member${i}_downstream\", \"stream\": \"downstream\", \"toxicity\": 1.0, \"type\": \"bandwidth\", \"attributes\": { \"rate\": 0 }}" "${TOXIPROXY_BASE_URL}/proxies/member${i}/toxics" | ||
| $toxiproxy toxic add --type=bandwidth \ | ||
| --downstream \ | ||
| --toxicName="member${i}_downstream" \ | ||
| --attribute="rate=0" \ | ||
| --toxicity=1 \ | ||
| "member${i}" | ||
| # curl --data "{\"name\": \"member${i}_upstream\", \"stream\": \"upstream\", \"toxicity\": 1.0, \"type\": \"bandwidth\", \"attributes\": { \"rate\": 0 }}" "${TOXIPROXY_BASE_URL}/proxies/member${i}/toxics" | ||
| $toxiproxy toxic add --type=bandwidth \ | ||
| --upstream \ | ||
| --toxicName="member${i}_upstream" \ | ||
| --attribute="rate=0" \ | ||
| --toxicity=1 \ | ||
| "member${i}" | ||
| echo | ||
| $toxiproxy inspect "member${i}" | ||
| echo | ||
| done | ||
|
|
||
| echo "=== Wait for a the Hazelcast cluster split-brain (60s)..." | ||
| sleep 60 | ||
|
|
||
| echo "=== Validate output of Toxiproxy and single member" | ||
| docker logs -t -n 10 member0 | ||
| docker logs -t -n 10 member-proxy | ||
|
|
||
| echo "=== Removing toxics from proxies" | ||
| for i in {0..2}; do | ||
| echo "[$(date)] > Remove downstream bandwith Toxic for member${i} proxy" | ||
| # curl -X DELETE "${TOXIPROXY_BASE_URL}/proxies/member${i}/toxics/member${i}_downstream" | ||
| $toxiproxy toxic delete --toxicName="member${i}_downstream" "member${i}" | ||
| echo "[$(date)] > Remove ustream bandwith Toxic for member${i} proxy" | ||
| # curl -X DELETE "${TOXIPROXY_BASE_URL}/proxies/member${i}/toxics/member${i}_upstream" | ||
| $toxiproxy toxic delete --toxicName="member${i}_upstream" "member${i}" | ||
| done | ||
|
|
||
| echo "=== Validate output of Toxiproxy and single member after removing toxics" | ||
| docker logs -t -n 10 member0 | ||
| docker logs -t -n 10 member-proxy | ||
|
|
||
| $toxiproxy list | ||
| $toxiproxy inspect member0 | ||
| $toxiproxy inspect member1 | ||
| $toxiproxy inspect member2 | ||
|
|
||
| echo -e "=================\n" | ||
|
|
||
| echo "Succcess!" | ||
| state="success" |
Oops, something went wrong.