diff --git a/cicd/k3s-calico-incluster/Vagrantfile b/cicd/k3s-calico-incluster/Vagrantfile index ce731e274..d92df0c26 100644 --- a/cicd/k3s-calico-incluster/Vagrantfile +++ b/cicd/k3s-calico-incluster/Vagrantfile @@ -31,8 +31,8 @@ Vagrant.configure("2") do |config| master.vm.network :private_network, ip: "192.168.90.10", :netmask => "255.255.255.0" master.vm.provision :shell, :path => "master1.sh" master.vm.provider :virtualbox do |vbox| - vbox.customize ["modifyvm", :id, "--memory", 2048] - vbox.customize ["modifyvm", :id, "--cpus", 2] + vbox.customize ["modifyvm", :id, "--memory", 4096] + vbox.customize ["modifyvm", :id, "--cpus", 3] end end @@ -42,8 +42,8 @@ Vagrant.configure("2") do |config| master.vm.network :private_network, ip: "192.168.90.11", :netmask => "255.255.255.0" master.vm.provision :shell, :path => "master2.sh" master.vm.provider :virtualbox do |vbox| - vbox.customize ["modifyvm", :id, "--memory", 2048] - vbox.customize ["modifyvm", :id, "--cpus", 2] + vbox.customize ["modifyvm", :id, "--memory", 4096] + vbox.customize ["modifyvm", :id, "--cpus", 3] end end diff --git a/cicd/k3s-calico-incluster/loxilb.yml b/cicd/k3s-calico-incluster/loxilb.yml index a216f8796..639966c96 100644 --- a/cicd/k3s-calico-incluster/loxilb.yml +++ b/cicd/k3s-calico-incluster/loxilb.yml @@ -32,7 +32,7 @@ spec: containers: - name: loxilb-app image: "ghcr.io/loxilb-io/loxilb:latest" - command: [ "/root/loxilb-io/loxilb/loxilb", "--bgp", "--egr-hooks", "--blacklist=cali.|tunl.|vxlan[.]calico|veth." ] + command: [ "/root/loxilb-io/loxilb/loxilb", "--bgp", "--egr-hooks", "--blacklist=cali.|tunl.|vxlan[.]calico|veth.|cni[0-9a-z]" ] ports: - containerPort: 11111 - containerPort: 1791 diff --git a/cicd/k3s-calico-incluster/master1.sh b/cicd/k3s-calico-incluster/master1.sh index 3592b42b9..074c3a68d 100755 --- a/cicd/k3s-calico-incluster/master1.sh +++ b/cicd/k3s-calico-incluster/master1.sh @@ -6,8 +6,8 @@ echo $MASTER_IP > /vagrant/master-ip cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token sed -i -e "s/127.0.0.1/${MASTER_IP}/g" /etc/rancher/k3s/k3s.yaml cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml -#sudo kubectl apply -f /vagrant/loxilb.yml -#sudo kubectl apply -f /vagrant/kube-loxilb.yml +sudo kubectl apply -f /vagrant/loxilb.yml +sudo kubectl apply -f /vagrant/kube-loxilb.yml sudo kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/tigera-operator.yaml sudo kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/custom-resources.yaml /vagrant/wait_ready.sh diff --git a/cicd/k3s-calico-incluster/master2.sh b/cicd/k3s-calico-incluster/master2.sh index 6a34fcf71..f9a09c13b 100644 --- a/cicd/k3s-calico-incluster/master2.sh +++ b/cicd/k3s-calico-incluster/master2.sh @@ -3,7 +3,6 @@ export WORKER_ADDR=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80' export MASTER_ADDR=$(cat /vagrant/master-ip) export NODE_TOKEN=$(cat /vagrant/node-token) -#curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.11 --disable-cloud-controller -t ${NODE_TOKEN} curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.11 -t ${NODE_TOKEN} /vagrant/wait_ready.sh diff --git a/cicd/k3s-calico-incluster/validation.sh b/cicd/k3s-calico-incluster/validation.sh index c96f4ff18..9b0d51013 100755 --- a/cicd/k3s-calico-incluster/validation.sh +++ b/cicd/k3s-calico-incluster/validation.sh @@ -1,6 +1,6 @@ #!/bin/bash source ../common.sh -echo k3s-flannel-cluster +echo k3s-calico-incluster if [ "$1" ]; then KUBECONFIG="$1" @@ -9,48 +9,34 @@ fi # Set space as the delimiter IFS=' ' -sleep 45 +sleep 5 extIP="123.123.123.1" echo $extIP echo "Service Info" -vagrant ssh master -c 'sudo kubectl get svc' -echo "LB Info" -vagrant ssh loxilb -c 'sudo docker exec -i loxilb loxicmd get lb -o wide' -echo "EP Info" -vagrant ssh loxilb -c 'sudo docker exec -i loxilb loxicmd get ep -o wide' +vagrant ssh master1 -c 'sudo kubectl get svc' print_debug_info() { - echo "llb1 route-info" - vagrant ssh loxilb -c 'ip route' - vagrant ssh master -c 'sudo kubectl get pods -A' - vagrant ssh master -c 'sudo kubectl get svc' - vagrant ssh master -c 'sudo kubectl get nodes' + echo "cluster-info" + vagrant ssh master1 -c 'sudo kubectl get pods -A' + vagrant ssh master1 -c 'sudo kubectl get svc' + vagrant ssh master1 -c 'sudo kubectl get nodes' } out=$(curl -s --connect-timeout 10 http://$extIP:55002) if [[ ${out} == *"Welcome to nginx"* ]]; then - echo "k3s-flannel-cluster (kube-loxilb) tcp [OK]" + echo "k3s-calico-incluster (kube-loxilb) tcp [OK]" else - echo "k3s-flannel-cluster (kube-loxilb) tcp [FAILED]" + echo "k3s-calico-incluster (kube-loxilb) tcp [FAILED]" print_debug_info exit 1 fi out=$(timeout 10 ../common/udp_client $extIP 55003) if [[ ${out} == *"Client"* ]]; then - echo "k3s-flannel-cluster (kube-loxilb) udp [OK]" + echo "k3s-calico-incluster (kube-loxilb) udp [OK]" else - echo "k3s-flannel-cluster (kube-loxilb) udp [FAILED]" - print_debug_info - exit 1 -fi - -out=$(timeout 10 ../common/sctp_client 192.168.90.1 41291 $extIP 55004) -if [[ ${out} == *"server1"* ]]; then - echo "k3s-flannel-cluster (kube-loxilb) sctp [OK]" -else - echo "k3s-flannel-cluster (kube-loxilb) sctp [FAILED]" + echo "k3s-calico-incluster (kube-loxilb) udp [FAILED]" print_debug_info exit 1 fi diff --git a/cicd/k3s-calico-incluster/worker.sh b/cicd/k3s-calico-incluster/worker.sh index 0e9350d89..c476838d8 100644 --- a/cicd/k3s-calico-incluster/worker.sh +++ b/cicd/k3s-calico-incluster/worker.sh @@ -5,8 +5,7 @@ export NODE_TOKEN=$(cat /vagrant/node-token) mkdir -p /etc/rancher/k3s cp -f /vagrant/k3s.yaml /etc/rancher/k3s/k3s.yaml curl -sfL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - agent --server https://192.168.80.10:6443 --node-ip=${WORKER_ADDR} --node-external-ip=${WORKER_ADDR} -t ${NODE_TOKEN} -#sudo kubectl apply -f /vagrant/loxilb-peer.yml +sudo kubectl apply -f /vagrant/loxilb-peer.yml sudo kubectl apply -f /vagrant/nginx.yml -#sudo kubectl apply -f /vagrant/udp.yml -#sudo kubectl apply -f /vagrant/sctp.yml +sudo kubectl apply -f /vagrant/udp.yml /vagrant/wait_ready.sh diff --git a/cicd/k3s-flannel-incluster/validation.sh b/cicd/k3s-flannel-incluster/validation.sh index c96f4ff18..d1a00eeec 100755 --- a/cicd/k3s-flannel-incluster/validation.sh +++ b/cicd/k3s-flannel-incluster/validation.sh @@ -9,23 +9,18 @@ fi # Set space as the delimiter IFS=' ' -sleep 45 +sleep 5 extIP="123.123.123.1" echo $extIP echo "Service Info" -vagrant ssh master -c 'sudo kubectl get svc' -echo "LB Info" -vagrant ssh loxilb -c 'sudo docker exec -i loxilb loxicmd get lb -o wide' -echo "EP Info" -vagrant ssh loxilb -c 'sudo docker exec -i loxilb loxicmd get ep -o wide' +vagrant ssh master1 -c 'sudo kubectl get svc' print_debug_info() { - echo "llb1 route-info" - vagrant ssh loxilb -c 'ip route' - vagrant ssh master -c 'sudo kubectl get pods -A' - vagrant ssh master -c 'sudo kubectl get svc' - vagrant ssh master -c 'sudo kubectl get nodes' + echo "cluster-info" + vagrant ssh master1 -c 'sudo kubectl get pods -A' + vagrant ssh master1 -c 'sudo kubectl get svc' + vagrant ssh master1 -c 'sudo kubectl get nodes' } out=$(curl -s --connect-timeout 10 http://$extIP:55002) diff --git a/cicd/k8s-calico-incluster/Vagrantfile b/cicd/k8s-calico-incluster/Vagrantfile index e18a50ccc..e68faecb6 100644 --- a/cicd/k8s-calico-incluster/Vagrantfile +++ b/cicd/k8s-calico-incluster/Vagrantfile @@ -6,21 +6,18 @@ settings = YAML.load_file "yaml/settings.yaml" workers = settings["nodes"]["workers"]["count"] -bname = ("sysnet4admin/Ubuntu-k8s") -bversion = "0.7.1" - Vagrant.configure("2") do |config| if Vagrant.has_plugin?("vagrant-vbguest") config.vbguest.auto_update = false end - config.vm.box = "#{bname}" - config.vm.box_version = "#{bversion}" + config.vm.box = settings["software"]["cluster"]["box"]["name"] + config.vm.box_version = settings["software"]["cluster"]["box"]["version"] config.vm.define "host" do |host| host.vm.hostname = 'host1' - #loxilb.vm.network "forwarded_port", guest: 55002, host: 5502, protocol: "tcp" + host.vm.network :private_network, ip: settings["network"]["client_ip"], :netmask => "255.255.255.0" host.vm.network :private_network, ip: "192.168.80.9", :netmask => "255.255.255.0" host.vm.network :private_network, ip: "192.168.90.9", :netmask => "255.255.255.0" host.vm.provision :shell, :path => "node_scripts/host.sh" @@ -30,10 +27,10 @@ Vagrant.configure("2") do |config| end end - #config.vm.box = settings["software"]["cluster"]["box"] config.vm.define "master" do |master| - master.vm.hostname = 'master' + master.vm.hostname = 'master1' master.vm.network :private_network, ip: settings["network"]["control_ip"], :netmask => "255.255.255.0" + master.vm.network :private_network, ip: settings["network"]["control_ip2"], :netmask => "255.255.255.0" master.vm.provision "shell", env: { "DNS_SERVERS" => settings["network"]["dns_servers"].join(" "), @@ -52,8 +49,8 @@ Vagrant.configure("2") do |config| path: "node_scripts/master.sh" master.vm.provider :virtualbox do |vbox| - vbox.customize ["modifyvm", :id, "--memory", 2048] - vbox.customize ["modifyvm", :id, "--cpus", 2] + vbox.customize ["modifyvm", :id, "--memory", 4096] + vbox.customize ["modifyvm", :id, "--cpus", 3] end end diff --git a/cicd/k8s-calico-incluster/config.sh b/cicd/k8s-calico-incluster/config.sh index 68ebe3d3a..85efe4c5e 100755 --- a/cicd/k8s-calico-incluster/config.sh +++ b/cicd/k8s-calico-incluster/config.sh @@ -30,7 +30,7 @@ do sleep 10 done -#Create fullnat Service +# Create fullnat Services vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/tcp_fullnat.yml' 2> /dev/null vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/udp_fullnat.yml' 2> /dev/null vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp_fullnat.yml' 2> /dev/null diff --git a/cicd/k8s-calico-incluster/validation.sh b/cicd/k8s-calico-incluster/validation.sh index 05debe23d..b33bda1e4 100755 --- a/cicd/k8s-calico-incluster/validation.sh +++ b/cicd/k8s-calico-incluster/validation.sh @@ -1,6 +1,6 @@ #!/bin/bash source ../common.sh -echo k8s-calico +echo k8s-calico-incluster if [ "$1" ]; then KUBECONFIG="$1" @@ -11,11 +11,11 @@ IFS=' ' for((i=0; i<120; i++)) do - extLB=$(vagrant ssh master -c 'kubectl get svc' 2> /dev/null | grep "tcp-lb-default") + extLB=$(vagrant ssh master -c 'kubectl get svc' 2> /dev/null | grep "tcp-lb-fullnat") read -a strarr <<< "$extLB" len=${#strarr[*]} if [[ $((len)) -lt 6 ]]; then - echo "Can't find tcp-lb service" + echo "Can't find tcp-lb-fullnat" sleep 1 continue fi @@ -42,61 +42,31 @@ echo "************************************************************************** echo -e "\nPod List" echo "******************************************************************************" vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null -echo "******************************************************************************" -echo -e "\nLB List" -echo "******************************************************************************" -vagrant ssh loxilb -c 'sudo docker exec -it loxilb loxicmd get lb -o wide' 2> /dev/null -echo "******************************************************************************" -echo -e "\nEP List" -echo "******************************************************************************" -vagrant ssh loxilb -c 'sudo docker exec -it loxilb loxicmd get ep -o wide' 2> /dev/null -echo "******************************************************************************" echo -e "\nTEST RESULTS" echo "******************************************************************************" -mode=( "default" "onearm" "fullnat" ) -tcp_port=( 55002 56002 57002 ) -udp_port=( 55003 56003 57003 ) -sctp_port=( 55004 56004 57004 ) +mode=( "fullnat" ) +tcp_port=( 57002 ) +udp_port=( 57003 ) +sctp_port=( 57004 ) code=0 -for ((i=0;i<=2;i++)); do -out=$(curl -s --connect-timeout 10 http://$extIP:${tcp_port[i]}) -if [[ ${out} == *"Welcome to nginx"* ]]; then - echo -e "K8s-calico TCP\t(${mode[i]})\t[OK]" +for ((i=0;i<1;i++)); do +out=$(vagrant ssh host -c "curl -s --connect-timeout 10 http://$extIP:${tcp_port[i]}") +echo $out +if [[ ${out} == *"nginx"* ]]; then + echo -e "K8s-calico-incluster TCP\t(${mode[i]})\t[OK]" else - echo -e "K8s-calico TCP\t(${mode[i]})\t[FAILED]" - ## Dump some debug info - echo "llb1 lb-info" - vagrant ssh loxilb -c 'sudo docker exec -it llb1 loxicmd get lb -o wide' 2> /dev/null - echo "llb1 route-info" - vagrant ssh loxilb -c 'sudo docker exec -it llb1 ip route' 2> /dev/null + echo -e "K8s-calico-incluster TCP\t(${mode[i]})\t[FAILED]" code=1 fi -out=$(timeout 5 ../common/udp_client $extIP ${udp_port[i]}) +out=$(vagrant ssh host -c "timeout 5 /vagrant/tools/udp_client $extIP ${udp_port[i]}") if [[ ${out} == *"Client"* ]]; then - echo -e "K8s-calico UDP\t(${mode[i]})\t[OK]" + echo -e "K8s-calico-incluster UDP\t(${mode[i]})\t[OK]" else - echo -e "K8s-calico UDP\t(${mode[i]})\t[FAILED]" - ## Dump some debug info - echo "llb1 lb-info" - vagrant ssh loxilb -c 'sudo docker exec -it llb1 loxicmd get lb -o wide' 2> /dev/null - echo "llb1 route-info" - vagrant ssh loxilb -c 'sudo docker exec -it llb1 ip route' 2> /dev/null - code=1 -fi - -out=$(timeout 5 ../common/sctp_client 192.168.90.1 34951 $extIP ${sctp_port[i]}) -if [[ ${out} == *"server1"* ]]; then - echo -e "K8s-calico SCTP\t(${mode[i]})\t[OK]" -else - echo -e "K8s-calico SCTP\t(${mode[i]})\t[FAILED]" - ## Dump some debug info - echo "llb1 lb-info" - vagrant ssh loxilb -c 'sudo docker exec -it llb1 loxicmd get lb -o wide' 2> /dev/null - echo "llb1 route-info" - vagrant ssh loxilb -c 'sudo docker exec -it llb1 ip route' 2> /dev/null + echo -e "K8s-calico-incluster UDP\t(${mode[i]})\t[FAILED]" code=1 fi done + exit $code diff --git a/cicd/k8s-calico-incluster/yaml/settings.yaml b/cicd/k8s-calico-incluster/yaml/settings.yaml index ffa7fe692..7b6e513bb 100644 --- a/cicd/k8s-calico-incluster/yaml/settings.yaml +++ b/cicd/k8s-calico-incluster/yaml/settings.yaml @@ -9,10 +9,12 @@ cluster_name: Kubernetes Cluster # NO_PROXY=127.0.0.1,localhost,master-node,node01,node02,node03 # All IPs/CIDRs should be private and allowed in /etc/vbox/networks.conf. network: - iloxilb_ip: 192.168.80.9 - oloxilb_ip: 192.168.90.9 # Worker IPs are simply incremented from the control IP. + client_ip: 192.168.90.9 control_ip: 192.168.80.10 + control_ip2: 192.168.90.10 + control2_ip: 192.168.80.11 + control2_ip2: 192.168.90.11 dns_servers: - 8.8.8.8 - 1.1.1.1 @@ -20,11 +22,11 @@ network: service_cidr: 172.17.1.0/18 nodes: control: - cpu: 2 + cpu: 4 memory: 4096 workers: count: 2 - cpu: 1 + cpu: 2 memory: 2048 # Mount additional shared folders from the host into each virtual machine. # Note that the project directory is automatically mounted at /vagrant. @@ -32,12 +34,10 @@ nodes: # - host_path: ../images # vm_path: /vagrant/images software: - loxilb: + cluster: box: name: sysnet4admin/Ubuntu-k8s version: 0.7.1 - cluster: - box: bento/ubuntu-22.04 calico: 3.26.0 # To skip the dashboard installation, set its version to an empty value or comment it out: kubernetes: 1.27.1-00