forked from sonic-net/sonic-buildimage
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Kubernetes]: The kube server could be used as http-proxy for docker (s…
…onic-net#7469) Why I did it The SONiC switches get their docker images from local repo, populated during install with container images pre-built into SONiC FW. With the introduction of kubernetes, new docker images available in remote repo could be deployed. This requires dockerd to be able to pull images from remote repo. Depending on the Switch network domain & config, it may or may not be able to reach the remote repo. In the case where remote repo is unreachable, we could potentially make Kubernetes server to also act as http-proxy. How I did it When admin explicitly enables, the kubernetes-server could be configured as docker-proxy. But any update to docker-proxy has to be via service-conf file environment variable, implying a "service restart docker" is required. But restart of dockerd is vey expensive, as it would restarts all dockers, including database docker. To avoid dockerd restart, pre-configure an http_proxy using an unused IP. When k8s server is enabled to act as http-proxy, an IP table entry would be created to direct all traffic to the configured-unused-proxy-ip to the kubernetes-master IP. This way any update to Kubernetes master config would be just manipulating IPTables, which will be transparent to all modules, until dockerd needs to download from remote repo. How to verify it Configure a switch such that image repo is unreachable Pre-configure dockerd with http_proxy.conf using an unused IP (e.g. 172.16.1.1) Update ctrmgrd.service to invoke ctrmgrd.py with "-p" option. Configure a k8s server, and deploy an image for feature with set_owner="kube" Check if switch could successfully download the image or not.
- Loading branch information
1 parent
0096765
commit f7ed82f
Showing
11 changed files
with
371 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,135 @@ | ||
| #!/usr/bin/env python3 | ||
|
|
||
| import ipaddress | ||
| import os | ||
| import re | ||
| import socket | ||
| import subprocess | ||
| import syslog | ||
|
|
||
| UNIT_TESTING = 0 | ||
|
|
||
| # NOTE: | ||
| # Unable to use python-iptables as that does not create rules per ip-tables default | ||
| # which is nf_tables. So rules added via iptc package will not be listed under | ||
| # "sudo iptables -t nat -L -n". But available in kernel. To list, we need to | ||
| # use legacy mode as "sudo iptables-legacy -t nat -L -n". | ||
| # As we can't use two modes and using non-default could make any debugging effort | ||
| # very tough. | ||
|
|
||
|
|
||
| from urllib.parse import urlparse | ||
|
|
||
| DST_FILE = "/etc/systemd/system/docker.service.d/http_proxy.conf" | ||
| DST_IP = None | ||
| DST_PORT = None | ||
| SQUID_PORT = "3128" | ||
|
|
||
| def _get_ip(ip_str): | ||
| ret = "" | ||
| if ip_str: | ||
| try: | ||
| ipaddress.ip_address(ip_str) | ||
| ret = ip_str | ||
| except ValueError: | ||
| pass | ||
|
|
||
| if not ret: | ||
| try: | ||
| ret = socket.gethostbyname(ip_str) | ||
| except (OSError, socket.error): | ||
| pass | ||
| if not ret: | ||
| syslog.syslog(syslog.LOG_ERR, "{} is neither IP nor resolves to IP". | ||
| format(ip_str)) | ||
| return ret | ||
|
|
||
|
|
||
| def _get_dst_info(): | ||
| global DST_IP, DST_PORT | ||
| DST_IP = None | ||
| DST_PORT = None | ||
| print("DST_FILE={}".format(DST_FILE)) | ||
| if os.path.exists(DST_FILE): | ||
| with open(DST_FILE, "r") as s: | ||
| for line in s.readlines(): | ||
| url_match = re.search('^Environment=.HTTP_PROXY=(.+?)"', line) | ||
| if url_match: | ||
| url = urlparse(url_match.group(1)) | ||
| DST_IP = _get_ip(url.hostname) | ||
| DST_PORT = url.port | ||
| break | ||
| else: | ||
| print("{} not available".format(DST_FILE)) | ||
| print("DST_IP={}".format(DST_IP)) | ||
|
|
||
|
|
||
| def _is_rule_match(rule): | ||
| expect = "DNAT tcp -- 0.0.0.0/0 {} tcp dpt:{} to:".format( | ||
| DST_IP, DST_PORT) | ||
|
|
||
| # Remove duplicate spaces | ||
| rule = " ".join(rule.split()).strip() | ||
|
|
||
| if rule.startswith(expect): | ||
| return rule[len(expect):] | ||
| else: | ||
| return "" | ||
|
|
||
|
|
||
| def check_proc(proc): | ||
| if proc.returncode: | ||
| syslog.syslog(syslog.LOG_ERR, "Failed to run: cmd: {}".format(proc.args)) | ||
| syslog.syslog(syslog.LOG_ERR, "Failed to run: stdout: {}".format(proc.stdout)) | ||
| syslog.syslog(syslog.LOG_ERR, "Failed to run: stderr: {}".format(proc.stderr)) | ||
| if not UNIT_TESTING: | ||
| assert False | ||
|
|
||
|
|
||
| def iptable_proxy_rule_upd(ip_str, port = SQUID_PORT): | ||
| _get_dst_info() | ||
| if not DST_IP: | ||
| # There is no proxy in use. Bail out. | ||
| return "" | ||
|
|
||
| destination = "" | ||
| if ip_str: | ||
| upd_ip = _get_ip(ip_str) | ||
| if not upd_ip: | ||
| return "" | ||
| destination = "{}:{}".format(upd_ip, port) | ||
|
|
||
| found = False | ||
| num = 0 | ||
|
|
||
| while True: | ||
| num += 1 | ||
|
|
||
| cmd = "sudo iptables -t nat -n -L OUTPUT {}".format(num) | ||
| proc = subprocess.run(cmd, shell=True, capture_output=True) | ||
| check_proc(proc) | ||
|
|
||
| if not proc.stdout: | ||
| # No more rule | ||
| break | ||
|
|
||
| rule_dest = _is_rule_match(proc.stdout.decode("utf-8").strip()) | ||
| if rule_dest: | ||
| if not found and destination and (rule_dest == destination): | ||
| found = True | ||
| else: | ||
| # Duplicate or different IP - delete it | ||
| cmd = "sudo iptables -t nat -D OUTPUT {}".format(num) | ||
| proc = subprocess.run(cmd, shell=True, capture_output=True) | ||
| check_proc(proc) | ||
| # Decrement number to accommodate deleted rule | ||
| num -= 1 | ||
|
|
||
| if destination and not found: | ||
| cmd = "sudo iptables -t nat -A OUTPUT -p tcp -d {} --dport {} -j DNAT --to-destination {}".format( | ||
| DST_IP, DST_PORT, destination) | ||
| proc = subprocess.run(cmd, shell=True, capture_output=True) | ||
|
|
||
| check_proc(proc) | ||
|
|
||
| return destination |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.