Merged PR 3845699: [linkmgrd]: Introduce MUX cable linkmgrd

Linkmgrd monitors link status, mux status, and link state. Has
the link becomes unhealthy, linkmgrd will trigger mux switchover
on a standby ToR ensuring uninterrupted service to servers/blades.
This PR is initial implementation of linkmgrd.

Also, docker-mux container hold packages related to maintaining and managing
mux cable. It currently runs linkmgrd binary that monitor and switches
the mux if needed.
This PR also introduces mux-container and starts linkmgrd as startup when
build is configured with INCLUDE_MUX=y

signed-off-by: Tamer Ahmed <tamer.ahmed@microsoft.com>

Related work items: sonic-net#2315, #3146150

Makefile.work

@@ -10,6 +10,7 @@
 #  * ENABLE_ZTP: Enables zero touch provisioning.
 #  * SHUTDOWN_BGP_ON_START: Sets admin-down state for all bgp peerings after restart.
 #  * INCLUDE_KUBERNETES: Allows including Kubernetes
+#  * INCLUDE_MUX: Include MUX feature/services for TOR switch.
 #  * ENABLE_PFCWD_ON_START: Enable PFC Watchdog (PFCWD) on server-facing ports
 #  * by default for TOR switch.
 #  * ENABLE_SYNCD_RPC: Enables rpc-based syncd builds.
@@ -243,6 +244,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            SONIC_INCLUDE_SYSTEM_TELEMETRY=$(INCLUDE_SYSTEM_TELEMETRY) \
                            SONIC_INCLUDE_RESTAPI=$(INCLUDE_RESTAPI) \
                            SONIC_INCLUDE_ACMS=$(INCLUDE_ACMS) \
+                           SONIC_INCLUDE_MUX=$(INCLUDE_MUX) \
                            TELEMETRY_WRITABLE=$(TELEMETRY_WRITABLE) \
                            EXTRA_DOCKER_TARGETS=$(EXTRA_DOCKER_TARGETS) \
                            BUILD_LOG_TIMESTAMP=$(BUILD_LOG_TIMESTAMP) \

dockers/docker-mux/Dockerfile.j2

@@ -0,0 +1,36 @@
+{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+FROM docker-config-engine-buster
+
+ARG docker_container_name
+RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf
+
+## Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update &&       \
+    apt-get install -f -y   \
+        libmnl0
+
+{% if docker_mux_debs.strip() -%}
+# Copy locally-built Debian package dependencies
+{{ copy_files("debs/", docker_mux_debs.split(' '), "/debs/") }}
+
+# Install locally-built Debian packages and implicitly install their dependencies
+{{ install_debian_packages(docker_mux_debs.split(' ')) }}
+{%- endif %}
+
+## Clean up
+RUN apt-get clean -y        && \
+    apt-get autoclean -y    && \
+    apt-get autoremove -y   && \
+    rm -rf /debs
+
+COPY ["docker-init.sh", "/usr/bin/"]
+COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor/"]
+
+## Copy all Jinja2 template files into the templates folder
+COPY ["*.j2", "/usr/share/sonic/templates/"]
+
+ENTRYPOINT ["/usr/bin/docker-init.sh"]

dockers/docker-mux/critical_processes

@@ -0,0 +1 @@
+program:linkmgrd

dockers/docker-mux/docker-init.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# Generate supervisord config file
+mkdir -p /etc/supervisor/conf.d/
+
+# The docker container should start this script as PID 1, so now that supervisord is
+# properly configured, we exec supervisord so that it runs as PID 1 for the
+# duration of the container's lifetime
+exec /usr/local/bin/supervisord

dockers/docker-mux/supervisord.conf

@@ -0,0 +1,41 @@
+[supervisord]
+logfile_maxbytes=1MB
+logfile_backups=2
+nodaemon=true
+
+[eventlistener:dependent-startup]
+command=python3 -m supervisord_dependent_startup
+autostart=true
+autorestart=unexpected
+startretries=0
+exitcodes=0,3
+events=PROCESS_STATE
+buffer_size=100
+
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener --container-name mux
+events=PROCESS_STATE_EXITED,PROCESS_STATE_RUNNING
+autostart=true
+autorestart=unexpected
+
+[program:rsyslogd]
+command=/usr/sbin/rsyslogd -n -iNONE
+priority=1
+autostart=false
+autorestart=unexpected
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+
+[program:linkmgrd]
+command=nice -n -20 /usr/sbin/linkmgrd -l /usr/share/sonic/templates/mux_config.json -v info
+priority=2
+autostart=false
+autorestart=false
+startsecs=0
+startretries=0
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=rsyslogd:running
+

files/build_templates/init_cfg.json.j2

@@ -31,6 +31,7 @@
 {%- if sonic_asic_platform == "vs" %}{% do features.append(("gbsyncd", "enabled", false, "enabled")) %}{% endif %}
 {%- if include_iccpd == "y" %}{% do features.append(("iccpd", "disabled", false, "enabled")) %}{% endif %}
 {%- if include_mgmt_framework == "y" %}{% do features.append(("mgmt-framework", "enabled", true, "enabled")) %}{% endif %}
+{%- if include_mux == "y" %}{% do features.append(("mux", "enabled", false, "enabled")) %}{% endif %}
 {%- if include_nat == "y" %}{% do features.append(("nat", "disabled", false, "enabled")) %}{% endif %}
 {%- if include_restapi == "y" %}{% do features.append(("restapi", "enabled", false, "enabled")) %}{% endif %}
 {%- if include_sflow == "y" %}{% do features.append(("sflow", "disabled", false, "enabled")) %}{% endif %}

files/build_templates/mux.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=MUX Cable Container
+Requires=database.service updategraph.service pmon.service swss.service
+After=pmon.service swss.service
+StartLimitIntervalSec=1200
+StartLimitBurst=3
+
+[Service]
+User={{ sonicadmin_user }}
+ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
+ExecStart=/usr/bin/{{docker_container_name}}.sh wait
+ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

rules/config

@@ -188,3 +188,6 @@ SONIC_VERSION_CONTROL_COMPONENTS ?= none
 # ENABLE_DOCKER_BASE_PULL = y
 REGISTRY_PORT=443
 REGISTRY_SERVER=sonicdev-microsoft.azurecr.io
+
+# INCLUDE_MUX - build docker-mux for dual ToR (Gemini)
+INCLUDE_MUX = y

rules/docker-mux.dep

@@ -0,0 +1,12 @@
+
+DPATH       := $($(DOCKER_MUX)_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/docker-mux.mk rules/docker-mux.dep
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(DPATH))
+
+$(DOCKER_MUX)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(DOCKER_MUX)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(DOCKER_MUX)_DEP_FILES   := $(DEP_FILES)
+
+$(eval $(call add_dbg_docker,$(DOCKER_MUX),$(DOCKER_MUX_DBG)))
+

rules/docker-mux.mk

@@ -0,0 +1,32 @@
+# Docker image for MUX
+
+DOCKER_MUX_STEM = docker-mux
+DOCKER_MUX = $(DOCKER_MUX_STEM).gz
+DOCKER_MUX_DBG = $(DOCKER_MUX_STEM)-$(DBG_IMAGE_MARK).gz
+
+$(DOCKER_MUX)_PATH = $(DOCKERS_PATH)/$(DOCKER_MUX_STEM)
+
+$(DOCKER_MUX)_DEPENDS = $(SONIC_LINKMGRD) $(LIBSWSSCOMMON) $(LIBHIREDIS)
+$(DOCKER_MUX)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS)
+$(DOCKER_MUX)_DBG_DEPENDS += $(SONIC_LINKMGRD_DBG) $(LIBSWSSCOMMON_DBG) $(LIBHIREDIS_DBG)
+
+$(DOCKER_MUX)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_IMAGE_PACKAGES)
+
+$(DOCKER_MUX)_LOAD_DOCKERS = $(DOCKER_CONFIG_ENGINE_BUSTER)
+
+ifeq ($(INCLUDE_MUX), y)
+SONIC_DOCKER_IMAGES += $(DOCKER_MUX)
+SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_MUX)
+endif
+
+ifeq ($(INCLUDE_MUX), y)
+SONIC_DOCKER_DBG_IMAGES += $(DOCKER_MUX_DBG)
+SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_MUX_DBG)
+endif
+
+$(DOCKER_MUX)_CONTAINER_NAME = mux
+$(DOCKER_MUX)_RUN_OPT += --privileged -t
+$(DOCKER_MUX)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
+$(DOCKER_ORCHAGENT)_RUN_OPT += -v /var/log/mux:/var/log/mux:rw
+$(DOCKER_MUX)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)
+

rules/linkmgrd.dep

@@ -0,0 +1,9 @@
+
+SPATH       := $($(SONIC_LINKMGRD)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/linkmgrd.mk rules/linkmgrd.dep
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(SPATH))
+
+$(SONIC_LINKMGRD)_CACHE_MODE  := GIT_CONTENT_SHA
+$(SONIC_LINKMGRD)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_LINKMGRD)_DEP_FILES   := $(DEP_FILES)

rules/linkmgrd.mk

@@ -0,0 +1,18 @@
+# SONiC LINK ManaGeR Daemon package
+
+SONIC_LINKMGRD_VERSION = 1.0.0-1
+SONIC_LINKMGRD_PKG_NAME = linkmgrd
+
+export SONIC_LINKMGRD_VERSION SONIC_LINKMGRD_PKG_NAME
+
+SONIC_LINKMGRD = sonic-$(SONIC_LINKMGRD_PKG_NAME)_$(SONIC_LINKMGRD_VERSION)_$(CONFIGURED_ARCH).deb
+$(SONIC_LINKMGRD)_SRC_PATH = $(SRC_PATH)/$(SONIC_LINKMGRD_PKG_NAME)
+$(SONIC_LINKMGRD)_DEPENDS = $(LIBSWSSCOMMON_DEV) $(LIBSWSSCOMMON) $(LIBHIREDIS_DEV) $(LIBHIREDIS)
+
+SONIC_DPKG_DEBS += $(SONIC_LINKMGRD)
+
+SONIC_LINKMGRD_DBG = sonic-$(SONIC_LINKMGRD_PKG_NAME)-dbgsym_$(SONIC_LINKMGRD_VERSION)_$(CONFIGURED_ARCH).deb
+$(SONIC_LINKMGRD)_DBG_DEPENDS = $(LIBSWSSCOMMON_DEV) $(LIBSWSSCOMMON_DBG) $(LIBHIREDIS_DEV) $(LIBHIREDIS_DBG)
+$(eval $(call add_derived_package,$(SONIC_LINKMGRD),$(SONIC_LINKMGRD_DBG)))
+
+export SONIC_LINKMGRD SONIC_LINKMGRD_DBG

slave.mk

@@ -142,6 +142,10 @@ ifeq ($(SONIC_INCLUDE_NAT),y)
 INCLUDE_NAT = y
 endif
 
+ifeq ($(SONIC_INCLUDE_MUX),y)
+INCLUDE_MUX = y
+endif
+
 
 include $(RULES_PATH)/functions
 
@@ -257,6 +261,7 @@ $(info "INCLUDE_RESTAPI"                 : "$(INCLUDE_RESTAPI)")
 $(info "INCLUDE_SFLOW"                   : "$(INCLUDE_SFLOW)")
 $(info "INCLUDE_NAT"                     : "$(INCLUDE_NAT)")
 $(info "INCLUDE_KUBERNETES"              : "$(INCLUDE_KUBERNETES)")
+$(info "INCLUDE_MUX"                     : "$(INCLUDE_MUX)")
 $(info "TELEMETRY_WRITABLE"              : "$(TELEMETRY_WRITABLE)")
 $(info "PDDF_SUPPORT"                    : "$(PDDF_SUPPORT)")
 $(info )
@@ -929,6 +934,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export python_swss_debs+=" $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(LIBSWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON_SWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON3_SWSSCOMMON))"
 	export sonic_utilities_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_UTILITIES_PY3))"
 	export sonic_host_services_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))"
+	export include_mux="$(INCLUDE_MUX)"
 
 	$(foreach docker, $($*_DOCKERS),\
 		export docker_image="$(docker)"

src/linkmgrd/.gitignore

@@ -0,0 +1,7 @@
+**/*.o
+**/*.d
+debian/debhelper-build-stamp
+debian/sonic-linkmgrd/
+linkmgrd
+linkmgrd-test
+

src/linkmgrd/Makefile

@@ -0,0 +1,92 @@
+-include ../makefile.init
+
+RM := rm -rf
+LINKMGRD_TARGET := linkmgrd
+LINKMGRD_TEST_TARGET := linkmgrd-test
+CP := cp
+MKDIR := mkdir
+CC := g++
+MV := mv
+CPP_FLAGS := -O3 -Wall -c -fmessage-length=0 -fPIC
+TOPDIR := $(dir $(firstword $(MAKEFILE_LIST)))
+
+INCLUDES := \
+    -I"$(TOPDIR)/src" \
+    -I"/usr/include/libnl3/"
+
+# All of the sources participating in the build are defined here
+-include sources.mk
+-include test/subdir.mk
+-include src/mux_state/subdir.mk
+-include src/link_state/subdir.mk
+-include src/link_prober/subdir.mk
+-include src/link_manager/subdir.mk
+-include src/common/subdir.mk
+-include src/subdir.mk
+-include subdir.mk
+-include objects.mk
+
+ifneq ($(MAKECMDGOALS),clean)
+ifneq ($(strip $(CC_DEPS)),)
+-include $(CC_DEPS)
+endif
+ifneq ($(strip $(C++_DEPS)),)
+-include $(C++_DEPS)
+endif
+ifneq ($(strip $(C_UPPER_DEPS)),)
+-include $(C_UPPER_DEPS)
+endif
+ifneq ($(strip $(CXX_DEPS)),)
+-include $(CXX_DEPS)
+endif
+ifneq ($(strip $(CPP_DEPS)),)
+-include $(CPP_DEPS)
+endif
+ifneq ($(strip $(C_DEPS)),)
+-include $(C_DEPS)
+endif
+endif
+
+-include ../makefile.defs
+
+# Add inputs and outputs from these tool invocations to the build variables 
+
+# All Target
+all: sonic-linkmgrd
+
+# Tool invocations
+sonic-linkmgrd: $(OBJS) $(USER_OBJS) $(OBJS_LINKMGRD)
+	@echo 'Building target: $@'
+	@echo 'Invoking: GCC C++ Linker'
+	$(CC) -pthread -o "$(LINKMGRD_TARGET)" $(OBJS) $(OBJS_LINKMGRD) $(USER_OBJS) $(LIBS)
+	@echo 'Finished building target: $@'
+	@echo ' '
+
+# Other Targets
+test: $(OBJS) $(USER_OBJS) $(OBJS_LINKMGRD_TEST)
+	@echo 'Building target: $@'
+	@echo 'Invoking: GCC C++ Linker'
+	$(CC) -pthread -o "$(LINKMGRD_TEST_TARGET)" $(OBJS) $(OBJS_LINKMGRD_TEST) $(USER_OBJS) $(LIBS) $(LIBS_TEST)
+	@echo 'Executing test target: $@'
+	LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.5 ./$(LINKMGRD_TEST_TARGET)
+	@echo 'Finished building target: $@'
+	@echo ' '
+
+install:
+	$(MKDIR) -p $(DESTDIR)/usr/sbin
+	$(MV) $(LINKMGRD_TARGET) $(DESTDIR)/usr/sbin
+	$(RM) $(CC_DEPS) $(C++_DEPS) $(EXECUTABLES) $(C_UPPER_DEPS) $(CXX_DEPS) $(OBJS) $(CPP_DEPS) $(C_DEPS) \
+		$(LINKMGRD_TARGET) $(LINKMGRD_TEST_TARGET) $(OBJS_LINKMGRD) $(OBJS_LINKMGRD_TEST)
+
+deinstall:
+	$(RM) $(DESTDIR)/usr/sbin/$(LINKMGRD_TARGET)
+	$(RM) -rf $(DESTDIR)/usr/sbin
+
+clean:
+	-$(RM) $(CC_DEPS) $(C++_DEPS) $(EXECUTABLES) $(C_UPPER_DEPS) $(CXX_DEPS) $(OBJS) $(CPP_DEPS) $(C_DEPS) \
+		$(LINKMGRD_TARGET) $(LINKMGRD_TEST_TARGET) $(OBJS_LINKMGRD) $(OBJS_LINKMGRD_TEST)
+	-@echo ' '
+
+.PHONY: all clean dependents
+
+-include ../makefile.targets

src/linkmgrd/debian/changelog

@@ -0,0 +1,5 @@
+sonic-linkmgrd (1.0.0-1) UNRELEASED; urgency=medium
+
+  * Initial release.
+
+ -- Tamer Ahmed <tamer.ahmed@microsoft.com>  Mon, 26 Oct 2020 12:00:00 -0700

src/linkmgrd/debian/compat

@@ -0,0 +1 @@
+10

src/linkmgrd/debian/control

@@ -0,0 +1,19 @@
+Source: sonic-linkmgrd
+Section: devel
+Priority: optional
+Maintainer: Tamer Ahmed <tamer.ahmed@microsoft.com>
+Build-Depends: debhelper (>= 8.0.0),
+               dh-systemd
+Standards-Version: 3.9.3
+Homepage: https://github.com/Azure/sonic-buildimage
+XS-Go-Import-Path: github.com/Azure/sonic-buildimage
+
+Package: sonic-linkmgrd
+Architecture: any
+Built-Using: ${misc:Built-Using}
+Depends: libboost-program-options1.71.0,
+         libboost-system1.71.0,
+         libboost-thread1.71.0,
+         libboost-date-time1.71.0,
+         libboost-log1.71.0
+Description: SONiC LINK ManaGeR Daemon (linkmgrd)

src/linkmgrd/debian/rules

@@ -0,0 +1,6 @@
+#!/usr/bin/make -f
+
+%:
+	dh $@ --parallel
+
+

src/linkmgrd/objects.mk

@@ -0,0 +1,17 @@
+
+USER_OBJS :=
+
+LIBS := \
+    -lswsscommon \
+    -lboost_system \
+    -lboost_date_time \
+    -lboost_thread \
+    -lboost_log \
+    -lboost_log_setup \
+    -lboost_program_options \
+    -lnl-3 \
+    -lnl-route-3
+
+LIBS_TEST := \
+    -lgtest_main \
+    -lgtest