From 75104bb35d86e3ec5565abe1cb49809e7790a155 Mon Sep 17 00:00:00 2001 From: padmanarayana Date: Sat, 14 Sep 2019 20:27:09 -0700 Subject: [PATCH] [sflow]: Build infrastructure changes to support sflow docker and utilities (#3251) Introduce a new "sflow" container (if ENABLE_SFLOW is set). The new docker will include: hsflowd : host-sflow based daemon is the sFlow agent psample : Built from libpsample repository. Useful in debugging sampled packets/groups. sflowtool : Locally dump sflow samples (e.g. with a in-unit collector) In case of SONiC-VS, enable psample & act_sample kernel modules. VS' syncd needs iproute2=4.20.0-2~bpo9+1 & libcap2-bin=1:2.25-1 to support tc-sample tc-syncd is provided as a convenience tool for debugging (e.g. tc-syncd filter show ...) --- dockers/docker-sflow/Dockerfile.j2 | 33 ++++++ dockers/docker-sflow/base_image_files/psample | 10 ++ .../docker-sflow/base_image_files/sflowtool | 10 ++ dockers/docker-sflow/start.sh | 7 ++ dockers/docker-sflow/supervisord.conf | 28 +++++ files/build_templates/sflow.service.j2 | 14 +++ .../build_templates/sonic_debian_extension.j2 | 8 ++ platform/vs/docker-syncd-vs/Dockerfile.j2 | 2 + rules/config | 3 + rules/docker-sflow.mk | 35 ++++++ rules/sflow.mk | 49 ++++++++ slave.mk | 5 + src/sflow/hsflowd/Makefile | 29 +++++ .../patch/0001-host_sflow_psample.patch | 39 +++++++ .../patch/0002-host_sflow_debian.patch | 108 ++++++++++++++++++ src/sflow/hsflowd/patch/series | 2 + src/sflow/psample/Makefile | 20 ++++ src/sflow/psample/debian/changelog | 6 + src/sflow/psample/debian/compat | 1 + src/sflow/psample/debian/control | 11 ++ src/sflow/psample/debian/psample.install | 4 + src/sflow/psample/debian/rules | 19 +++ src/sflow/sflowtool/Makefile | 19 +++ src/sflow/sflowtool/debian/changelog | 6 + src/sflow/sflowtool/debian/compat | 1 + src/sflow/sflowtool/debian/control | 11 ++ src/sflow/sflowtool/debian/rules | 19 +++ src/sflow/sflowtool/debian/sflowtool.install | 1 + 28 files changed, 500 insertions(+) create mode 100644 dockers/docker-sflow/Dockerfile.j2 create mode 100755 dockers/docker-sflow/base_image_files/psample create mode 100755 dockers/docker-sflow/base_image_files/sflowtool create mode 100755 dockers/docker-sflow/start.sh create mode 100644 dockers/docker-sflow/supervisord.conf create mode 100644 files/build_templates/sflow.service.j2 create mode 100644 rules/docker-sflow.mk create mode 100644 rules/sflow.mk create mode 100644 src/sflow/hsflowd/Makefile create mode 100644 src/sflow/hsflowd/patch/0001-host_sflow_psample.patch create mode 100644 src/sflow/hsflowd/patch/0002-host_sflow_debian.patch create mode 100644 src/sflow/hsflowd/patch/series create mode 100644 src/sflow/psample/Makefile create mode 100644 src/sflow/psample/debian/changelog create mode 100644 src/sflow/psample/debian/compat create mode 100644 src/sflow/psample/debian/control create mode 100644 src/sflow/psample/debian/psample.install create mode 100755 src/sflow/psample/debian/rules create mode 100644 src/sflow/sflowtool/Makefile create mode 100644 src/sflow/sflowtool/debian/changelog create mode 100644 src/sflow/sflowtool/debian/compat create mode 100644 src/sflow/sflowtool/debian/control create mode 100755 src/sflow/sflowtool/debian/rules create mode 100644 src/sflow/sflowtool/debian/sflowtool.install diff --git a/dockers/docker-sflow/Dockerfile.j2 b/dockers/docker-sflow/Dockerfile.j2 new file mode 100644 index 000000000000..e88789fb6992 --- /dev/null +++ b/dockers/docker-sflow/Dockerfile.j2 @@ -0,0 +1,33 @@ +{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %} +FROM docker-config-engine-stretch + +ARG docker_container_name +RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf + +## Make apt-get non-interactive +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -f -y \ + dmidecode \ + libmnl0=1.0.4-2 + +{% if docker_sflow_debs.strip() -%} +# Copy locally-built Debian package dependencies +{{ copy_files("debs/", docker_sflow_debs.split(' '), "/debs/") }} + +# Install locally-built Debian packages and implicitly install their dependencies +{{ install_debian_packages(docker_sflow_debs.split(' ')) }} +{%- endif %} + +RUN apt-get clean -y && \ + apt-get autoclean -y && \ + apt-get autoremove -y && \ + rm -rf /debs + +RUN sed -ri '/^DAEMON_ARGS=""/c DAEMON_ARGS="-c /var/log/hsflowd.crash"' /etc/init.d/hsflowd + +COPY ["start.sh", "/usr/bin/"] +COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] + +ENTRYPOINT ["/usr/bin/supervisord"] diff --git a/dockers/docker-sflow/base_image_files/psample b/dockers/docker-sflow/base_image_files/psample new file mode 100755 index 000000000000..0269eb6e5d09 --- /dev/null +++ b/dockers/docker-sflow/base_image_files/psample @@ -0,0 +1,10 @@ +#!/bin/bash + +DOCKER_EXEC_FLAGS="i" + +# Determine whether stdout is on a terminal +if [ -t 1 ] ; then + DOCKER_EXEC_FLAGS+="t" +fi + +docker exec -$DOCKER_EXEC_FLAGS sflow psample "$@" diff --git a/dockers/docker-sflow/base_image_files/sflowtool b/dockers/docker-sflow/base_image_files/sflowtool new file mode 100755 index 000000000000..26e5c7d967cf --- /dev/null +++ b/dockers/docker-sflow/base_image_files/sflowtool @@ -0,0 +1,10 @@ +#!/bin/bash + +DOCKER_EXEC_FLAGS="i" + +# Determine whether stdout is on a terminal +if [ -t 1 ] ; then + DOCKER_EXEC_FLAGS+="t" +fi + +docker exec -$DOCKER_EXEC_FLAGS sflow sflowtool "$@" diff --git a/dockers/docker-sflow/start.sh b/dockers/docker-sflow/start.sh new file mode 100755 index 000000000000..aaefb4d6d004 --- /dev/null +++ b/dockers/docker-sflow/start.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +rm -f /var/run/rsyslogd.pid + +supervisorctl start rsyslogd + +supervisorctl start sflowmgrd diff --git a/dockers/docker-sflow/supervisord.conf b/dockers/docker-sflow/supervisord.conf new file mode 100644 index 000000000000..e4f9259712b5 --- /dev/null +++ b/dockers/docker-sflow/supervisord.conf @@ -0,0 +1,28 @@ +[supervisord] +logfile_maxbytes=1MB +logfile_backups=2 +nodaemon=true + +[program:start.sh] +command=/usr/bin/start.sh +priority=1 +autostart=true +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:rsyslogd] +command=/usr/sbin/rsyslogd -n +priority=2 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:sflowmgrd] +command=/usr/bin/sflowmgrd +priority=3 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog diff --git a/files/build_templates/sflow.service.j2 b/files/build_templates/sflow.service.j2 new file mode 100644 index 000000000000..d77979bb338d --- /dev/null +++ b/files/build_templates/sflow.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=sFlow container +Requires=swss.service +After=swss.service +Before=ntp-config.service + +[Service] +User={{ sonicadmin_user }} +ExecStartPre=/usr/bin/{{docker_container_name}}.sh start +ExecStart=/usr/bin/{{docker_container_name}}.sh wait +ExecStop=/usr/bin/{{docker_container_name}}.sh stop + +[Install] +WantedBy=multi-user.target diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 0f27c15273c1..bcdecaa71766 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -300,6 +300,14 @@ sudo cp $IMAGE_CONFIGS/platform/rc.local $FILESYSTEM_ROOT/etc/ ## copy blacklist file sudo cp $IMAGE_CONFIGS/platform/linux_kernel_bde.conf $FILESYSTEM_ROOT/etc/modprobe.d/ +# Enable psample drivers to support sFlow on vs +{% if sonic_asic_platform == "vs" %} +sudo tee -a $FILESYSTEM_ROOT/etc/modules-load.d/modules.conf > /dev/null < Mon, 29 Jul 2019 07:08:02 -0400 +diff -ruN a/DEBIAN_build/compat b/DEBIAN_build/compat +--- a/DEBIAN_build/compat 1969-12-31 19:00:00.000000000 -0500 ++++ b/DEBIAN_build/compat 2019-08-16 23:28:58.020938096 -0400 +@@ -0,0 +1 @@ ++9 +diff -ruN a/DEBIAN_build/control b/DEBIAN_build/control +--- a/DEBIAN_build/control 2019-08-16 05:11:33.974949327 -0400 ++++ b/DEBIAN_build/control 2019-08-19 21:28:07.155722725 -0400 +@@ -1,9 +1,22 @@ +-Package: _PACKAGE_ +-Version: _VERSION_ +-Section: admin ++Source: hsflowd ++Maintainer: Neil McKee [neil.mckee@inmon.com] ++Uploaders: DellEMC ++Section: net ++Priority: optional ++Build-Depends: dh-exec (>=0.3), debhelper (>= 9), autotools-dev ++Standards-Version: 1.0.0 ++ ++Package: hsflowd ++Section: admin + Priority: optional +-Architecture: all ++Architecture: any + Essential: no +-Maintainer: Neil McKee [neil.mckee@inmon.com] + Description: sFlow(R) monitoring agent + Homepage: sflow.net ++ ++Package: hsflowd-dbg ++Architecture: any ++Section: debug ++Priority: extra ++Depends: hsflowd ++Description: debugging symbols for hsflowd +diff -ruN a/DEBIAN_build/rules b/DEBIAN_build/rules +--- a/DEBIAN_build/rules 1969-12-31 19:00:00.000000000 -0500 ++++ b/DEBIAN_build/rules 2019-08-19 22:20:42.998569601 -0400 +@@ -0,0 +1,31 @@ ++#!/usr/bin/make -f ++ ++# See debhelper(7) (uncomment to enable) ++# output every command that modifies files on the build system. ++export DH_VERBOSE = 1 ++ ++# see EXAMPLES in dpkg-buildflags(1) and read /usr/share/dpkg/* ++DPKG_EXPORT_BUILDFLAGS = 1 ++include /usr/share/dpkg/default.mk ++ ++%: ++ dh $@ ++ ++binary: ++ dh_gencontrol ++ dh_strip -phsflowd --dbg-package=hsflowd-dbg ++ dpkg-deb --build debian/hsflowd-dbg hsflowd-dbg_$(HSFLOWD_VERSION)-$(HSFLOWD_SUBVERSION)_$(CONFIGURED_ARCH).deb ++ dpkg-deb --build debian/hsflowd hsflowd_$(HSFLOWD_VERSION)-$(HSFLOWD_SUBVERSION)_$(CONFIGURED_ARCH).deb ++ ++override_dh_auto_build: ++ make sonic-deb FEATURES="SONIC" ++ ++override_dh_auto_configure: ++ ++override_dh_auto_install: ++ ++override_dh_auto_test: ++ ++override_dh_auto_clean: ++ ++override_dh_clean: +diff -ruN a/Makefile b/Makefile +--- a/Makefile 2019-08-16 21:34:25.167679297 -0400 ++++ b/Makefile 2019-08-19 22:20:23.758479002 -0400 +@@ -146,6 +146,23 @@ + cd ..; \ + dpkg-deb --build debian hsflowd_$${MYVER}-$${MYREL}_$$MYARCH.deb + ++sonic-deb: $(PROG) ++ MYARCH=`uname -m|sed 's/x86_64/amd64/'`; \ ++ MYVER=`./getVersion`; \ ++ MYREL=`./getRelease`; \ ++ PLATFORM=`uname`; \ ++ mkdir -p debian/usr/sbin; \ ++ mkdir -p debian/etc/init.d; \ ++ mkdir -p debian/etc/hsflowd/modules; \ ++ mkdir -p debian/lib/systemd/system; \ ++ mkdir -p debian/etc/dbus-1/system.d; \ ++ cd src/$$PLATFORM; $(MAKE) VERSION=$$MYVER RELEASE=$$MYREL INSTROOT="../../debian/hsflowd" install; cd ../..; \ ++ pwd; \ ++ cd debian/hsflowd; \ ++ find . -type d | xargs chmod 755; \ ++ md5sum `find usr etc -type f` > md5sums; \ ++ cd ../..; ++ + xenserver: xenrpm + cd xenserver-ddk; $(MAKE) clean; $(MAKE) + diff --git a/src/sflow/hsflowd/patch/series b/src/sflow/hsflowd/patch/series new file mode 100644 index 000000000000..90f9e47462e5 --- /dev/null +++ b/src/sflow/hsflowd/patch/series @@ -0,0 +1,2 @@ +0001-host_sflow_psample.patch +0002-host_sflow_debian.patch diff --git a/src/sflow/psample/Makefile b/src/sflow/psample/Makefile new file mode 100644 index 000000000000..3f03c249989a --- /dev/null +++ b/src/sflow/psample/Makefile @@ -0,0 +1,20 @@ +.ONESHELL: +SHELL = /bin/bash +.SHELLFLAGS += -e + +MAIN_TARGET = $(PSAMPLE) + +$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : + + rm -fr ./libpsample + git clone https://github.com/Mellanox/libpsample.git + cp -r debian libpsample + + pushd ./libpsample + git checkout -b libpsample -f e48fad2 + + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + popd + + mv $* $(DEST)/ + diff --git a/src/sflow/psample/debian/changelog b/src/sflow/psample/debian/changelog new file mode 100644 index 000000000000..be9af0d6d42a --- /dev/null +++ b/src/sflow/psample/debian/changelog @@ -0,0 +1,6 @@ +psample (1.1-1) UNRELEASED; urgency=medium + + [ DellEMC ] + * Initial release : based on https://github.com/Mellanox/libpsample + + -- DellEMC Mon, 29 Jul 2019 07:08:02 -0400 diff --git a/src/sflow/psample/debian/compat b/src/sflow/psample/debian/compat new file mode 100644 index 000000000000..ec635144f600 --- /dev/null +++ b/src/sflow/psample/debian/compat @@ -0,0 +1 @@ +9 diff --git a/src/sflow/psample/debian/control b/src/sflow/psample/debian/control new file mode 100644 index 000000000000..5c65e1e774a6 --- /dev/null +++ b/src/sflow/psample/debian/control @@ -0,0 +1,11 @@ +Source: psample +Maintainer: DellEMC +Section: devel +Priority: optional +Build-Depends: dh-exec (>=0.3), debhelper (>= 9), autotools-dev +Standards-Version: 1.0.0 + +Package: psample +Architecture: any +Depends: ${shlibs:Depends} +Description: This package contains psample from https://github.com/Mellanox/libpsample diff --git a/src/sflow/psample/debian/psample.install b/src/sflow/psample/debian/psample.install new file mode 100644 index 000000000000..36d4eac1a85b --- /dev/null +++ b/src/sflow/psample/debian/psample.install @@ -0,0 +1,4 @@ +bin/psample usr/bin +lib/libpsample.so.1.0 lib/x86_64-linux-gnu +lib/libpsample.so.1 lib/x86_64-linux-gnu +lib/libpsample.so lib/x86_64-linux-gnu diff --git a/src/sflow/psample/debian/rules b/src/sflow/psample/debian/rules new file mode 100755 index 000000000000..73f484d32581 --- /dev/null +++ b/src/sflow/psample/debian/rules @@ -0,0 +1,19 @@ +#!/usr/bin/make -f + +# main packaging script based on dh7 syntax +%: + dh $@ + +override_dh_auto_build: + cmake . && make + +override_dh_auto_install: + +override_dh_auto_test: + +override_dh_auto_clean: + +override_dh_strip: + +override_dh_clean: + diff --git a/src/sflow/sflowtool/Makefile b/src/sflow/sflowtool/Makefile new file mode 100644 index 000000000000..10a3f2d24dea --- /dev/null +++ b/src/sflow/sflowtool/Makefile @@ -0,0 +1,19 @@ +.ONESHELL: +SHELL = /bin/bash +.SHELLFLAGS += -e + +MAIN_TARGET = $(SFLOWTOOL) + +$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : + + rm -fr ./sflowtool + git clone https://github.com/sflow/sflowtool + cp -r debian sflowtool + + pushd ./sflowtool + git checkout -b sflowtool -f 6c2963b + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + popd + + mv $* $(DEST)/ + diff --git a/src/sflow/sflowtool/debian/changelog b/src/sflow/sflowtool/debian/changelog new file mode 100644 index 000000000000..df40c7c37de1 --- /dev/null +++ b/src/sflow/sflowtool/debian/changelog @@ -0,0 +1,6 @@ +sflowtool (5.04) UNRELEASED; urgency=medium + + [ DellEMC ] + * Initial release : based on https://github.com/sflow/sflowtool + + -- DellEMC Mon, 29 Jul 2019 07:08:02 -0400 diff --git a/src/sflow/sflowtool/debian/compat b/src/sflow/sflowtool/debian/compat new file mode 100644 index 000000000000..ec635144f600 --- /dev/null +++ b/src/sflow/sflowtool/debian/compat @@ -0,0 +1 @@ +9 diff --git a/src/sflow/sflowtool/debian/control b/src/sflow/sflowtool/debian/control new file mode 100644 index 000000000000..f2d971aee2bd --- /dev/null +++ b/src/sflow/sflowtool/debian/control @@ -0,0 +1,11 @@ +Source: sflowtool +Maintainer: DellEMC +Section: devel +Priority: optional +Build-Depends: dh-exec (>=0.3), debhelper (>= 9), autotools-dev +Standards-Version: 1.0.0 + +Package: sflowtool +Architecture: any +Depends: ${shlibs:Depends} +Description: This package contains sflowtool from https://github.com/sflow/sflowtool diff --git a/src/sflow/sflowtool/debian/rules b/src/sflow/sflowtool/debian/rules new file mode 100755 index 000000000000..37ee344a2a4a --- /dev/null +++ b/src/sflow/sflowtool/debian/rules @@ -0,0 +1,19 @@ +#!/usr/bin/make -f + +# main packaging script based on dh7 syntax +%: + dh $@ + +override_dh_auto_build: + ./boot.sh && ./configure && make + +override_dh_auto_install: + +override_dh_auto_test: + +override_dh_auto_clean: + +override_dh_strip: + +override_dh_clean: + diff --git a/src/sflow/sflowtool/debian/sflowtool.install b/src/sflow/sflowtool/debian/sflowtool.install new file mode 100644 index 000000000000..9b32e0709b5a --- /dev/null +++ b/src/sflow/sflowtool/debian/sflowtool.install @@ -0,0 +1 @@ +src/sflowtool usr/bin