Add sonic-host-services and sonic-host-services-data packages (sonic-…

…net#5694)

**- Why I did it**

Install all host services and their data files in package format rather than file-by-file

**- How I did it**

- Create sonic-host-services Python wheel package, currently including procdockerstatsd
  - Also add the framework for unit tests by adding one simple procdockerstatsd test case
- Create sonic-host-services-data Debian package which is responsible for installing the related systemd unit files to control the services in the Python wheel. This package will also be responsible for installing any Jinja2 templates and other data files needed by the host services.

files/build_templates/sonic_debian_extension.j2

@@ -206,6 +206,16 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-utilities-data_*.deb || \
 # in bash.bashrc, so we copy a version of the file with it enabled here.
 sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/
 
+# Install SONiC host services package
+SONIC_HOST_SERVICES_PY3_WHEEL_NAME=$(basename {{sonic_host_services_py3_wheel_path}})
+sudo cp {{sonic_host_services_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_HOST_SERVICES_PY3_WHEEL_NAME
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_HOST_SERVICES_PY3_WHEEL_NAME
+sudo rm -rf $FILESYSTEM_ROOT/$SONIC_HOST_SERVICES_PY3_WHEEL_NAME
+
+# Install SONiC host services data files (and any dependencies via 'apt-get -y install -f')
+sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-host-services-data_*.deb || \
+    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
+
 {% if enable_ztp == "y" %}
 # Install ZTP (and its dependencies via 'apt-get -y install -f')
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-ztp_*.deb || \
@@ -402,11 +412,6 @@ sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd.service $FILESYSTEM_ROOT_USR_LIB_SYSTEM
 echo "caclmgrd.service" | sudo tee -a $GENERATED_SERVICE_FILE
 sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd $FILESYSTEM_ROOT/usr/bin/
 
-# Copy process/docker cpu/memory utilization data export daemon
-sudo cp $IMAGE_CONFIGS/procdockerstatsd/procdockerstatsd.service $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM
-echo "procdockerstatsd.service" | sudo tee -a $GENERATED_SERVICE_FILE
-sudo cp $IMAGE_CONFIGS/procdockerstatsd/procdockerstatsd $FILESYSTEM_ROOT/usr/bin/
-
 # Copy systemd timer configuration
 sudo cp $BUILD_TEMPLATES/pcie-check.timer $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM
 sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable pcie-check.timer

platform/vs/docker-sonic-vs.mk

@@ -12,7 +12,8 @@ $(DOCKER_SONIC_VS)_DEPENDS += $(SWSS) \
                               $(LIBYANG) \
                               $(LIBYANG_CPP) \
                               $(LIBYANG_PY2) \
-                              $(SONIC_UTILITIES_DATA)
+                              $(SONIC_UTILITIES_DATA) \
+                              $(SONIC_HOST_SERVICES_DATA)
 
 # swsssdk is a dependency of sonic-py-common
 # TODO: sonic-py-common should depend on swsscommon instead
@@ -22,7 +23,8 @@ $(DOCKER_SONIC_VS)_PYTHON_WHEELS += $(SWSSSDK_PY2) \
                                     $(SONIC_PY_COMMON_PY3) \
                                     $(SONIC_YANG_MODELS_PY3) \
                                     $(SONIC_YANG_MGMT_PY) \
-                                    $(SONIC_UTILITIES_PY2)
+                                    $(SONIC_UTILITIES_PY2) \
+                                    $(SONIC_HOST_SERVICES_PY3)
 
 ifeq ($(INSTALL_DEBUG_TOOLS), y)
 $(DOCKER_SONIC_VS)_DEPENDS += $(SWSS_DBG) \

rules/sonic-host-services-data.dep

@@ -0,0 +1,8 @@
+SPATH       := $($(SONIC_HOST_SERVICES_DATA)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-host-services-data.mk rules/sonic-host-services-data.dep
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(SPATH))
+
+$(SONIC_HOST_SERVICES_DATA)_CACHE_MODE  := GIT_CONTENT_SHA
+$(SONIC_HOST_SERVICES_DATA)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_HOST_SERVICES_DATA)_DEP_FILES   := $(DEP_FILES)

rules/sonic-host-services-data.mk

@@ -0,0 +1,5 @@
+# SONiC host services data package
+
+SONIC_HOST_SERVICES_DATA = sonic-host-services-data_1.0-1_all.deb
+$(SONIC_HOST_SERVICES_DATA)_SRC_PATH = $(SRC_PATH)/sonic-host-services-data
+SONIC_DPKG_DEBS += $(SONIC_HOST_SERVICES_DATA)

rules/sonic-host-services.dep

@@ -0,0 +1,10 @@
+SPATH       := $($(SONIC_HOST_SERVICES_PY3)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-host-services.mk rules/sonic-host-services.dep
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
+
+$(SONIC_HOST_SERVICES_PY3)_CACHE_MODE  := GIT_CONTENT_SHA
+$(SONIC_HOST_SERVICES_PY3)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_HOST_SERVICES_PY3)_DEP_FILES   := $(DEP_FILES)
+$(SONIC_HOST_SERVICES_PY3)_SMDEP_FILES := $(SMDEP_FILES)
+$(SONIC_HOST_SERVICES_PY3)_SMDEP_PATHS := $(SPATH)

rules/sonic-host-services.mk

@@ -0,0 +1,8 @@
+# SONiC host services package
+
+SONIC_HOST_SERVICES_PY3 = sonic_host_services-1.0-py3-none-any.whl
+$(SONIC_HOST_SERVICES_PY3)_SRC_PATH = $(SRC_PATH)/sonic-host-services
+$(SONIC_HOST_SERVICES_PY3)_PYTHON_VERSION = 3
+$(SONIC_HOST_SERVICES_PY3)_DEPENDS += $(SONIC_PY_COMMON_PY3) \
+                                      $(SWSSSDK_PY3)
+SONIC_PYTHON_WHEELS += $(SONIC_HOST_SERVICES_PY3)

slave.mk

@@ -804,7 +804,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
                 $(MONIT) \
                 $(PYTHON_SWSSCOMMON) \
                 $(PYTHON3_SWSSCOMMON) \
-                $(SONIC_UTILITIES_DATA)) \
+                $(SONIC_UTILITIES_DATA) \
+                $(SONIC_HOST_SERVICES_DATA)) \
         $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \
         $$(addprefix $(FILES_PATH)/,$$($$*_FILES)) \
         $(if $(findstring y,$(ENABLE_ZTP)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_ZTP))) \
@@ -820,7 +821,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_API_PY2)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY)) \
-        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH))
+        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH)) \
+        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))
 	$(HEADER)
 	# Pass initramfs and linux kernel explicitly. They are used for all platforms
 	export debs_path="$(IMAGE_DISTRO_DEBS_PATH)"
@@ -866,6 +868,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export python_swss_debs="$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$($(LIBSWSSCOMMON)_RDEPENDS))"
 	export python_swss_debs+=" $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(LIBSWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON_SWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON3_SWSSCOMMON))"
 	export sonic_utilities_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_UTILITIES_PY2))"
+	export sonic_host_services_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))"
 
 	$(foreach docker, $($*_DOCKERS),\
 		export docker_image="$(docker)"

src/sonic-host-services-data/.gitignore

@@ -0,0 +1,6 @@
+debian/*.debhelper
+debian/debhelper-build-stamp
+debian/sonic-host-services-data/
+sonic-host-services-data_*.buildinfo
+sonic-host-services-data_*.changes
+sonic-host-services-data_*.deb

src/sonic-host-services-data/LICENSE

@@ -0,0 +1,15 @@
+Copyright (C) 2020 Microsoft
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

src/sonic-host-services-data/MAINTAINERS

@@ -0,0 +1,7 @@
+# This file describes the maintainers for sonic-device-data
+# See the SONiC project governance document for more information
+
+Name = "Joe LeVeque"
+Email = "jolevequ@microsoft.com"
+Github = jleveque
+Mailinglist = sonicproject@googlegroups.com

src/sonic-host-services-data/README.md

@@ -0,0 +1,19 @@
+# sonic-host-services-data
+Data files required for SONiC host services
+
+
+## To build
+
+```
+dpkg-buildpackage -rfakeroot -b -us -uc
+```
+
+## To clean
+
+```
+dpkg-buildpackage -rfakeroot -Tclean
+```
+
+---
+
+See the [SONiC Website](http://azure.github.io/SONiC/) for more information about the SONiC project.

src/sonic-host-services-data/debian/changelog

@@ -0,0 +1,5 @@
+sonic-host-services-data (1.0-1) UNRELEASED; urgency=low
+
+  * Initial release
+
+ -- Joe LeVeque <jolevequ@microsoft.com>  Tue, 20 Oct 2020 02:35:43 +0000

src/sonic-host-services-data/debian/compat

@@ -0,0 +1 @@
+11

src/sonic-host-services-data/debian/control

@@ -0,0 +1,11 @@
+Source: sonic-host-services-data
+Maintainer: Joe LeVeque <jolevequ@microsoft.com>
+Section: misc
+Priority: optional
+Standards-Version: 0.1
+Build-Depends: debhelper (>=11)
+
+Package: sonic-host-services-data
+Architecture: all
+Depends: ${misc:Depends}
+Description: Data files required for SONiC host services

src/sonic-host-services-data/debian/rules

@@ -0,0 +1,9 @@
+#!/usr/bin/make -f
+
+build:
+
+%:
+	dh $@
+
+override_dh_installsystemd:
+	dh_installsystemd --no-start --name=procdockerstatsd

files/image_config/procdockerstatsd/procdockerstatsd.service → src/sonic-host-services-data/debian/sonic-host-services-data.procdockerstatsd.service

@@ -5,7 +5,7 @@ After=database.service updategraph.service
 
 [Service]
 Type=simple
-ExecStart=/usr/bin/procdockerstatsd
+ExecStart=/usr/local/bin/procdockerstatsd
 Restart=always
 
 [Install]

src/sonic-host-services/.gitignore

@@ -0,0 +1,14 @@
+# Compiled Python files
+*.pyc
+scripts/procdockerstatsdc
+
+# Generated by packaging
+*.egg-info/
+.eggs/
+build/
+dist/
+
+# Unit test coverage
+.coverage
+coverage.xml
+htmlcov/

src/sonic-host-services/setup.cfg

@@ -0,0 +1,2 @@
+[aliases]
+test=pytest

src/sonic-host-services/setup.py

@@ -0,0 +1,41 @@
+from setuptools import setup
+
+setup(
+    name = 'sonic-host-services',
+    version = '1.0',
+    description = 'Python services which run in the SONiC host OS',
+    license = 'Apache 2.0',
+    author = 'SONiC Team',
+    author_email = 'linuxnetdev@microsoft.com',
+    url = 'https://github.com/Azure/sonic-buildimage',
+    maintainer = 'Joe LeVeque',
+    maintainer_email = 'jolevequ@microsoft.com',
+    scripts = [
+        'scripts/procdockerstatsd',
+    ],
+    install_requires = [
+        'sonic-py-common',
+        'swsssdk>=2.0.1',
+    ],
+    setup_requires = [
+        'pytest-runner',
+        'wheel'
+    ],
+    tests_require = [
+        'pytest',
+    ],
+    classifiers = [
+        'Development Status :: 3 - Alpha',
+        'Environment :: Console',
+        'Intended Audience :: Developers',
+        'Intended Audience :: Information Technology',
+        'Intended Audience :: System Administrators',
+        'License :: OSI Approved :: Apache Software License',
+        'Natural Language :: English',
+        'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python :: 3.7',
+        'Topic :: System',
+    ],
+    keywords = 'sonic SONiC host services',
+    test_suite = 'setup.get_test_suite'
+)

src/sonic-host-services/tests/mock_connector.py

@@ -0,0 +1,24 @@
+class MockConnector(object):
+    STATE_DB = None
+    data = {}
+
+    def __init__(self, host):
+        pass
+
+    def connect(self, db_id):
+        pass
+
+    def get(self, db_id, key, field):
+        return MockConnector.data[key][field]
+
+    def keys(self, db_id, pattern):
+        match = pattern.split('*')[0]
+        ret = []
+        for key in MockConnector.data.keys():
+            if match in key:
+                ret.append(key)
+
+        return ret
+
+    def get_all(self, db_id, key):
+        return MockConnector.data[key]

src/sonic-host-services/tests/procdockerstatsd_test.py

@@ -0,0 +1,42 @@
+import imp
+import sys
+import os
+import pytest
+
+import swsssdk
+
+from .mock_connector import MockConnector
+
+swsssdk.SonicV2Connector = MockConnector
+
+test_path = os.path.dirname(os.path.abspath(__file__))
+modules_path = os.path.dirname(test_path)
+scripts_path = os.path.join(modules_path, "scripts")
+sys.path.insert(0, modules_path)
+
+imp.load_source('procdockerstatsd', scripts_path + '/procdockerstatsd')
+from procdockerstatsd import *
+
+class TestProcDockerStatsDaemon(object):
+    def test_convert_to_bytes(self):
+        test_data = [
+            ('1B', 1),
+            ('500B', 500),
+            ('1KB', 1000),
+            ('500KB', 500000),
+            ('1MB', 1000000),
+            ('500MB', 500000000),
+            ('1MiB', 1048576),
+            ('500MiB', 524288000),
+            ('66.41MiB', 69635932),
+            ('333.6MiB', 349804954),
+            ('1GiB', 1073741824),
+            ('500GiB', 536870912000),
+            ('7.751GiB', 8322572878)
+        ]
+
+        pdstatsd = ProcDockerStats(SYSLOG_IDENTIFIER)
+
+        for test_input, expected_output in test_data:
+            res = pdstatsd.convert_to_bytes(test_input)
+            assert res == expected_output