Add sonic-host-services and sonic-host-services-data packages

files/build_templates/sonic_debian_extension.j2

@@ -206,6 +206,16 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-utilities-data_*.deb || \
 # in bash.bashrc, so we copy a version of the file with it enabled here.
 sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/
 
+# Install SONiC host services package
+SONIC_HOST_SERVICES_PY3_WHEEL_NAME=$(basename {{sonic_host_services_py3_wheel_path}})
+sudo cp {{sonic_host_services_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_HOST_SERVICES_PY3_WHEEL_NAME
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_HOST_SERVICES_PY3_WHEEL_NAME
+sudo rm -rf $FILESYSTEM_ROOT/$SONIC_HOST_SERVICES_PY3_WHEEL_NAME
+
+# Install SONiC host services data files (and any dependencies via 'apt-get -y install -f')
+sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-host-services-data_*.deb || \
+    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
+
 {% if enable_ztp == "y" %}
 # Install ZTP (and its dependencies via 'apt-get -y install -f')
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-ztp_*.deb || \
@@ -402,11 +412,6 @@ sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd.service $FILESYSTEM_ROOT_USR_LIB_SYSTEM
 echo "caclmgrd.service" | sudo tee -a $GENERATED_SERVICE_FILE
 sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd $FILESYSTEM_ROOT/usr/bin/
 
-# Copy process/docker cpu/memory utilization data export daemon
-sudo cp $IMAGE_CONFIGS/procdockerstatsd/procdockerstatsd.service $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM
-echo "procdockerstatsd.service" | sudo tee -a $GENERATED_SERVICE_FILE
-sudo cp $IMAGE_CONFIGS/procdockerstatsd/procdockerstatsd $FILESYSTEM_ROOT/usr/bin/
-
 # Copy systemd timer configuration
 sudo cp $BUILD_TEMPLATES/pcie-check.timer $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM
 sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable pcie-check.timer

platform/vs/docker-sonic-vs.mk

@@ -12,7 +12,8 @@ $(DOCKER_SONIC_VS)_DEPENDS += $(SWSS) \
                               $(LIBYANG) \
                               $(LIBYANG_CPP) \
                               $(LIBYANG_PY2) \
-                              $(SONIC_UTILITIES_DATA)
+                              $(SONIC_UTILITIES_DATA) \
+                              $(SONIC_HOST_SERVICES_DATA)
 
 # swsssdk is a dependency of sonic-py-common
 # TODO: sonic-py-common should depend on swsscommon instead
@@ -22,7 +23,8 @@ $(DOCKER_SONIC_VS)_PYTHON_WHEELS += $(SWSSSDK_PY2) \
                                     $(SONIC_PY_COMMON_PY3) \
                                     $(SONIC_YANG_MODELS_PY3) \
                                     $(SONIC_YANG_MGMT_PY) \
-                                    $(SONIC_UTILITIES_PY2)
+                                    $(SONIC_UTILITIES_PY2) \
+                                    $(SONIC_HOST_SERVICES_PY3)
 
 ifeq ($(INSTALL_DEBUG_TOOLS), y)
 $(DOCKER_SONIC_VS)_DEPENDS += $(SWSS_DBG) \

rules/sonic-host-services-data.dep

@@ -0,0 +1,8 @@
+SPATH       := $($(SONIC_HOST_SERVICES_DATA)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-host-services-data.mk rules/sonic-host-services-data.dep
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(SPATH))
+
+$(SONIC_HOST_SERVICES_DATA)_CACHE_MODE  := GIT_CONTENT_SHA
+$(SONIC_HOST_SERVICES_DATA)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_HOST_SERVICES_DATA)_DEP_FILES   := $(DEP_FILES)

rules/sonic-host-services-data.mk

@@ -0,0 +1,5 @@
+# SONiC host services data package
+
+SONIC_HOST_SERVICES_DATA = sonic-host-services-data_1.0-1_all.deb
+$(SONIC_HOST_SERVICES_DATA)_SRC_PATH = $(SRC_PATH)/sonic-host-services-data
+SONIC_DPKG_DEBS += $(SONIC_HOST_SERVICES_DATA)

rules/sonic-host-services.dep

@@ -0,0 +1,10 @@
+SPATH       := $($(SONIC_HOST_SERVICES_PY3)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-host-services.mk rules/sonic-host-services.dep
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
+
+$(SONIC_HOST_SERVICES_PY3)_CACHE_MODE  := GIT_CONTENT_SHA
+$(SONIC_HOST_SERVICES_PY3)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_HOST_SERVICES_PY3)_DEP_FILES   := $(DEP_FILES)
+$(SONIC_HOST_SERVICES_PY3)_SMDEP_FILES := $(SMDEP_FILES)
+$(SONIC_HOST_SERVICES_PY3)_SMDEP_PATHS := $(SPATH)

rules/sonic-host-services.mk

@@ -0,0 +1,8 @@
+# SONiC host services package
+
+SONIC_HOST_SERVICES_PY3 = sonic_host_services-1.0-py3-none-any.whl
+$(SONIC_HOST_SERVICES_PY3)_SRC_PATH = $(SRC_PATH)/sonic-host-services
+$(SONIC_HOST_SERVICES_PY3)_PYTHON_VERSION = 3
+$(SONIC_HOST_SERVICES_PY3)_DEPENDS += $(SONIC_PY_COMMON_PY3) \
+                                      $(SWSSSDK_PY3)
+SONIC_PYTHON_WHEELS += $(SONIC_HOST_SERVICES_PY3)

slave.mk

@@ -804,7 +804,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
                 $(MONIT) \
                 $(PYTHON_SWSSCOMMON) \
                 $(PYTHON3_SWSSCOMMON) \
-                $(SONIC_UTILITIES_DATA)) \
+                $(SONIC_UTILITIES_DATA) \
+                $(SONIC_HOST_SERVICES_DATA)) \
         $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \
         $$(addprefix $(FILES_PATH)/,$$($$*_FILES)) \
         $(if $(findstring y,$(ENABLE_ZTP)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_ZTP))) \
@@ -820,7 +821,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_API_PY2)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY)) \
-        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH))
+        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH)) \
+        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))
 	$(HEADER)
 	# Pass initramfs and linux kernel explicitly. They are used for all platforms
 	export debs_path="$(IMAGE_DISTRO_DEBS_PATH)"
@@ -866,6 +868,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export python_swss_debs="$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$($(LIBSWSSCOMMON)_RDEPENDS))"
 	export python_swss_debs+=" $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(LIBSWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON_SWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON3_SWSSCOMMON))"
 	export sonic_utilities_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_UTILITIES_PY2))"
+	export sonic_host_services_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))"
 
 	$(foreach docker, $($*_DOCKERS),\
 		export docker_image="$(docker)"

src/sonic-host-services-data/.gitignore

@@ -0,0 +1,6 @@
+debian/*.debhelper
+debian/debhelper-build-stamp
+debian/sonic-host-services-data/
+sonic-host-services-data_*.buildinfo
+sonic-host-services-data_*.changes
+sonic-host-services-data_*.deb

src/sonic-host-services-data/LICENSE

@@ -0,0 +1,15 @@
+Copyright (C) 2020 Microsoft
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

src/sonic-host-services-data/MAINTAINERS

@@ -0,0 +1,7 @@
+# This file describes the maintainers for sonic-device-data
+# See the SONiC project governance document for more information
+
+Name = "Joe LeVeque"
+Email = "jolevequ@microsoft.com"
+Github = jleveque
+Mailinglist = sonicproject@googlegroups.com

src/sonic-host-services-data/README.md

@@ -0,0 +1,19 @@
+# sonic-host-services-data
+Data files required for SONiC host services
+
+
+## To build
+
+```
+dpkg-buildpackage -rfakeroot -b -us -uc
+```
+
+## To clean
+
+```
+dpkg-buildpackage -rfakeroot -Tclean
+```
+
+---
+
+See the [SONiC Website](http://azure.github.io/SONiC/) for more information about the SONiC project.

src/sonic-host-services-data/debian/changelog

@@ -0,0 +1,5 @@
+sonic-host-services-data (1.0-1) UNRELEASED; urgency=low
+
+  * Initial release
+
+ -- Joe LeVeque <jolevequ@microsoft.com>  Tue, 20 Oct 2020 02:35:43 +0000

src/sonic-host-services-data/debian/compat

@@ -0,0 +1 @@
+11

src/sonic-host-services-data/debian/control

@@ -0,0 +1,11 @@
+Source: sonic-host-services-data
+Maintainer: Joe LeVeque <jolevequ@microsoft.com>
+Section: misc
+Priority: optional
+Standards-Version: 0.1
+Build-Depends: debhelper (>=11)
+
+Package: sonic-host-services-data
+Architecture: all
+Depends: ${misc:Depends}
+Description: Data files required for SONiC host services

src/sonic-host-services-data/debian/rules

@@ -0,0 +1,9 @@
+#!/usr/bin/make -f
+
+build:
+
+%:
+	dh $@
+
+override_dh_installsystemd:
+	dh_installsystemd --no-start --name=procdockerstatsd

files/image_config/procdockerstatsd/procdockerstatsd.service → src/sonic-host-services-data/debian/sonic-host-services-data.procdockerstatsd.service

@@ -5,7 +5,7 @@ After=database.service updategraph.service
 
 [Service]
 Type=simple
-ExecStart=/usr/bin/procdockerstatsd
+ExecStart=/usr/local/bin/procdockerstatsd
 Restart=always
 
 [Install]

src/sonic-host-services/.gitignore

@@ -0,0 +1,14 @@
+# Compiled Python files
+*.pyc
+scripts/procdockerstatsdc
+
+# Generated by packaging
+*.egg-info/
+.eggs/
+build/
+dist/
+
+# Unit test coverage
+.coverage
+coverage.xml
+htmlcov/

src/sonic-host-services/setup.cfg

@@ -0,0 +1,2 @@
+[aliases]
+test=pytest

src/sonic-host-services/setup.py

@@ -0,0 +1,41 @@
+from setuptools import setup
+
+setup(
+    name = 'sonic-host-services',
+    version = '1.0',
+    description = 'Python services which run in the SONiC host OS',
+    license = 'Apache 2.0',
+    author = 'SONiC Team',
+    author_email = 'linuxnetdev@microsoft.com',
+    url = 'https://github.com/Azure/sonic-buildimage',
+    maintainer = 'Joe LeVeque',
+    maintainer_email = 'jolevequ@microsoft.com',
+    scripts = [
+        'scripts/procdockerstatsd',
+    ],
+    install_requires = [
+        'sonic-py-common',
+        'swsssdk>=2.0.1',
+    ],
+    setup_requires = [
+        'pytest-runner',
+        'wheel'
+    ],
+    tests_require = [
+        'pytest',
+    ],
+    classifiers = [
+        'Development Status :: 3 - Alpha',
+        'Environment :: Console',
+        'Intended Audience :: Developers',
+        'Intended Audience :: Information Technology',
+        'Intended Audience :: System Administrators',
+        'License :: OSI Approved :: Apache Software License',
+        'Natural Language :: English',
+        'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python :: 3.7',
+        'Topic :: System',
+    ],
+    keywords = 'sonic SONiC host services',
+    test_suite = 'setup.get_test_suite'
+)

src/sonic-host-services/tests/mock_connector.py

@@ -0,0 +1,24 @@
+class MockConnector(object):
+    STATE_DB = None
+    data = {}
+
+    def __init__(self, host):
+        pass
+
+    def connect(self, db_id):
+        pass
+
+    def get(self, db_id, key, field):
+        return MockConnector.data[key][field]
+
+    def keys(self, db_id, pattern):
+        match = pattern.split('*')[0]
+        ret = []
+        for key in MockConnector.data.keys():
+            if match in key:
+                ret.append(key)
+
+        return ret
+
+    def get_all(self, db_id, key):
+        return MockConnector.data[key]

src/sonic-host-services/tests/procdockerstatsd_test.py

@@ -0,0 +1,42 @@
+import imp
+import sys
+import os
+import pytest
+
+import swsssdk
+
+from .mock_connector import MockConnector
+
+swsssdk.SonicV2Connector = MockConnector
+
+test_path = os.path.dirname(os.path.abspath(__file__))
+modules_path = os.path.dirname(test_path)
+scripts_path = os.path.join(modules_path, "scripts")
+sys.path.insert(0, modules_path)
+
+imp.load_source('procdockerstatsd', scripts_path + '/procdockerstatsd')
+from procdockerstatsd import *
+
+class TestProcDockerStatsDaemon(object):
+    def test_convert_to_bytes(self):
+        test_data = [
+            ('1B', 1),
+            ('500B', 500),
+            ('1KB', 1000),
+            ('500KB', 500000),
+            ('1MB', 1000000),
+            ('500MB', 500000000),
+            ('1MiB', 1048576),
+            ('500MiB', 524288000),
+            ('66.41MiB', 69635932),
+            ('333.6MiB', 349804954),
+            ('1GiB', 1073741824),
+            ('500GiB', 536870912000),
+            ('7.751GiB', 8322572878)
+        ]
+
+        pdstatsd = ProcDockerStats(SYSLOG_IDENTIFIER)
+
+        for test_input, expected_output in test_data:
+            res = pdstatsd.convert_to_bytes(test_input)
+            assert res == expected_output