From 159e69f3464450f0e4eeaf3f5416cc4f5560a0dd Mon Sep 17 00:00:00 2001 From: Prabhu Sreenivasan Date: Tue, 18 Jun 2019 21:03:47 -0700 Subject: [PATCH 01/22] Added sonic-mgmt-framework as submodule / docker --- .gitmodules | 3 ++ .../docker-sonic-mgmt-framework/Dockerfile.j2 | 34 +++++++++++++++++++ .../rest-server.sh | 7 ++++ dockers/docker-sonic-mgmt-framework/start.sh | 10 ++++++ .../supervisord.conf | 28 +++++++++++++++ .../build_templates/mgmt-framework.service.j2 | 14 ++++++++ rules/docker-sonic-mgmt-framework.mk | 22 ++++++++++++ rules/sonic-mgmt-framework.mk | 5 +++ src/sonic-mgmt-framework | 1 + 9 files changed, 124 insertions(+) create mode 100644 dockers/docker-sonic-mgmt-framework/Dockerfile.j2 create mode 100755 dockers/docker-sonic-mgmt-framework/rest-server.sh create mode 100755 dockers/docker-sonic-mgmt-framework/start.sh create mode 100644 dockers/docker-sonic-mgmt-framework/supervisord.conf create mode 100644 files/build_templates/mgmt-framework.service.j2 create mode 100644 rules/docker-sonic-mgmt-framework.mk create mode 100644 rules/sonic-mgmt-framework.mk create mode 160000 src/sonic-mgmt-framework diff --git a/.gitmodules b/.gitmodules index 3d2a8e3f7a1d..908b70834d51 100644 --- a/.gitmodules +++ b/.gitmodules @@ -66,3 +66,6 @@ [submodule "platform/mellanox/mlnx-sai/SAI-Implementation"] path = platform/mellanox/mlnx-sai/SAI-Implementation url = https://github.com/Mellanox/SAI-Implementation +[submodule "src/sonic-mgmt-framework"] + path = src/sonic-mgmt-framework + url = https://github.com/project-arlo/sonic-mgmt-framework.git diff --git a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 new file mode 100644 index 000000000000..a181020d6b43 --- /dev/null +++ b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 @@ -0,0 +1,34 @@ +FROM docker-config-engine-stretch + +ARG docker_container_name +RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf + +## Make apt-get non-interactive +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update + +#RUN apt-get install -f -y libdbus-1-3 libdaemon0 libjansson4 + +## Install redis-tools dependencies +## TODO: implicitly install dependencies +RUN apt-get -y install libjemalloc1 + +COPY \ +{% for deb in docker_sonic_mgmt_framework_debs.split(' ') -%} +debs/{{ deb }}{{' '}} +{%- endfor -%} +debs/ + +RUN dpkg -i \ +{% for deb in docker_sonic_mgmt_framework_debs.split(' ') -%} +debs/{{ deb }}{{' '}} +{%- endfor %} + +COPY ["start.sh", "rest-server.sh", "/usr/bin/"] +COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] + +RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y +RUN rm -rf /debs + +ENTRYPOINT ["/usr/bin/supervisord"] diff --git a/dockers/docker-sonic-mgmt-framework/rest-server.sh b/dockers/docker-sonic-mgmt-framework/rest-server.sh new file mode 100755 index 000000000000..ebc4193a39c4 --- /dev/null +++ b/dockers/docker-sonic-mgmt-framework/rest-server.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +REST_SERVER_ARGS=" -ui /rest_ui -port=80 -logtostderr" + +exec /usr/sbin/rest_server ${REST_SERVER_ARGS} + + diff --git a/dockers/docker-sonic-mgmt-framework/start.sh b/dockers/docker-sonic-mgmt-framework/start.sh new file mode 100755 index 000000000000..24d355670e87 --- /dev/null +++ b/dockers/docker-sonic-mgmt-framework/start.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +mkdir -p /var/sonic +echo "# Config files managed by sonic-config-engine" > /var/sonic/config_status + +rm -f /var/run/rsyslogd.pid + +supervisorctl start rsyslogd + +supervisorctl start rest-server diff --git a/dockers/docker-sonic-mgmt-framework/supervisord.conf b/dockers/docker-sonic-mgmt-framework/supervisord.conf new file mode 100644 index 000000000000..e26f815f5fb9 --- /dev/null +++ b/dockers/docker-sonic-mgmt-framework/supervisord.conf @@ -0,0 +1,28 @@ +[supervisord] +logfile_maxbytes=1MB +logfile_backups=2 +nodaemon=true + +[program:start.sh] +command=/usr/bin/start.sh +priority=1 +autostart=true +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:rsyslogd] +command=/usr/sbin/rsyslogd -n +priority=2 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:rest-server] +command=/usr/bin/rest-server.sh +priority=3 +autostart=false +autorestart=true +stdout_logfile=syslog +stderr_logfile=syslog diff --git a/files/build_templates/mgmt-framework.service.j2 b/files/build_templates/mgmt-framework.service.j2 new file mode 100644 index 000000000000..d0a030347b51 --- /dev/null +++ b/files/build_templates/mgmt-framework.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Management Framework container +Requires=swss.service +After=swss.service +Before=ntp-config.service + +[Service] +User={{ sonicadmin_user }} +ExecStartPre=/usr/bin/{{docker_container_name}}.sh start +ExecStart=/usr/bin/{{docker_container_name}}.sh wait +ExecStop=/usr/bin/{{docker_container_name}}.sh stop + +[Install] +WantedBy=multi-user.target diff --git a/rules/docker-sonic-mgmt-framework.mk b/rules/docker-sonic-mgmt-framework.mk new file mode 100644 index 000000000000..f963a77251ad --- /dev/null +++ b/rules/docker-sonic-mgmt-framework.mk @@ -0,0 +1,22 @@ +# docker image for mgmt-framework + +DOCKER_MGMT-FRAMEWORK_STEM = docker-sonic-mgmt-framework +DOCKER_MGMT-FRAMEWORK = $(DOCKER_MGMT-FRAMEWORK_STEM).gz +DOCKER_MGMT-FRAMEWORK_DBG = $(DOCKER_MGMT-FRAMEWORK_STEM)-$(DBG_IMAGE_MARK).gz + +$(DOCKER_MGMT-FRAMEWORK)_PATH = $(DOCKERS_PATH)/$(DOCKER_MGMT-FRAMEWORK_STEM) + +$(DOCKER_MGMT-FRAMEWORK)_DEPENDS += $(REDIS_TOOLS) $(SONIC_MGMT-FRAMEWORK) +$(DOCKER_MGMT-FRAMEWORK)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_DEPENDS) + +SONIC_DOCKER_IMAGES += $(DOCKER_MGMT-FRAMEWORK) +$(DOCKER_MGMT-FRAMEWORK)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_STRETCH) +$(DOCKER_MGMT-FRAMEWORK)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_IMAGE_PACKAGES) + +SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_MGMT-FRAMEWORK) +SONIC_STRETCH_DOCKERS += $(DOCKER_MGMT-FRAMEWORK) + +$(DOCKER_MGMT-FRAMEWORK)_CONTAINER_NAME = mgmt-framework +$(DOCKER_MGMT-FRAMEWORK)_RUN_OPT += --net=host --privileged -t +$(DOCKER_MGMT-FRAMEWORK)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro + diff --git a/rules/sonic-mgmt-framework.mk b/rules/sonic-mgmt-framework.mk new file mode 100644 index 000000000000..c74ad5757e8b --- /dev/null +++ b/rules/sonic-mgmt-framework.mk @@ -0,0 +1,5 @@ +# SONiC mgmt-framework package + +SONIC_MGMT-FRAMEWORK = sonic-mgmt-framework_1.0-01_amd64.deb +$(SONIC_MGMT-FRAMEWORK)_SRC_PATH = $(SRC_PATH)/sonic-mgmt-framework +SONIC_DPKG_DEBS += $(SONIC_MGMT-FRAMEWORK) diff --git a/src/sonic-mgmt-framework b/src/sonic-mgmt-framework new file mode 160000 index 000000000000..6dd9125f1f0a --- /dev/null +++ b/src/sonic-mgmt-framework @@ -0,0 +1 @@ +Subproject commit 6dd9125f1f0a5dc950857676239497b885b17b3e From 3a8ef73d0e9a39db42510a08ff2896b52afc590b Mon Sep 17 00:00:00 2001 From: Prabhu Sreenivasan Date: Wed, 19 Jun 2019 11:01:06 -0700 Subject: [PATCH 02/22] fix build issues --- dockers/docker-sonic-mgmt-framework/Dockerfile.j2 | 2 +- rules/docker-sonic-mgmt-framework.mk | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 index a181020d6b43..c38e0b4d2970 100644 --- a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 +++ b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 @@ -1,4 +1,4 @@ -FROM docker-config-engine-stretch +FROM {{ docker_sonic_mgmt_framework_load_image }} ARG docker_container_name RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf diff --git a/rules/docker-sonic-mgmt-framework.mk b/rules/docker-sonic-mgmt-framework.mk index f963a77251ad..8e973da313b4 100644 --- a/rules/docker-sonic-mgmt-framework.mk +++ b/rules/docker-sonic-mgmt-framework.mk @@ -7,11 +7,11 @@ DOCKER_MGMT-FRAMEWORK_DBG = $(DOCKER_MGMT-FRAMEWORK_STEM)-$(DBG_IMAGE_MARK).gz $(DOCKER_MGMT-FRAMEWORK)_PATH = $(DOCKERS_PATH)/$(DOCKER_MGMT-FRAMEWORK_STEM) $(DOCKER_MGMT-FRAMEWORK)_DEPENDS += $(REDIS_TOOLS) $(SONIC_MGMT-FRAMEWORK) -$(DOCKER_MGMT-FRAMEWORK)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_DEPENDS) +#$(DOCKER_MGMT-FRAMEWORK)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_DEPENDS) SONIC_DOCKER_IMAGES += $(DOCKER_MGMT-FRAMEWORK) $(DOCKER_MGMT-FRAMEWORK)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_STRETCH) -$(DOCKER_MGMT-FRAMEWORK)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_IMAGE_PACKAGES) +#$(DOCKER_MGMT-FRAMEWORK)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_IMAGE_PACKAGES) SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_MGMT-FRAMEWORK) SONIC_STRETCH_DOCKERS += $(DOCKER_MGMT-FRAMEWORK) From 477f233ca6b91592da555af85e9d7d642967505c Mon Sep 17 00:00:00 2001 From: Prabhu Sreenivasan Date: Wed, 19 Jun 2019 12:04:36 -0700 Subject: [PATCH 03/22] update sonic-mgmt-framework submodule branch to master --- src/sonic-mgmt-framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-mgmt-framework b/src/sonic-mgmt-framework index 6dd9125f1f0a..13c9cf139a8e 160000 --- a/src/sonic-mgmt-framework +++ b/src/sonic-mgmt-framework @@ -1 +1 @@ -Subproject commit 6dd9125f1f0a5dc950857676239497b885b17b3e +Subproject commit 13c9cf139a8ee438b2f62ccd54bcc4b3762bcb9b From 3d804a49d7dfe65eefd3c21585264a6aa9329df6 Mon Sep 17 00:00:00 2001 From: Partha Dutta Date: Fri, 21 Jun 2019 23:55:09 -0700 Subject: [PATCH 04/22] Merged changes 70007e6d2ba3a4c0b371cd693ccc63e0a8906e77..00d4fcfed6a759e40d7b92120ea0ee1f08300fc6 00d4fcfed6a759e40d7b92120ea0ee1f08300fc6 Modified environemnt variables --- dockers/docker-sonic-mgmt-framework/rest-server.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dockers/docker-sonic-mgmt-framework/rest-server.sh b/dockers/docker-sonic-mgmt-framework/rest-server.sh index ebc4193a39c4..0fec2c7fa2a6 100755 --- a/dockers/docker-sonic-mgmt-framework/rest-server.sh +++ b/dockers/docker-sonic-mgmt-framework/rest-server.sh @@ -2,6 +2,11 @@ REST_SERVER_ARGS=" -ui /rest_ui -port=80 -logtostderr" -exec /usr/sbin/rest_server ${REST_SERVER_ARGS} - +#export LIBYANG_EXTENSIONS_PLUGINS_DIR +export CVL_SCHEMA_PATH=/usr/sbin/schema +export LIBYANG_EXTENSIONS_PLUGINS_DIR=/usr/sbin/lib +export LIBYANG_USER_TYPES_PLUGINS_DIR=/usr/sbin/lib +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PATH:/usr/sbin/lib +exec /usr/sbin/rest_server ${REST_SERVER_ARGS} +#while [ true ]; do sleep 1; done From efd773fb5376a03989c7e6b32548deaf8833424d Mon Sep 17 00:00:00 2001 From: Srideep Devireddy Date: Tue, 25 Jun 2019 20:24:09 -0400 Subject: [PATCH 05/22] Changes to build sonic-mgmt-framework docker --- dockers/docker-sonic-mgmt-framework/Dockerfile.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 index c38e0b4d2970..a181020d6b43 100644 --- a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 +++ b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 @@ -1,4 +1,4 @@ -FROM {{ docker_sonic_mgmt_framework_load_image }} +FROM docker-config-engine-stretch ARG docker_container_name RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf From b4d36e7678655ef76fd8c51a297e192f016b76bb Mon Sep 17 00:00:00 2001 From: Prabhu Sreenivasan Date: Tue, 25 Jun 2019 21:50:28 -0700 Subject: [PATCH 06/22] bumped up sonic-mgmt-framework commit-id --- src/sonic-mgmt-framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-mgmt-framework b/src/sonic-mgmt-framework index 13c9cf139a8e..0091f93bedde 160000 --- a/src/sonic-mgmt-framework +++ b/src/sonic-mgmt-framework @@ -1 +1 @@ -Subproject commit 13c9cf139a8ee438b2f62ccd54bcc4b3762bcb9b +Subproject commit 0091f93beddeda7914b922ab036c35a36aaa9d93 From e292c98446c4438694e2a58e3c9c6a12a756b36c Mon Sep 17 00:00:00 2001 From: Srideep Devireddy Date: Thu, 27 Jun 2019 16:02:49 -0400 Subject: [PATCH 07/22] version bump for sonic-mgmt-framework commit-it --- src/sonic-mgmt-framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-mgmt-framework b/src/sonic-mgmt-framework index 0091f93bedde..74d2c011b5f1 160000 --- a/src/sonic-mgmt-framework +++ b/src/sonic-mgmt-framework @@ -1 +1 @@ -Subproject commit 0091f93beddeda7914b922ab036c35a36aaa9d93 +Subproject commit 74d2c011b5f135fb9a47179fbd62c8f5e2ae60a9 From 387b8b0b3d4889aa2713cfcf9fadc5bd5ba49b27 Mon Sep 17 00:00:00 2001 From: Srideep Devireddy Date: Thu, 27 Jun 2019 20:29:03 -0400 Subject: [PATCH 08/22] bumped up sonic-mgmt-framework commit-id --- src/sonic-mgmt-framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-mgmt-framework b/src/sonic-mgmt-framework index 74d2c011b5f1..00d9f3ad8aa4 160000 --- a/src/sonic-mgmt-framework +++ b/src/sonic-mgmt-framework @@ -1 +1 @@ -Subproject commit 74d2c011b5f135fb9a47179fbd62c8f5e2ae60a9 +Subproject commit 00d9f3ad8aa4272a50ad59dafa245a9cf9c6a246 From a825f7e601b7b42a8216fac982bf467b0626b2a4 Mon Sep 17 00:00:00 2001 From: Srideep Devireddy Date: Thu, 27 Jun 2019 20:30:55 -0400 Subject: [PATCH 09/22] Add python packages to docker --- dockers/docker-sonic-mgmt-framework/Dockerfile.j2 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 index a181020d6b43..836cc6107ab0 100644 --- a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 +++ b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 @@ -7,6 +7,16 @@ RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%s ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update +RUN pip install connexion==1.1.15 \ + python_dateutil==2.6.0\ + setuptools >= 21.0.0 \ + grpcio-tools == 1.20.0 \ + pyangbind == 0.8.1 \ + certifi==2017.4.17 \ + python-dateutil==2.6.0 \ + six==1.8.0 \ + urllib3==1.21.1 + #RUN apt-get install -f -y libdbus-1-3 libdaemon0 libjansson4 From 8e4b212694e57dfed8f57d35a8134d0dac7721a2 Mon Sep 17 00:00:00 2001 From: Srideep Devireddy Date: Thu, 27 Jun 2019 21:13:23 -0400 Subject: [PATCH 10/22] Build fix for docker with python packages --- dockers/docker-sonic-mgmt-framework/Dockerfile.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 index 836cc6107ab0..cc8ca47ccf67 100644 --- a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 +++ b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 @@ -9,12 +9,12 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update RUN pip install connexion==1.1.15 \ python_dateutil==2.6.0\ - setuptools >= 21.0.0 \ - grpcio-tools == 1.20.0 \ - pyangbind == 0.8.1 \ + setuptools==21.0.0 \ + grpcio-tools==1.20.0 \ + pyangbind==0.6.0 \ certifi==2017.4.17 \ python-dateutil==2.6.0 \ - six==1.8.0 \ + six==1.11.0 \ urllib3==1.21.1 From bc95cef5e9a88aeafd5caf6c8fddf35c96b7493f Mon Sep 17 00:00:00 2001 From: Prabhu Sreenivasan Date: Tue, 9 Jul 2019 11:35:07 -0700 Subject: [PATCH 11/22] added libyang as dependent package --- rules/sonic-mgmt-framework.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rules/sonic-mgmt-framework.mk b/rules/sonic-mgmt-framework.mk index c74ad5757e8b..42da60c6c716 100644 --- a/rules/sonic-mgmt-framework.mk +++ b/rules/sonic-mgmt-framework.mk @@ -2,4 +2,6 @@ SONIC_MGMT-FRAMEWORK = sonic-mgmt-framework_1.0-01_amd64.deb $(SONIC_MGMT-FRAMEWORK)_SRC_PATH = $(SRC_PATH)/sonic-mgmt-framework +$(SONIC_MGMT-FRAMEWORK)_DEPENDS = $(LIBYANG_DEV) $(LIBYANG) +$(SONIC_MGMT-FRAMEWORK)_RDEPENDS = $(LIBYANG) SONIC_DPKG_DEBS += $(SONIC_MGMT-FRAMEWORK) From de3678191684ca96a9a70c29a427d8c44d3b0b57 Mon Sep 17 00:00:00 2001 From: Nirenjan Krishnan Date: Tue, 9 Jul 2019 15:32:01 -0700 Subject: [PATCH 12/22] Allow building images on NFS-mounted clones Prior to this change, `build_debian.sh` would generate a Debian filesystem in `./fsroot`. This needs root permissions, and one of the tests that is performed is whether the user can create a character special file in the filesystem (using mknod). On most NFS deployments, `root` is the least privileged user, and cannot run mknod. Also, attempting to run commands like rm or mv as root would fail due to permission errors, since the root user gets mapped to an unprivileged user like `nobody`. This commit changes the location of the Debian filesystem to `/fsroot`, which is a tmpfs mount within the slave Docker. The default squashfs, docker tarball and zip files are also created within /tmp, before being copied back to /sonic as the regular user. The side effect of this change is that the contents of `/fsroot` are no longer available once the slave container exits, however they are available within the squashfs image. Signed-off-by: Nirenjan Krishnan --- Makefile.work | 1 + build_debian.sh | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Makefile.work b/Makefile.work index 88780724ea76..5ca59d66f2ba 100644 --- a/Makefile.work +++ b/Makefile.work @@ -80,6 +80,7 @@ endif DOCKER_RUN := docker run --rm=true --privileged \ -v $(DOCKER_BUILDER_MOUNT) \ -w $(DOCKER_BUILDER_WORKDIR) \ + --tmpfs /fsroot:exec,dev \ -e "http_proxy=$(http_proxy)" \ -e "https_proxy=$(https_proxy)" \ -i$(if $(TERM),t,) diff --git a/build_debian.sh b/build_debian.sh index 44c3524ed96b..46a8e0d62e58 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -33,7 +33,7 @@ DOCKER_VERSION=5:18.09.2~3-0~debian-stretch LINUX_KERNEL_VERSION=4.9.0-9-2 ## Working directory to prepare the file system -FILESYSTEM_ROOT=./fsroot +FILESYSTEM_ROOT=/fsroot PLATFORM_DIR=platform ## Hostname for the linux image HOSTNAME=sonic @@ -56,9 +56,8 @@ DEFAULT_USERINFO="Default admin user,,," ## Prepare the file system directory if [[ -d $FILESYSTEM_ROOT ]]; then - sudo rm -rf $FILESYSTEM_ROOT || die "Failed to clean chroot directory" + sudo chown $(id -nu):$(id -ng) $FILESYSTEM_ROOT || die "Failed to own chroot directory" fi -mkdir -p $FILESYSTEM_ROOT mkdir -p $FILESYSTEM_ROOT/$PLATFORM_DIR mkdir -p $FILESYSTEM_ROOT/$PLATFORM_DIR/x86_64-grub touch $FILESYSTEM_ROOT/$PLATFORM_DIR/firsttime @@ -408,8 +407,7 @@ fi ## Organization specific extensions such as Configuration & Scripts for features like AAA, ZTP... if [ "${enable_organization_extensions}" = "y" ]; then - if [ -f files/build_templates/organization_extensions.sh ]; then - sudo chmod 755 files/build_templates/organization_extensions.sh + if [ -x files/build_templates/organization_extensions.sh ]; then ./files/build_templates/organization_extensions.sh -f $FILESYSTEM_ROOT -h $HOSTNAME fi fi @@ -447,16 +445,19 @@ sudo umount $FILESYSTEM_ROOT/proc || true sudo mkdir $FILESYSTEM_ROOT/host ## Compress most file system into squashfs file -sudo rm -f $ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS +rm -f $ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS ## Output the file system total size for diag purpose ## Note: -x to skip directories on different file systems, such as /proc sudo du -hsx $FILESYSTEM_ROOT sudo mkdir -p $FILESYSTEM_ROOT/var/lib/docker -sudo mksquashfs $FILESYSTEM_ROOT $FILESYSTEM_SQUASHFS -e boot -e var/lib/docker -e $PLATFORM_DIR +sudo mksquashfs $FILESYSTEM_ROOT /tmp/$FILESYSTEM_SQUASHFS -e boot -e var/lib/docker -e $PLATFORM_DIR +cp /tmp/$FILESYSTEM_SQUASHFS $FILESYSTEM_SQUASHFS ## Compress docker files -pushd $FILESYSTEM_ROOT && sudo tar czf $OLDPWD/$FILESYSTEM_DOCKERFS -C var/lib/docker .; popd +pushd $FILESYSTEM_ROOT && sudo tar czf /tmp/$FILESYSTEM_DOCKERFS -C var/lib/docker .; popd +cp /tmp/$FILESYSTEM_DOCKERFS $FILESYSTEM_DOCKERFS ## Compress together with /boot, /var/lib/docker and $PLATFORM_DIR as an installer payload zip file -pushd $FILESYSTEM_ROOT && sudo zip $OLDPWD/$ONIE_INSTALLER_PAYLOAD -r boot/ $PLATFORM_DIR/; popd -sudo zip -g $ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS $FILESYSTEM_DOCKERFS +pushd $FILESYSTEM_ROOT && sudo zip /tmp/$ONIE_INSTALLER_PAYLOAD -r boot/ $PLATFORM_DIR/; popd +sudo zip -g /tmp/$ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS $FILESYSTEM_DOCKERFS +cp /tmp/$ONIE_INSTALLER_PAYLOAD $ONIE_INSTALLER_PAYLOAD From c7ecb4fdafe62b295b243ef217269759db5a4daa Mon Sep 17 00:00:00 2001 From: Prabhu Sreenivasan Date: Tue, 9 Jul 2019 22:30:46 -0700 Subject: [PATCH 13/22] bumped up sonc-mgmt-framework commit to include PR #18 --- src/sonic-mgmt-framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-mgmt-framework b/src/sonic-mgmt-framework index 00d9f3ad8aa4..0f1e1d9d1cee 160000 --- a/src/sonic-mgmt-framework +++ b/src/sonic-mgmt-framework @@ -1 +1 @@ -Subproject commit 00d9f3ad8aa4272a50ad59dafa245a9cf9c6a246 +Subproject commit 0f1e1d9d1cee78d76074c8fa2430dad310c8cb4d From ef7d06b317771027dc5149ad8eaef8fc84f746d2 Mon Sep 17 00:00:00 2001 From: Prabhu Sreenivasan Date: Wed, 10 Jul 2019 01:25:46 -0700 Subject: [PATCH 14/22] REST Server startup script is enahnced to read the settings from ConfigDB. Below table provides mapping of db field to command line argument name. ============================================================ ConfigDB entry key Field name REST Server argument ============================================================ REST_SERVER|default port -port REST_SERVER|default client_auth -client_auth REST_SERVER|default log_level -v DEVICE_METADATA|x509 server_crt -cert DEVICE_METADATA|x509 server_key -key DEVICE_METADATA|x509 ca_crt -cacert ============================================================ --- .../rest-server.sh | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/dockers/docker-sonic-mgmt-framework/rest-server.sh b/dockers/docker-sonic-mgmt-framework/rest-server.sh index 0fec2c7fa2a6..b32552b419ec 100755 --- a/dockers/docker-sonic-mgmt-framework/rest-server.sh +++ b/dockers/docker-sonic-mgmt-framework/rest-server.sh @@ -1,6 +1,49 @@ #!/usr/bin/env bash -REST_SERVER_ARGS=" -ui /rest_ui -port=80 -logtostderr" +# Startup script for SONiC Management REST Server + +SERVER_PORT= +LOG_LEVEL= +CLIENT_AUTH= +SERVER_CRT= +SERVER_KEY= +CA_CERT= + +# Read basic server settings from REST_SERVER|default entry +HAS_REST_CONFIG=$(sonic-cfggen -d -v "1 if REST_SERVER and REST_SERVER['default']") +if [ "$HAS_REST_CONFIG" == "1" ]; then + SERVER_PORT=$(sonic-cfggen -d -v "REST_SERVER['default']['port']") + CLIENT_AUTH=$(sonic-cfggen -d -v "REST_SERVER['default']['client_auth']") + LOG_LEVEL=$(sonic-cfggen -d -v "REST_SERVER['default']['log_level']") +fi + +# Read certificate file paths from DEVICE_METADATA|x509 entry. +HAS_X509_CONFIG=$(sonic-cfggen -d -v "1 if DEVICE_METADATA and DEVICE_METADATA['x509']") +if [ "$HAS_X509_CONFIG" == "1" ]; then + SERVER_CRT=$(sonic-cfggen -d -v "DEVICE_METADATA['x509']['server_crt']") + SERVER_KEY=$(sonic-cfggen -d -v "DEVICE_METADATA['x509']['server_key']") + CA_CRT=$(sonic-cfggen -d -v "DEVICE_METADATA['x509']['ca_crt']") +fi + +# Create temporary server certificate if they not configured in ConfigDB +if [ -z $SERVER_CRT ] && [ -z $SERVER_KEY ]; then + echo "Generating temporary TLS server certificate ..." + (cd /tmp && /usr/sbin/generate_cert --host="localhost,127.0.0.1") + SERVER_CRT=/tmp/cert.pem + SERVER_KEY=/tmp/key.pem +fi + + +REST_SERVER_ARGS="-ui /rest_ui -logtostderr" +[ ! -z $SERVER_PORT ] && REST_SERVER_ARGS+=" -port $SERVER_PORT" +[ ! -z $LOG_LEVEL ] && REST_SERVER_ARGS+=" -v $LOG_LEVEL" +[ ! -z $CLIENT_AUTH ] && REST_SERVER_ARGS+=" -client_auth $CLIENT_AUTH" +[ ! -z $SERVER_CRT ] && REST_SERVER_ARGS+=" -cert $SERVER_CRT" +[ ! -z $SERVER_KEY ] && REST_SERVER_ARGS+=" -key $SERVER_KEY" +[ ! -z $CA_CRT ] && REST_SERVER_ARGS+=" -cacert $CA_CRT" + +echo "REST_SERVER_ARGS = $REST_SERVER_ARGS" + #export LIBYANG_EXTENSIONS_PLUGINS_DIR export CVL_SCHEMA_PATH=/usr/sbin/schema From 11c31929759a17122782d4944066a6ac8453b78d Mon Sep 17 00:00:00 2001 From: Nirenjan Krishnan Date: Wed, 10 Jul 2019 12:41:50 -0700 Subject: [PATCH 15/22] Replace src/telemetry as submodule to sonic-telemetry --- .gitmodules | 3 +++ src/telemetry | 1 + src/telemetry/Makefile | 20 -------------------- src/telemetry/debian/changelog | 5 ----- src/telemetry/debian/compat | 1 - src/telemetry/debian/control | 17 ----------------- src/telemetry/debian/rules | 3 --- src/telemetry/debian/telemetry.init.d | 14 -------------- 8 files changed, 4 insertions(+), 60 deletions(-) create mode 160000 src/telemetry delete mode 100644 src/telemetry/Makefile delete mode 100644 src/telemetry/debian/changelog delete mode 100644 src/telemetry/debian/compat delete mode 100644 src/telemetry/debian/control delete mode 100755 src/telemetry/debian/rules delete mode 100644 src/telemetry/debian/telemetry.init.d diff --git a/.gitmodules b/.gitmodules index 908b70834d51..f82acbeeb1da 100644 --- a/.gitmodules +++ b/.gitmodules @@ -69,3 +69,6 @@ [submodule "src/sonic-mgmt-framework"] path = src/sonic-mgmt-framework url = https://github.com/project-arlo/sonic-mgmt-framework.git +[submodule "src/telemetry"] + path = src/telemetry + url = https://github.com/project-arlo/sonic-telemetry.git diff --git a/src/telemetry b/src/telemetry new file mode 160000 index 000000000000..2c9b88da22b5 --- /dev/null +++ b/src/telemetry @@ -0,0 +1 @@ +Subproject commit 2c9b88da22b58006ad9257b80ff57a74cda3c216 diff --git a/src/telemetry/Makefile b/src/telemetry/Makefile deleted file mode 100644 index 91a822d4f93c..000000000000 --- a/src/telemetry/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -export GOPATH=/tmp/go - -INSTALL := /usr/bin/install - -all: sonic-telemetry - -sonic-telemetry: - /usr/local/go/bin/go get -v github.com/Azure/sonic-telemetry/telemetry - /usr/local/go/bin/go get -v github.com/Azure/sonic-telemetry/dialout/dialout_client_cli - -install: - $(INSTALL) -D ${GOPATH}/bin/telemetry $(DESTDIR)/usr/sbin/telemetry - $(INSTALL) -D ${GOPATH}/bin/dialout_client_cli $(DESTDIR)/usr/sbin/dialout_client_cli - -deinstall: - rm $(DESTDIR)/usr/sbin/telemetry - rm $(DESTDIR)/usr/sbin/dialout_client_cli - -clean: - rm -fr ${GOPATH} diff --git a/src/telemetry/debian/changelog b/src/telemetry/debian/changelog deleted file mode 100644 index 77dab629040e..000000000000 --- a/src/telemetry/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -sonic-telemetry (0.1) UNRELEASED; urgency=medium - - * Initial release. - - -- Jipan Yang Sat, 24 Mar 2018 12:48:22 -0700 diff --git a/src/telemetry/debian/compat b/src/telemetry/debian/compat deleted file mode 100644 index ec635144f600..000000000000 --- a/src/telemetry/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/src/telemetry/debian/control b/src/telemetry/debian/control deleted file mode 100644 index f1b6a7c07282..000000000000 --- a/src/telemetry/debian/control +++ /dev/null @@ -1,17 +0,0 @@ -Source: sonic-telemetry -Section: devel -Priority: optional -Maintainer: Jipan Yang -Build-Depends: debhelper (>= 8.0.0), - dh-systemd -Standards-Version: 3.9.3 -Homepage: https://github.com/Azure/sonic-telemetry -XS-Go-Import-Path: github.com/Azure/sonic-telemetry - -Package: sonic-telemetry -Architecture: any -Built-Using: ${misc:Built-Using} -Depends: ${misc:Depends}, - ${shlibs:Depends} -Description: SONiC telemetry - sonic-telemetry \ No newline at end of file diff --git a/src/telemetry/debian/rules b/src/telemetry/debian/rules deleted file mode 100755 index 3995a26d7fcd..000000000000 --- a/src/telemetry/debian/rules +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/make -f -%: - dh $@ --with systemd diff --git a/src/telemetry/debian/telemetry.init.d b/src/telemetry/debian/telemetry.init.d deleted file mode 100644 index 2fea32e17c49..000000000000 --- a/src/telemetry/debian/telemetry.init.d +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# -### BEGIN INIT INFO -# Provides: sonic-telemetry -# Required-Start: $local_fs $network $remote_fs $syslog -# Required-Stop: $local_fs $network $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: start and stop the telemetry -# Description: sonic-telemetry is an implementation of sonic telemetry daemon in Go -### END INIT INFO -# - -exit 0 From 4c5b82431a6bb095d840c8a3365ecdb3ef212fe6 Mon Sep 17 00:00:00 2001 From: Nirenjan Krishnan Date: Wed, 10 Jul 2019 14:04:35 -0700 Subject: [PATCH 16/22] Update telemetry commit HEAD --- src/telemetry | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/telemetry b/src/telemetry index 2c9b88da22b5..c98c346172d0 160000 --- a/src/telemetry +++ b/src/telemetry @@ -1 +1 @@ -Subproject commit 2c9b88da22b58006ad9257b80ff57a74cda3c216 +Subproject commit c98c346172d02aa77d8ffd6337004b8e0050e730 From ff0e23a6fa5fc74f48acf75a9dc8a96ca7f210d5 Mon Sep 17 00:00:00 2001 From: Nirenjan Krishnan Date: Wed, 10 Jul 2019 15:15:35 -0700 Subject: [PATCH 17/22] Update sonic-telemetry commit HEAD --- src/telemetry | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/telemetry b/src/telemetry index c98c346172d0..dd18f0dd8c12 160000 --- a/src/telemetry +++ b/src/telemetry @@ -1 +1 @@ -Subproject commit c98c346172d02aa77d8ffd6337004b8e0050e730 +Subproject commit dd18f0dd8c1251d7d0bb054a8db32f7d54cd9b9b From 7597470795f8d2a376825e3a80035e918e435cb7 Mon Sep 17 00:00:00 2001 From: Prabhu Sreenivasan Date: Wed, 10 Jul 2019 22:09:43 -0700 Subject: [PATCH 18/22] libyang env path update --- dockers/docker-sonic-mgmt-framework/rest-server.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dockers/docker-sonic-mgmt-framework/rest-server.sh b/dockers/docker-sonic-mgmt-framework/rest-server.sh index b32552b419ec..f2a29c9b1ed2 100755 --- a/dockers/docker-sonic-mgmt-framework/rest-server.sh +++ b/dockers/docker-sonic-mgmt-framework/rest-server.sh @@ -45,11 +45,8 @@ REST_SERVER_ARGS="-ui /rest_ui -logtostderr" echo "REST_SERVER_ARGS = $REST_SERVER_ARGS" -#export LIBYANG_EXTENSIONS_PLUGINS_DIR export CVL_SCHEMA_PATH=/usr/sbin/schema -export LIBYANG_EXTENSIONS_PLUGINS_DIR=/usr/sbin/lib -export LIBYANG_USER_TYPES_PLUGINS_DIR=/usr/sbin/lib -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PATH:/usr/sbin/lib +export LIBYANG_EXTENSIONS_PLUGINS_DIR=/usr/lib/x86_64-linux-gnu/libyang/extensions +export LIBYANG_USER_TYPES_PLUGINS_DIR=/usr/lib/x86_64-linux-gnu/libyang/user_types exec /usr/sbin/rest_server ${REST_SERVER_ARGS} -#while [ true ]; do sleep 1; done From 9385bc4f49b8c177b6d888f94765fd57a39ba626 Mon Sep 17 00:00:00 2001 From: Eric Seifert Date: Fri, 12 Jul 2019 10:34:10 -0700 Subject: [PATCH 19/22] Add libyang dependency to telemetry --- rules/telemetry.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rules/telemetry.mk b/rules/telemetry.mk index a160bf50899c..a5179b438ce0 100644 --- a/rules/telemetry.mk +++ b/rules/telemetry.mk @@ -2,4 +2,6 @@ SONIC_TELEMETRY = sonic-telemetry_0.1_amd64.deb $(SONIC_TELEMETRY)_SRC_PATH = $(SRC_PATH)/telemetry +$(SONIC_TELEMETRY)_DEPENDS = $(LIBYANG_DEV) $(LIBYANG) +$(SONIC_TELEMETRY)_RDEPENDS = $(LIBYANG) SONIC_DPKG_DEBS += $(SONIC_TELEMETRY) From 1430d49824b9752e8d65cb45657c96256ec49c01 Mon Sep 17 00:00:00 2001 From: Howard Persh Date: Fri, 12 Jul 2019 10:43:53 -0700 Subject: [PATCH 20/22] Add scripts to create JSON files for CLI backend Scripts to create /var/platform/syseeprom and /var/platform/system, which are back-end files for CLI, for system EEPROM and system information. Signed-off-by: Howard Persh --- .gitmodules | 2 +- files/image_config/platform/rc.local | 6 ++++++ src/sonic-utilities | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index f82acbeeb1da..7419536b2650 100644 --- a/.gitmodules +++ b/.gitmodules @@ -34,7 +34,7 @@ url = https://github.com/p4lang/ptf.git [submodule "src/sonic-utilities"] path = src/sonic-utilities - url = https://github.com/Azure/sonic-utilities + url = https://github.com/project-arlo/sonic-utilities.git [submodule "platform/broadcom/sonic-platform-modules-arista"] path = platform/broadcom/sonic-platform-modules-arista url = https://github.com/aristanetworks/sonic diff --git a/files/image_config/platform/rc.local b/files/image_config/platform/rc.local index 3596eb18419c..ab15b949c16b 100755 --- a/files/image_config/platform/rc.local +++ b/files/image_config/platform/rc.local @@ -354,4 +354,10 @@ if [ -f $FIRST_BOOT_FILE ]; then firsttime_exit fi +# Write contents of system EEPROM as JSON file +/usr/bin/decode-syseeprom | /usr/bin/syseeprom-to-json > /var/platform/syseeprom + +# Start system information poller +/usr/bin/syspoll & + exit 0 diff --git a/src/sonic-utilities b/src/sonic-utilities index 09806b861486..bd16c4467e3c 160000 --- a/src/sonic-utilities +++ b/src/sonic-utilities @@ -1 +1 @@ -Subproject commit 09806b861486091d9db5cb75bdd2cc9428e46844 +Subproject commit bd16c4467e3c4aefecce6f545529aea7cc633d6a From e86f0a22e4d2567d32267062abbca253f6a53daa Mon Sep 17 00:00:00 2001 From: Howard Persh Date: Wed, 17 Jul 2019 10:52:45 -0700 Subject: [PATCH 21/22] In startup script, create directory where CLI back-end files live Signed-off-by: Howard Persh --- files/image_config/platform/rc.local | 3 +++ 1 file changed, 3 insertions(+) diff --git a/files/image_config/platform/rc.local b/files/image_config/platform/rc.local index ab15b949c16b..2574f6dbf760 100755 --- a/files/image_config/platform/rc.local +++ b/files/image_config/platform/rc.local @@ -354,6 +354,9 @@ if [ -f $FIRST_BOOT_FILE ]; then firsttime_exit fi +# Create dir where following scripts put their output files +mkdir -p /var/platform + # Write contents of system EEPROM as JSON file /usr/bin/decode-syseeprom | /usr/bin/syseeprom-to-json > /var/platform/syseeprom From 4125d6ac621390099aa50471b79c5f3e9c683505 Mon Sep 17 00:00:00 2001 From: Srideep Devireddy Date: Wed, 17 Jul 2019 15:00:23 -0400 Subject: [PATCH 22/22] build dependency pkgs added to docker for build failure fix --- dockers/docker-sonic-mgmt-framework/Dockerfile.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 index cc8ca47ccf67..ec8d1781ae3f 100644 --- a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 +++ b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 @@ -22,7 +22,7 @@ RUN pip install connexion==1.1.15 \ ## Install redis-tools dependencies ## TODO: implicitly install dependencies -RUN apt-get -y install libjemalloc1 +RUN apt-get -y install libjemalloc1 libatomic1 liblua5.1-0 lua-bitop lua-cjson COPY \ {% for deb in docker_sonic_mgmt_framework_debs.split(' ') -%}